{ "edition": "Bid Bench 1.1 \u2014 GPT-6.1 Sol CLI extension", "version": "1.1", "run_date": "October 13, 2026", "generated_at": "2026-10-13T19:24:24.704013+00:00", "complete": true, "publication_ready": true, "delivery_measurements_complete": true, "reviewed_with_limitations": false, "completeness_scope": "The five GPT-6.1 Sol configurations and all 630 October 13 attempts. Historical comparison rows preserve their original measurement coverage.", "reported_attempts": 630, "expected_attempts": 630, "reasoning_configurations": 5, "delivery_tasks": 30, "repeats": 3, "default_featured_model": "openai/gpt-6.1-sol", "default_metric": "tokens", "cost_mode": "subscription", "measurement_label": "Bid Bench 1.1 extension \u00b7 October 13, 2026", "measurement_basis": "Recorded subscription CLI tokens and timing; no dollar cost assigned.", "download_json": "/benchmarks/gpt-6.1-sol-bid-bench-2026-10-13.json", "download_csv": "/benchmarks/gpt-6.1-sol-bid-bench-2026-10-13.csv", "method_url": "/benchmarks/gpt-6.1-sol-bid-bench-2026-10-13-method.md", "configurations": [ { "id": "openai/gpt-6.1-sol@low", "model_id": "openai/gpt-6.1-sol", "model_label": "GPT-6.1 Sol", "effort": "low", "label": "GPT-6.1 Sol \u00b7 Low", "short_label": "GPT-6.1 Sol Low", "score": 100, "passed": 90, "attempts": 90, "tokens": 37221.955555555556, "tokens_known_attempts": 90, "cost": null, "cost_known_attempts": 0, "latency": 22.058017103991006, "ci": [ 100, 100 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0, "median_correct_stop_s": 11.318006603993126 }, "predictions": { "brier": { "metric": 0.06716778148611112, "valid_output_metric": 0.06716778148611112, "metric_complete": true, "baseline_full": 0.06768977925381477, "baseline_matched": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.540422232445835, "valid_output_metric": 0.540422232445835, "metric_complete": true, "baseline_full": 0.3567919219293358, "baseline_matched": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "codex-cli-subscription": 90 }, "run_date": "2026-10-13", "cost_basis": "Subscription usage; no dollar estimate." }, { "id": "openai/gpt-6.1-sol@medium", "model_id": "openai/gpt-6.1-sol", "model_label": "GPT-6.1 Sol", "effort": "medium", "label": "GPT-6.1 Sol \u00b7 Medium", "short_label": "GPT-6.1 Sol Medium", "score": 100, "passed": 90, "attempts": 90, "tokens": 41114.62222222222, "tokens_known_attempts": 90, "cost": null, "cost_known_attempts": 0, "latency": 25.100128208490787, "ci": [ 100, 100 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0, "median_correct_stop_s": 11.005162999994354 }, "predictions": { "brier": { "metric": 0.06718499844444445, "valid_output_metric": 0.06718499844444445, "metric_complete": true, "baseline_full": 0.06768977925381477, "baseline_matched": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5114230728956974, "valid_output_metric": 0.5114230728956974, "metric_complete": true, "baseline_full": 0.3567919219293358, "baseline_matched": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "codex-cli-subscription": 90 }, "run_date": "2026-10-13", "cost_basis": "Subscription usage; no dollar estimate." }, { "id": "openai/gpt-6.1-sol@high", "model_id": "openai/gpt-6.1-sol", "model_label": "GPT-6.1 Sol", "effort": "high", "label": "GPT-6.1 Sol \u00b7 High", "short_label": "GPT-6.1 Sol High", "score": 100, "passed": 90, "attempts": 90, "tokens": 45628.333333333336, "tokens_known_attempts": 90, "cost": null, "cost_known_attempts": 0, "latency": 28.90415856249456, "ci": [ 100, 100 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0, "median_correct_stop_s": 12.948806458487525 }, "predictions": { "brier": { "metric": 0.06713928559722222, "valid_output_metric": 0.06713928559722222, "metric_complete": true, "baseline_full": 0.06768977925381477, "baseline_matched": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.46418113551985385, "valid_output_metric": 0.46418113551985385, "metric_complete": true, "baseline_full": 0.3567919219293358, "baseline_matched": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "codex-cli-subscription": 90 }, "run_date": "2026-10-13", "cost_basis": "Subscription usage; no dollar estimate." }, { "id": "openai/gpt-6.1-sol@xhigh", "model_id": "openai/gpt-6.1-sol", "model_label": "GPT-6.1 Sol", "effort": "xhigh", "label": "GPT-6.1 Sol \u00b7 XHigh", "short_label": "GPT-6.1 Sol Xhigh", "score": 100, "passed": 90, "attempts": 90, "tokens": 50340.055555555555, "tokens_known_attempts": 90, "cost": null, "cost_known_attempts": 0, "latency": 35.301311500006705, "ci": [ 100, 100 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0, "median_correct_stop_s": 14.000537875006557 }, "predictions": { "brier": { "metric": 0.06738516875, "valid_output_metric": 0.06738516875, "metric_complete": true, "baseline_full": 0.06768977925381477, "baseline_matched": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4811065023289495, "valid_output_metric": 0.4811065023289495, "metric_complete": true, "baseline_full": 0.3567919219293358, "baseline_matched": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "codex-cli-subscription": 90 }, "run_date": "2026-10-13", "cost_basis": "Subscription usage; no dollar estimate." }, { "id": "openai/gpt-6.1-sol@max", "model_id": "openai/gpt-6.1-sol", "model_label": "GPT-6.1 Sol", "effort": "max", "label": "GPT-6.1 Sol \u00b7 Max", "short_label": "GPT-6.1 Sol Max", "score": 100, "passed": 90, "attempts": 90, "tokens": 52175.51111111111, "tokens_known_attempts": 90, "cost": null, "cost_known_attempts": 0, "latency": 39.06859858351527, "ci": [ 100, 100 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0, "median_correct_stop_s": 17.02719345848891 }, "predictions": { "brier": { "metric": 0.06750645422222222, "valid_output_metric": 0.06750645422222222, "metric_complete": true, "baseline_full": 0.06768977925381477, "baseline_matched": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.46004687567682706, "valid_output_metric": 0.46004687567682706, "metric_complete": true, "baseline_full": 0.3567919219293358, "baseline_matched": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "codex-cli-subscription": 90 }, "run_date": "2026-10-13", "cost_basis": "Subscription usage; no dollar estimate." }, { "id": "anthropic/claude-fable-5.1@high", "model_id": "anthropic/claude-fable-5.1", "model_label": "Claude Fable 5.1", "effort": "high", "label": "Claude Fable 5.1 \u00b7 High", "short_label": "Claude Fable 5.1 High", "score": 90.0, "passed": 81, "attempts": 90, "tokens": 12474.855555555556, "ci": [ 80.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 6.9438910215 }, "predictions": { "brier": { "metric": 0.06710434444444445, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06710434444444445, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.41666032298788497, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.41666032298788497, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 18.680888437499988, "bidlo-api": 30.808591832999998 }, "cost": 21.609098765432098, "average_task_cost_estimate": 0.1944818888888889, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 18.727719208999986, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-fable-5.1@low", "model_id": "anthropic/claude-fable-5.1", "model_label": "Claude Fable 5.1", "effort": "low", "label": "Claude Fable 5.1 \u00b7 Low", "short_label": "Claude Fable 5.1 Low", "score": 90.0, "passed": 81, "attempts": 90, "tokens": 7969.677777777778, "ci": [ 80.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 5.907317708000001 }, "predictions": { "brier": { "metric": 0.06679776944444445, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06679776944444445, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4076282597095879, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4076282597095879, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 13.4306135625, "bidlo-api": 9.396270708999996 }, "cost": 13.588037037037038, "average_task_cost_estimate": 0.12229233333333334, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 13.354574125, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-fable-5.1@max", "model_id": "anthropic/claude-fable-5.1", "model_label": "Claude Fable 5.1", "effort": "max", "label": "Claude Fable 5.1 \u00b7 Max", "short_label": "Claude Fable 5.1 Max", "score": 73.33333333333333, "passed": 66, "attempts": 90, "tokens": 80000.04444444444, "ci": [ 62.22, 83.33 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 3, "attempts": 18, "rate": 16.666666666666668 } }, "difficulty": { "Routine": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Expert": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Frontier": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Stress": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 16.027346375000008 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.052245502612021585, "baseline_full": 0.06768977925381477, "valid_batches": 7, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 2, "execution_error_batches": 3, "metric_complete": false, "valid_output_metric": 0.049188802380952384, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.44140843662950957, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.44140843662950957, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 44.91806229200006, "bidlo-api": 41.880012541999996 }, "cost": 201.60430303030304, "average_task_cost_estimate": 1.4784315555555556, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 43.956258313000035, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 3 }, { "id": "anthropic/claude-fable-5.1@medium", "model_id": "anthropic/claude-fable-5.1", "model_label": "Claude Fable 5.1", "effort": "medium", "label": "Claude Fable 5.1 \u00b7 Medium", "short_label": "Claude Fable 5.1 Medium", "score": 90.0, "passed": 81, "attempts": 90, "tokens": 10133.9, "ci": [ 80.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 6.110781708499999 }, "predictions": { "brier": { "metric": 0.06684522083333333, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06684522083333333, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4060955258862978, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4060955258862978, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 17.677462020499988, "bidlo-api": 9.767492542000006 }, "cost": 17.967493827160492, "average_task_cost_estimate": 0.16170744444444446, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 17.510923290999983, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-fable-5.1@xhigh", "model_id": "anthropic/claude-fable-5.1", "model_label": "Claude Fable 5.1", "effort": "xhigh", "label": "Claude Fable 5.1 \u00b7 XHigh", "short_label": "Claude Fable 5.1 XHigh", "score": 90.0, "passed": 81, "attempts": 90, "tokens": 22490.966666666667, "ci": [ 80.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 8.6284363335 }, "predictions": { "brier": { "metric": 0.06695632638888889, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06695632638888889, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4356245781561751, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4356245781561751, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 24.616474583500008, "bidlo-api": 31.127135916999993 }, "cost": 44.08448148148148, "average_task_cost_estimate": 0.3967603333333334, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 24.76808837500002, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-haiku-4.5@high", "model_id": "anthropic/claude-haiku-4.5", "model_label": "Claude Haiku 4.5", "effort": "high", "label": "Claude Haiku 4.5 \u00b7 High", "short_label": "Claude Haiku 4.5 High", "score": 37.77777777777778, "passed": 34, "attempts": 90, "tokens": 86716.01111111112, "ci": [ 26.67, 47.78 ], "categories": { "Documents": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Takeoffs": { "passed": 6, "attempts": 18, "rate": 33.333333333333336 }, "Analysis": { "passed": 5, "attempts": 18, "rate": 27.77777777777778 }, "Knowledge": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Apps": { "passed": 0, "attempts": 18, "rate": 0.0 } }, "difficulty": { "Routine": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Multi-step": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Complex": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Expert": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Frontier": { "passed": 2, "attempts": 15, "rate": 13.333333333333334 }, "Stress": { "passed": 1, "attempts": 15, "rate": 6.666666666666667 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 4.222743020499941 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.052245502612021585, "baseline_full": 0.06768977925381477, "valid_batches": 7, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 5, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.156973, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.3364307159519818, "baseline_full": 0.3567919219293358, "valid_batches": 10, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 2, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.3364794370238455, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 26.771920000000044 }, "cost": 33.76715588235294, "average_task_cost_estimate": 0.12756481111111112, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 26.771920000000044, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-haiku-4.5@medium", "model_id": "anthropic/claude-haiku-4.5", "model_label": "Claude Haiku 4.5", "effort": "medium", "label": "Claude Haiku 4.5 \u00b7 Medium", "short_label": "Claude Haiku 4.5 Medium", "score": 32.22222222222222, "passed": 29, "attempts": 90, "tokens": 85445.3111111111, "ci": [ 23.33, 42.22 ], "categories": { "Documents": { "passed": 9, "attempts": 18, "rate": 50.0 }, "Takeoffs": { "passed": 6, "attempts": 18, "rate": 33.333333333333336 }, "Analysis": { "passed": 3, "attempts": 18, "rate": 16.666666666666668 }, "Knowledge": { "passed": 10, "attempts": 18, "rate": 55.55555555555556 }, "Apps": { "passed": 1, "attempts": 18, "rate": 5.555555555555555 } }, "difficulty": { "Routine": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Multi-step": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Complex": { "passed": 5, "attempts": 15, "rate": 33.333333333333336 }, "Expert": { "passed": 4, "attempts": 15, "rate": 26.666666666666668 }, "Frontier": { "passed": 3, "attempts": 15, "rate": 20.0 }, "Stress": { "passed": 2, "attempts": 15, "rate": 13.333333333333334 } }, "judgment": { "correct": 11, "attempts": 12, "correct_rate": 91.66666666666667, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 6.236981334000127 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.07421171351471649, "baseline_full": 0.06768977925381477, "valid_batches": 5, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 7, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.1770731412, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.3058889069859509, "baseline_full": 0.3567919219293358, "valid_batches": 8, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 4, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.307874466552738, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 29.326590417 }, "cost": 38.457579310344826, "average_task_cost_estimate": 0.12391886666666667, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 29.326590417, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-haiku-4.5@none", "model_id": "anthropic/claude-haiku-4.5", "model_label": "Claude Haiku 4.5", "effort": "none", "label": "Claude Haiku 4.5 \u00b7 Off", "short_label": "Claude Haiku 4.5 Off", "score": 32.22222222222222, "passed": 29, "attempts": 90, "tokens": 85640.83333333333, "ci": [ 22.22, 42.22 ], "categories": { "Documents": { "passed": 8, "attempts": 18, "rate": 44.44444444444444 }, "Takeoffs": { "passed": 7, "attempts": 18, "rate": 38.888888888888886 }, "Analysis": { "passed": 3, "attempts": 18, "rate": 16.666666666666668 }, "Knowledge": { "passed": 10, "attempts": 18, "rate": 55.55555555555556 }, "Apps": { "passed": 1, "attempts": 18, "rate": 5.555555555555555 } }, "difficulty": { "Routine": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Multi-step": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 }, "Complex": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Expert": { "passed": 5, "attempts": 15, "rate": 33.333333333333336 }, "Frontier": { "passed": 3, "attempts": 15, "rate": 20.0 }, "Stress": { "passed": 0, "attempts": 15, "rate": 0.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 3.8575351454997433 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.07330511232398954, "baseline_full": 0.06768977925381477, "valid_batches": 5, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 7, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.23789199933333333, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.3894748361271277, "baseline_full": 0.3567919219293358, "valid_batches": 10, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 2, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.38947483612712774, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 25.229062208999995 }, "cost": 38.67627241379311, "average_task_cost_estimate": 0.12462354444444444, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 25.229062208999995, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5.5@high", "model_id": "anthropic/claude-opus-5.5", "model_label": "Claude Opus 5.5", "effort": "high", "label": "Claude Opus 5.5 \u00b7 High", "short_label": "Claude Opus 5.5 High", "score": 90.0, "passed": 81, "attempts": 90, "tokens": 15449.655555555555, "ci": [ 80.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 6.395389979499953 }, "predictions": { "brier": { "metric": 0.06859569722222222, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06859569722222222, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.40894735426779677, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.40894735426779677, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 16.1686719585, "bidlo-api": 13.140010584000004 }, "cost": 11.334622222222222, "average_task_cost_estimate": 0.10201160000000001, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 15.829177875000028, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5.5@low", "model_id": "anthropic/claude-opus-5.5", "model_label": "Claude Opus 5.5", "effort": "low", "label": "Claude Opus 5.5 \u00b7 Low", "short_label": "Claude Opus 5.5 Low", "score": 88.88888888888889, "passed": 80, "attempts": 90, "tokens": 12068.777777777777, "ci": [ 78.89, 97.78 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 8.140202291500003 }, "predictions": { "brier": { "metric": 0.07296351805555555, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.07296351805555555, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.38664415355076304, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.38664415355076304, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 88, "bidlo-api": 1, "claude-code-cli-pilot-v2": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 18.243876291999996, "bidlo-api": 11.511969167000004, "claude-code-cli-pilot-v2": 8.006562708 }, "cost": 8.453190000000001, "average_task_cost_estimate": 0.07513946666666667, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 17.697558917000023, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5.5@max", "model_id": "anthropic/claude-opus-5.5", "model_label": "Claude Opus 5.5", "effort": "max", "label": "Claude Opus 5.5 \u00b7 Max", "short_label": "Claude Opus 5.5 Max", "score": 76.66666666666667, "passed": 69, "attempts": 90, "tokens": 65589.24444444444, "ci": [ 65.56, 87.78 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 6, "attempts": 18, "rate": 33.333333333333336 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Complex": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Expert": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Frontier": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Stress": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 12.083255020499791 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.07200101560264839, "baseline_full": 0.06768977925381477, "valid_batches": 11, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 1, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.07141269242424242, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.4334522431253744, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4334522431253744, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 40.733291374999794, "bidlo-api": 49.857258833 }, "cost": 64.9744231884058, "average_task_cost_estimate": 0.49813724444444446, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 40.86492333299998, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5.5@medium", "model_id": "anthropic/claude-opus-5.5", "model_label": "Claude Opus 5.5", "effort": "medium", "label": "Claude Opus 5.5 \u00b7 Medium", "short_label": "Claude Opus 5.5 Medium", "score": 90.0, "passed": 81, "attempts": 90, "tokens": 13553.988888888889, "ci": [ 80.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 6.819216666500012 }, "predictions": { "brier": { "metric": 0.06843473888888889, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06843473888888889, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.43221360080754123, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.43221360080754123, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 18.153003562500018, "bidlo-api": 11.320690958000007 }, "cost": 9.86836049382716, "average_task_cost_estimate": 0.08881524444444444, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 17.980346583, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5.5@xhigh", "model_id": "anthropic/claude-opus-5.5", "model_label": "Claude Opus 5.5", "effort": "xhigh", "label": "Claude Opus 5.5 \u00b7 XHigh", "short_label": "Claude Opus 5.5 XHigh", "score": 90.0, "passed": 81, "attempts": 90, "tokens": 17693.477777777778, "ci": [ 80.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 8.387576853999974 }, "predictions": { "brier": { "metric": 0.06833395972222223, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06833395972222223, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4369852307204499, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4369852307204499, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 22.504016458000024, "bidlo-api": 14.387444875000016 }, "cost": 14.368143209876544, "average_task_cost_estimate": 0.1293132888888889, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 22.419022625000025, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5@high", "model_id": "anthropic/claude-opus-5", "model_label": "Claude Opus 5", "effort": "high", "label": "Claude Opus 5 \u00b7 High", "short_label": "Claude Opus 5 High", "score": 88.88888888888889, "passed": 80, "attempts": 90, "tokens": 24964.655555555557, "ci": [ 78.89, 98.89 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 12.002197187499974 }, "predictions": { "brier": { "metric": 0.07169985, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.07169985, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4794768968912502, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4794768968912502, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 31.478692541999862, "bidlo-api": 17.99890291599999 }, "cost": 24.774168750000005, "average_task_cost_estimate": 0.22021483333333336, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 31.43873147949992, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5@low", "model_id": "anthropic/claude-opus-5", "model_label": "Claude Opus 5", "effort": "low", "label": "Claude Opus 5 \u00b7 Low", "short_label": "Claude Opus 5 Low", "score": 90.0, "passed": 81, "attempts": 90, "tokens": 16072.522222222222, "ci": [ 80.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 5.675542208000024 }, "predictions": { "brier": { "metric": 0.07593783333333334, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.07593783333333334, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5426485625081794, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5426485625081794, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 14.409512187500013, "bidlo-api": 7.1938208750000046 }, "cost": 13.183672839506173, "average_task_cost_estimate": 0.11865305555555557, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 14.231638958000076, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5@max", "model_id": "anthropic/claude-opus-5", "model_label": "Claude Opus 5", "effort": "max", "label": "Claude Opus 5 \u00b7 Max", "short_label": "Claude Opus 5 Max", "score": 77.77777777777777, "passed": 70, "attempts": 90, "tokens": 56724.91111111111, "ci": [ 65.56, 90.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 7, "attempts": 18, "rate": 38.888888888888886 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Expert": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Frontier": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Stress": { "passed": 6, "attempts": 15, "rate": 40.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 7, "solvable_attempts": 90, "false_stop_rate": 7.777777777777778, "median_correct_stop_s": 19.90981041650025 }, "predictions": { "brier": { "metric": 0.06712275277777778, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06712275277777778, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": null, "baseline_matched": 0.37219405720488324, "baseline_full": 0.3567919219293358, "valid_batches": 8, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 4, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.46794136708667994, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 44.132288833000075, "bidlo-api": 33.839100167 }, "cost": 64.18798571428572, "average_task_cost_estimate": 0.4992398888888889, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 43.685768375000066, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5@medium", "model_id": "anthropic/claude-opus-5", "model_label": "Claude Opus 5", "effort": "medium", "label": "Claude Opus 5 \u00b7 Medium", "short_label": "Claude Opus 5 Medium", "score": 90.0, "passed": 81, "attempts": 90, "tokens": 18432.744444444445, "ci": [ 80.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 7.8656089375000136 }, "predictions": { "brier": { "metric": 0.07474639444444445, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.07474639444444445, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4619614662593778, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4619614662593778, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 23.971426250500002, "bidlo-api": 10.00403449999998 }, "cost": 16.832734567901237, "average_task_cost_estimate": 0.15149461111111112, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 23.574787458999992, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5@none", "model_id": "anthropic/claude-opus-5", "model_label": "Claude Opus 5", "effort": "none", "label": "Claude Opus 5 \u00b7 Off", "short_label": "Claude Opus 5 Off", "score": 85.55555555555556, "passed": 77, "attempts": 90, "tokens": 29059.977777777778, "ci": [ 74.44, 95.56 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Apps": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Frontier": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Stress": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 11.890652812999974 }, "predictions": { "brier": { "metric": 0.08787243333333333, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.08787243333333333, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4558910096429143, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4558910096429143, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 33.38334491649999, "bidlo-api": 19.801552874999995 }, "cost": 28.027779220779223, "average_task_cost_estimate": 0.23979322222222224, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 33.21552270799998, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-opus-5@xhigh", "model_id": "anthropic/claude-opus-5", "model_label": "Claude Opus 5", "effort": "xhigh", "label": "Claude Opus 5 \u00b7 XHigh", "short_label": "Claude Opus 5 XHigh", "score": 85.55555555555556, "passed": 77, "attempts": 90, "tokens": 38837.02222222222, "ci": [ 74.44, 95.56 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Frontier": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Stress": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 16.10762666650004 }, "predictions": { "brier": { "metric": 0.06734025833333333, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06734025833333333, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": null, "baseline_matched": 0.32903720003055864, "baseline_full": 0.3567919219293358, "valid_batches": 11, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 1, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.4153827889519518, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 38.53995770800003, "bidlo-api": 32.99932404199999 }, "cost": 40.576805194805196, "average_task_cost_estimate": 0.34715711111111114, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 38.00943649999999, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-sonnet-5@high", "model_id": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "effort": "high", "label": "Claude Sonnet 5 \u00b7 High", "short_label": "Claude Sonnet 5 High", "score": 63.333333333333336, "passed": 57, "attempts": 90, "tokens": 63611.24444444444, "ci": [ 51.11, 74.44 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Analysis": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Knowledge": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Apps": { "passed": 1, "attempts": 18, "rate": 5.555555555555555 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Complex": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Expert": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Frontier": { "passed": 5, "attempts": 15, "rate": 33.333333333333336 }, "Stress": { "passed": 4, "attempts": 15, "rate": 26.666666666666668 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 10.88613062499999 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.06620838640054864, "baseline_full": 0.06768977925381477, "valid_batches": 10, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 2, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.21131127, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.5883475871187638, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5883475871187638, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 29.945982499999925 }, "cost": 33.72457543859649, "average_task_cost_estimate": 0.21358897777777777, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 29.945982499999925, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-sonnet-5@low", "model_id": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "effort": "low", "label": "Claude Sonnet 5 \u00b7 Low", "short_label": "Claude Sonnet 5 Low", "score": 68.88888888888889, "passed": 62, "attempts": 90, "tokens": 68002.4, "ci": [ 56.67, 80.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Analysis": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Knowledge": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Apps": { "passed": 4, "attempts": 18, "rate": 22.22222222222222 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Complex": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Expert": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Frontier": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 }, "Stress": { "passed": 6, "attempts": 15, "rate": 40.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 5.74396095800004 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.07200101560264839, "baseline_full": 0.06768977925381477, "valid_batches": 11, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 1, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.20850546363636363, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.5019151494502038, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5019151494502038, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 26.906351104000116 }, "cost": 30.776283870967738, "average_task_cost_estimate": 0.2120144, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 26.906351104000116, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-sonnet-5@max", "model_id": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "effort": "max", "label": "Claude Sonnet 5 \u00b7 Max", "short_label": "Claude Sonnet 5 Max", "score": 80.0, "passed": 72, "attempts": 90, "tokens": 83078.5, "ci": [ 68.89, 90.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 10, "attempts": 18, "rate": 55.55555555555556 } }, "difficulty": { "Routine": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Multi-step": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Complex": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Expert": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Frontier": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 8, "solvable_attempts": 90, "false_stop_rate": 8.88888888888889, "median_correct_stop_s": 16.167072604000026 }, "predictions": { "brier": { "metric": 0.06830723890277778, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06830723890277778, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5346239333291634, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5346239333291634, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "claude-code-cli-v1": 89, "bidlo-api": 1 }, "mixed_harness": true, "latency_by_harness": { "claude-code-cli-v1": 67.27344308400001, "bidlo-api": 27.433729416999995 }, "cost": 38.60055833333333, "average_task_cost_estimate": 0.30880446666666667, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 62.41274541699997, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-sonnet-5@medium", "model_id": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "effort": "medium", "label": "Claude Sonnet 5 \u00b7 Medium", "short_label": "Claude Sonnet 5 Medium", "score": 64.44444444444444, "passed": 58, "attempts": 90, "tokens": 61005.7, "ci": [ 51.11, 76.67 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Analysis": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Knowledge": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Apps": { "passed": 5, "attempts": 18, "rate": 27.77777777777778 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Complex": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Expert": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Frontier": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 }, "Stress": { "passed": 3, "attempts": 15, "rate": 20.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 8.057889104500006 }, "predictions": { "brier": { "metric": 0.22387304166666666, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.22387304166666666, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5869266868463527, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5869266868463527, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 24.507434666499904 }, "cost": 30.41074827586207, "average_task_cost_estimate": 0.19598037777777777, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 24.507434666499904, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-sonnet-5@none", "model_id": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "effort": "none", "label": "Claude Sonnet 5 \u00b7 Off", "short_label": "Claude Sonnet 5 Off", "score": 55.55555555555556, "passed": 50, "attempts": 90, "tokens": 61787.38888888889, "ci": [ 44.44, 66.67 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Analysis": { "passed": 9, "attempts": 18, "rate": 50.0 }, "Knowledge": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Apps": { "passed": 0, "attempts": 18, "rate": 0.0 } }, "difficulty": { "Routine": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Expert": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Frontier": { "passed": 4, "attempts": 15, "rate": 26.666666666666668 }, "Stress": { "passed": 4, "attempts": 15, "rate": 26.666666666666668 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 8.627152458500001 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.052245502612021585, "baseline_full": 0.06768977925381477, "valid_batches": 7, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 5, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.1560055761904762, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.3857407785884858, "baseline_full": 0.3567919219293358, "valid_batches": 5, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 7, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.8997204271983046, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 33.210549021000006 }, "cost": 34.666244, "average_task_cost_estimate": 0.19259024444444445, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 33.210549021000006, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "anthropic/claude-sonnet-5@xhigh", "model_id": "anthropic/claude-sonnet-5", "model_label": "Claude Sonnet 5", "effort": "xhigh", "label": "Claude Sonnet 5 \u00b7 XHigh", "short_label": "Claude Sonnet 5 XHigh", "score": 73.33333333333333, "passed": 66, "attempts": 90, "tokens": 54315.85555555556, "ci": [ 63.33, 83.33 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 10, "attempts": 18, "rate": 55.55555555555556 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 5, "attempts": 18, "rate": 27.77777777777778 } }, "difficulty": { "Routine": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Expert": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Frontier": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Stress": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 9.330500333500026 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.06472526368067763, "baseline_full": 0.06768977925381477, "valid_batches": 11, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 1, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.09399427559090909, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.4700329355414978, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4700329355414978, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 25, "claude-code-cli-v1": 65 }, "mixed_harness": true, "latency_by_harness": { "bidlo-api": 38.53893012499996, "claude-code-cli-v1": 31.78902808299995 }, "cost": 24.596748484848487, "average_task_cost_estimate": 0.18037615555555556, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 32.16925522899999, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "google/gemini-3.1-pro-preview@high", "model_id": "google/gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "effort": "high", "label": "Gemini 3.1 Pro Preview \u00b7 High", "short_label": "Gemini 3.1 Pro Preview High", "score": 78.88888888888889, "passed": 71, "attempts": 90, "tokens": 30832.411111111112, "ci": [ 66.67, 90.0 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 10, "attempts": 18, "rate": 55.55555555555556 } }, "difficulty": { "Routine": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 12.153207958499902 }, "predictions": { "brier": { "metric": 0.10605874551718182, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.10605874551718182, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.617040326301264, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.617040326301264, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 66.57771304200007 }, "cost": 17.969921126760564, "average_task_cost_estimate": 0.1417627111111111, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 66.57771304200007, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "google/gemini-3.1-pro-preview@low", "model_id": "google/gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "effort": "low", "label": "Gemini 3.1 Pro Preview \u00b7 Low", "short_label": "Gemini 3.1 Pro Preview Low", "score": 90.0, "passed": 81, "attempts": 90, "tokens": 12763.066666666668, "ci": [ 80.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 8.026528916500043 }, "predictions": { "brier": { "metric": 0.1295629178238513, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.1295629178238513, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.6830180335390641, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.6830180335390641, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 24.53602037499938 }, "cost": 6.079903703703703, "average_task_cost_estimate": 0.054719133333333336, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 24.53602037499938, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "google/gemini-3.1-pro-preview@medium", "model_id": "google/gemini-3.1-pro-preview", "model_label": "Gemini 3.1 Pro Preview", "effort": "medium", "label": "Gemini 3.1 Pro Preview \u00b7 Medium", "short_label": "Gemini 3.1 Pro Preview Medium", "score": 88.88888888888889, "passed": 80, "attempts": 90, "tokens": 14975.488888888889, "ci": [ 78.89, 97.78 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 } }, "judgment": { "correct": 10, "attempts": 12, "correct_rate": 83.33333333333333, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 10.537503353999927 }, "predictions": { "brier": { "metric": 0.1451453887870496, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.1451453887870496, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.6766801542684024, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.6766801542684024, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 35.64942943750019 }, "cost": 7.9449725, "average_task_cost_estimate": 0.07062197777777778, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 35.64942943750019, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "google/gemini-3.5-flash-lite@high", "model_id": "google/gemini-3.5-flash-lite", "model_label": "Gemini 3.5 Flash Lite", "effort": "high", "label": "Gemini 3.5 Flash Lite \u00b7 High", "short_label": "Gemini 3.5 Flash Lite High", "score": 46.666666666666664, "passed": 42, "attempts": 90, "tokens": 66165.82222222222, "ci": [ 35.56, 58.89 ], "categories": { "Documents": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Takeoffs": { "passed": 8, "attempts": 18, "rate": 44.44444444444444 }, "Analysis": { "passed": 7, "attempts": 18, "rate": 38.888888888888886 }, "Knowledge": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Apps": { "passed": 1, "attempts": 18, "rate": 5.555555555555555 } }, "difficulty": { "Routine": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Expert": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Frontier": { "passed": 2, "attempts": 15, "rate": 13.333333333333334 }, "Stress": { "passed": 1, "attempts": 15, "rate": 6.666666666666667 } }, "judgment": { "correct": 11, "attempts": 12, "correct_rate": 91.66666666666667, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 13.489078041999601 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.05442704243560219, "baseline_full": 0.06768977925381477, "valid_batches": 1, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 11, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.43609475, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.2551020408163266, "baseline_full": 0.3567919219293358, "valid_batches": 2, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 10, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.3380102040816326, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 30.37339079199964 }, "cost": 8.457460476190477, "average_task_cost_estimate": 0.03946814888888889, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 30.37339079199964, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "google/gemini-3.5-flash-lite@low", "model_id": "google/gemini-3.5-flash-lite", "model_label": "Gemini 3.5 Flash Lite", "effort": "low", "label": "Gemini 3.5 Flash Lite \u00b7 Low", "short_label": "Gemini 3.5 Flash Lite Low", "score": 57.77777777777778, "passed": 52, "attempts": 90, "tokens": 47659.02222222222, "ci": [ 43.33, 72.22 ], "categories": { "Documents": { "passed": 9, "attempts": 18, "rate": 50.0 }, "Takeoffs": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Apps": { "passed": 6, "attempts": 18, "rate": 33.333333333333336 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Expert": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Frontier": { "passed": 5, "attempts": 15, "rate": 33.333333333333336 }, "Stress": { "passed": 3, "attempts": 15, "rate": 20.0 } }, "judgment": { "correct": 10, "attempts": 12, "correct_rate": 83.33333333333333, "false_stops": 2, "solvable_attempts": 90, "false_stop_rate": 2.2222222222222223, "median_correct_stop_s": 3.732497165999841 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.06472526368067763, "baseline_full": 0.06768977925381477, "valid_batches": 11, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 1, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.235208667, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.36603645657597295, "baseline_full": 0.3567919219293358, "valid_batches": 11, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 1, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.6344295053024949, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 13.254216583500005 }, "cost": 4.399844230769231, "average_task_cost_estimate": 0.025421322222222223, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 13.254216583500005, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "google/gemini-3.5-flash-lite@medium", "model_id": "google/gemini-3.5-flash-lite", "model_label": "Gemini 3.5 Flash Lite", "effort": "medium", "label": "Gemini 3.5 Flash Lite \u00b7 Medium", "short_label": "Gemini 3.5 Flash Lite Medium", "score": 63.333333333333336, "passed": 57, "attempts": 90, "tokens": 58511.76666666667, "ci": [ 51.11, 75.56 ], "categories": { "Documents": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Takeoffs": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Analysis": { "passed": 9, "attempts": 18, "rate": 50.0 }, "Knowledge": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Apps": { "passed": 8, "attempts": 18, "rate": 44.44444444444444 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Complex": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Expert": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Frontier": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Stress": { "passed": 2, "attempts": 15, "rate": 13.333333333333334 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 13.447978395999991 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.06958445594213085, "baseline_full": 0.06768977925381477, "valid_batches": 7, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 5, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.39652299585714285, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.3360809818088684, "baseline_full": 0.3567919219293358, "valid_batches": 7, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 5, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.5410784573842766, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 26.653550374999643 }, "cost": 5.427816666666667, "average_task_cost_estimate": 0.03437617222222222, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 26.653550374999643, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "google/gemini-3.5-flash-lite@minimal", "model_id": "google/gemini-3.5-flash-lite", "model_label": "Gemini 3.5 Flash Lite", "effort": "minimal", "label": "Gemini 3.5 Flash Lite \u00b7 Minimal", "short_label": "Gemini 3.5 Flash Lite Minimal", "score": 46.666666666666664, "passed": 42, "attempts": 90, "tokens": 47104.92222222222, "ci": [ 31.11, 61.11 ], "categories": { "Documents": { "passed": 5, "attempts": 18, "rate": 27.77777777777778 }, "Takeoffs": { "passed": 8, "attempts": 18, "rate": 44.44444444444444 }, "Analysis": { "passed": 8, "attempts": 18, "rate": 44.44444444444444 }, "Knowledge": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Apps": { "passed": 9, "attempts": 18, "rate": 50.0 } }, "difficulty": { "Routine": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Multi-step": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Complex": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Expert": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Frontier": { "passed": 3, "attempts": 15, "rate": 20.0 }, "Stress": { "passed": 3, "attempts": 15, "rate": 20.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 3.180724750500173 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.07769393814652457, "baseline_full": 0.06768977925381477, "valid_batches": 8, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 4, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.3246552396458333, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.37222309439832335, "baseline_full": 0.3567919219293358, "valid_batches": 8, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 4, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.5911144886418579, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 13.456935958499233 }, "cost": 5.845095476190476, "average_task_cost_estimate": 0.027277112222222222, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 13.456935958499233, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "google/gemini-3.8-flash@high", "model_id": "google/gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "effort": "high", "label": "Gemini 3.8 Flash \u00b7 High", "short_label": "Gemini 3.8 Flash High", "score": 47.77777777777778, "passed": 43, "attempts": 90, "tokens": 68736.52222222222, "ci": [ 35.56, 60.0 ], "categories": { "Documents": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Takeoffs": { "passed": 7, "attempts": 18, "rate": 38.888888888888886 }, "Analysis": { "passed": 6, "attempts": 18, "rate": 33.333333333333336 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 0, "attempts": 18, "rate": 0.0 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Complex": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Expert": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Frontier": { "passed": 3, "attempts": 15, "rate": 20.0 }, "Stress": { "passed": 3, "attempts": 15, "rate": 20.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 22.43144487499993 }, "predictions": { "brier": { "metric": null, "baseline_matched": null, "baseline_full": 0.06768977925381477, "valid_batches": 0, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 12, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": null, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.2551020408163266, "baseline_full": 0.3567919219293358, "valid_batches": 2, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 10, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.27653061224489794, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 78.05775924999999 }, "cost": 18.735612209302328, "average_task_cost_estimate": 0.08951459166666667, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 78.05775924999999, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "google/gemini-3.8-flash@low", "model_id": "google/gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "effort": "low", "label": "Gemini 3.8 Flash \u00b7 Low", "short_label": "Gemini 3.8 Flash Low", "score": 84.44444444444444, "passed": 76, "attempts": 90, "tokens": 46893.188888888886, "ci": [ 73.33, 94.44 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 } }, "difficulty": { "Routine": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Multi-step": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Stress": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 5.945441978999879 }, "predictions": { "brier": { "metric": 0.06814781618055556, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06814781618055556, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": null, "baseline_matched": 0.35918383316171026, "baseline_full": 0.3567919219293358, "valid_batches": 9, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 3, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.6214327990521099, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 23.367117666999903 }, "cost": 6.325613486842105, "average_task_cost_estimate": 0.05341629166666667, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 23.367117666999903, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "google/gemini-3.8-flash@medium", "model_id": "google/gemini-3.8-flash", "model_label": "Gemini 3.8 Flash", "effort": "medium", "label": "Gemini 3.8 Flash \u00b7 Medium", "short_label": "Gemini 3.8 Flash Medium", "score": 57.77777777777778, "passed": 52, "attempts": 90, "tokens": 54583.9, "ci": [ 47.78, 67.78 ], "categories": { "Documents": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Takeoffs": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Analysis": { "passed": 7, "attempts": 18, "rate": 38.888888888888886 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 0, "attempts": 18, "rate": 0.0 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Complex": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Expert": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Frontier": { "passed": 5, "attempts": 15, "rate": 33.333333333333336 }, "Stress": { "passed": 5, "attempts": 15, "rate": 33.333333333333336 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 13.609862521000089 }, "predictions": { "brier": { "metric": null, "baseline_matched": null, "baseline_full": 0.06768977925381477, "valid_batches": 0, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 12, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": null, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.3772591274625094, "baseline_full": 0.3567919219293358, "valid_batches": 11, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 1, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.4989790389904026, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 36.82981270800001 }, "cost": 11.834219711538461, "average_task_cost_estimate": 0.06837549166666666, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 36.82981270800001, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-luna@high", "model_id": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "effort": "high", "label": "GPT-5.6 Luna \u00b7 High", "short_label": "GPT-5.6 Luna High", "score": 78.88888888888889, "passed": 71, "attempts": 90, "tokens": 27833.766666666666, "ci": [ 71.11, 86.67 ], "categories": { "Documents": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Takeoffs": { "passed": 8, "attempts": 18, "rate": 44.44444444444444 }, "Analysis": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 } }, "difficulty": { "Routine": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Expert": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 } }, "judgment": { "correct": 10, "attempts": 12, "correct_rate": 83.33333333333333, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 3.5176495834997623 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.06962447270244437, "baseline_full": 0.06768977925381477, "valid_batches": 10, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 2, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.070265128427295, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.3566715849858942, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.3566715849858942, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 16.04657466699928 }, "cost": 1.14604338028169, "average_task_cost_estimate": 0.009041008888888888, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 16.04657466699928, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-luna@low", "model_id": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "effort": "low", "label": "GPT-5.6 Luna \u00b7 Low", "short_label": "GPT-5.6 Luna Low", "score": 66.66666666666667, "passed": 60, "attempts": 90, "tokens": 11330.944444444445, "ci": [ 55.56, 76.67 ], "categories": { "Documents": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Takeoffs": { "passed": 5, "attempts": 18, "rate": 27.77777777777778 }, "Analysis": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Knowledge": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Apps": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Complex": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Expert": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Frontier": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Stress": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 } }, "judgment": { "correct": 10, "attempts": 12, "correct_rate": 83.33333333333333, "false_stops": 4, "solvable_attempts": 90, "false_stop_rate": 4.444444444444445, "median_correct_stop_s": 3.488273291500169 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.0691711721070809, "baseline_full": 0.06768977925381477, "valid_batches": 10, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 2, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.11708950295, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.35679192192933584, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.35679192192933584, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 6.861755854000338 }, "cost": 0.49827999999999995, "average_task_cost_estimate": 0.0033218666666666665, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 6.861755854000338, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-luna@max", "model_id": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "effort": "max", "label": "GPT-5.6 Luna \u00b7 Max", "short_label": "GPT-5.6 Luna Max", "score": 81.11111111111111, "passed": 73, "attempts": 90, "tokens": 35135.433333333334, "ci": [ 72.22, 90.0 ], "categories": { "Documents": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Takeoffs": { "passed": 9, "attempts": 18, "rate": 50.0 }, "Analysis": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Complex": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Expert": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 12, "attempts": 15, "rate": 80.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 5.097012853500084 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.07285720452495058, "baseline_full": 0.06768977925381477, "valid_batches": 8, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 4, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.07407973772350833, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.35679192192933584, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.35679192192933584, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 27.339537332999985 }, "cost": 1.6744463013698627, "average_task_cost_estimate": 0.013581619999999999, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 27.339537332999985, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-luna@medium", "model_id": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "effort": "medium", "label": "GPT-5.6 Luna \u00b7 Medium", "short_label": "GPT-5.6 Luna Medium", "score": 73.33333333333333, "passed": 66, "attempts": 90, "tokens": 12353.755555555555, "ci": [ 61.11, 85.56 ], "categories": { "Documents": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Takeoffs": { "passed": 9, "attempts": 18, "rate": 50.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Apps": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Complex": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Expert": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Frontier": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Stress": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 } }, "judgment": { "correct": 9, "attempts": 12, "correct_rate": 75.0, "false_stops": 3, "solvable_attempts": 90, "false_stop_rate": 3.3333333333333335, "median_correct_stop_s": 4.1791458329998425 }, "predictions": { "brier": { "metric": 0.07235710118055555, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.07235710118055555, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.35679192192933584, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.35679192192933584, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 13.893788874999998 }, "cost": 0.5326645454545454, "average_task_cost_estimate": 0.0039062066666666665, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 13.893788874999998, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-luna@none", "model_id": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "effort": "none", "label": "GPT-5.6 Luna \u00b7 Off", "short_label": "GPT-5.6 Luna Off", "score": 56.666666666666664, "passed": 51, "attempts": 90, "tokens": 13522.822222222223, "ci": [ 43.33, 71.11 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 9, "attempts": 18, "rate": 50.0 }, "Analysis": { "passed": 5, "attempts": 18, "rate": 27.77777777777778 }, "Knowledge": { "passed": 10, "attempts": 18, "rate": 55.55555555555556 }, "Apps": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 } }, "difficulty": { "Routine": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Expert": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Frontier": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 }, "Stress": { "passed": 0, "attempts": 15, "rate": 0.0 } }, "judgment": { "correct": 9, "attempts": 12, "correct_rate": 75.0, "false_stops": 13, "solvable_attempts": 90, "false_stop_rate": 14.444444444444445, "median_correct_stop_s": 1.9414729579999113 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.061389102494270285, "baseline_full": 0.06768977925381477, "valid_batches": 8, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 4, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.09055862799742084, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.37683259193474544, "baseline_full": 0.3567919219293358, "valid_batches": 8, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 4, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.40330963086655397, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 4.602526041001082 }, "cost": 0.6952623529411764, "average_task_cost_estimate": 0.00393982, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 4.602526041001082, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-luna@xhigh", "model_id": "openai/gpt-5.6-luna", "model_label": "GPT-5.6 Luna", "effort": "xhigh", "label": "GPT-5.6 Luna \u00b7 XHigh", "short_label": "GPT-5.6 Luna XHigh", "score": 81.11111111111111, "passed": 73, "attempts": 90, "tokens": 24311.17777777778, "ci": [ 71.11, 90.0 ], "categories": { "Documents": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Takeoffs": { "passed": 9, "attempts": 18, "rate": 50.0 }, "Analysis": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Expert": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Frontier": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Stress": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 } }, "judgment": { "correct": 10, "attempts": 12, "correct_rate": 83.33333333333333, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 3.868886832999764 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.0659438330808992, "baseline_full": 0.06768977925381477, "valid_batches": 5, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 7, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.06548940015425667, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.3566715849858942, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.3566715849858942, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 13.629914792000026 }, "cost": 1.1416194520547944, "average_task_cost_estimate": 0.009259802222222223, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 13.629914792000026, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-sol@high", "model_id": "openai/gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "effort": "high", "label": "GPT-5.6 Sol \u00b7 High", "short_label": "GPT-5.6 Sol High", "score": 100.0, "passed": 90, "attempts": 90, "tokens": 21905.98888888889, "ci": [ 100.0, 100.0 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100.0 } }, "judgment": { "correct": 11, "attempts": 12, "correct_rate": 91.66666666666667, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 5.001593875000253 }, "predictions": { "brier": { "metric": 0.06809089473611112, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06809089473611112, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.588438939128508, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.588438939128508, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 17.646755333499982 }, "cost": 13.153204444444446, "average_task_cost_estimate": 0.13153204444444444, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 17.646755333499982, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-sol@low", "model_id": "openai/gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "effort": "low", "label": "GPT-5.6 Sol \u00b7 Low", "short_label": "GPT-5.6 Sol Low", "score": 95.55555555555556, "passed": 86, "attempts": 90, "tokens": 16797.31111111111, "ci": [ 88.89, 100.0 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Stress": { "passed": 12, "attempts": 15, "rate": 80.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 3, "solvable_attempts": 90, "false_stop_rate": 3.3333333333333335, "median_correct_stop_s": 3.945015375499846 }, "predictions": { "brier": { "metric": 0.06757576527777778, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06757576527777778, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.6066352858767079, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.6066352858767079, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 13.098202687500452 }, "cost": 9.094334883720931, "average_task_cost_estimate": 0.08690142222222222, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 13.098202687500452, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-sol@max", "model_id": "openai/gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "effort": "max", "label": "GPT-5.6 Sol \u00b7 Max", "short_label": "GPT-5.6 Sol Max", "score": 97.77777777777777, "passed": 88, "attempts": 90, "tokens": 30152.866666666665, "ci": [ 95.56, 100.0 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Complex": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 4.7536205420001645 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.06413063069944532, "baseline_full": 0.06768977925381477, "valid_batches": 5, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 7, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.06449371586666666, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.588438939128508, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.588438939128508, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 17.167578103999837 }, "cost": 19.678045454545455, "average_task_cost_estimate": 0.19240755555555555, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 17.167578103999837, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-sol@medium", "model_id": "openai/gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "effort": "medium", "label": "GPT-5.6 Sol \u00b7 Medium", "short_label": "GPT-5.6 Sol Medium", "score": 96.66666666666667, "passed": 87, "attempts": 90, "tokens": 21280.3, "ci": [ 90.0, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 12, "attempts": 15, "rate": 80.0 } }, "judgment": { "correct": 10, "attempts": 12, "correct_rate": 83.33333333333333, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 4.164828916499857 }, "predictions": { "brier": { "metric": 0.06779102477777778, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06779102477777778, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5781923404890522, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5781923404890522, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 14.350415917000268 }, "cost": 12.386754022988505, "average_task_cost_estimate": 0.11973862222222222, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 14.350415917000268, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-sol@none", "model_id": "openai/gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "effort": "none", "label": "GPT-5.6 Sol \u00b7 Off", "short_label": "GPT-5.6 Sol Off", "score": 84.44444444444444, "passed": 76, "attempts": 90, "tokens": 19565.633333333335, "ci": [ 73.33, 94.44 ], "categories": { "Documents": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Takeoffs": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Analysis": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Knowledge": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Stress": { "passed": 5, "attempts": 15, "rate": 33.333333333333336 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 5, "solvable_attempts": 90, "false_stop_rate": 5.555555555555555, "median_correct_stop_s": 2.031696708000032 }, "predictions": { "brier": { "metric": 0.11856569454166667, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.11856569454166667, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.43034103471524443, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.43034103471524443, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 9.090221458000014 }, "cost": 11.923657894736841, "average_task_cost_estimate": 0.10068866666666666, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 9.090221458000014, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-sol@xhigh", "model_id": "openai/gpt-5.6-sol", "model_label": "GPT-5.6 Sol", "effort": "xhigh", "label": "GPT-5.6 Sol \u00b7 XHigh", "short_label": "GPT-5.6 Sol XHigh", "score": 100.0, "passed": 90, "attempts": 90, "tokens": 26419.244444444445, "ci": [ 100.0, 100.0 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100.0 } }, "judgment": { "correct": 11, "attempts": 12, "correct_rate": 91.66666666666667, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 3.817218708000146 }, "predictions": { "brier": { "metric": 0.06800736801685417, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06800736801685417, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5781923404890522, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5781923404890522, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 16.776120916499963 }, "cost": 16.077724444444446, "average_task_cost_estimate": 0.16077724444444444, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 16.776120916499963, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-terra@high", "model_id": "openai/gpt-5.6-terra", "model_label": "GPT-5.6 Terra", "effort": "high", "label": "GPT-5.6 Terra \u00b7 High", "short_label": "GPT-5.6 Terra High", "score": 94.44444444444444, "passed": 85, "attempts": 90, "tokens": 19705.144444444446, "ci": [ 87.78, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Knowledge": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Stress": { "passed": 12, "attempts": 15, "rate": 80.0 } }, "judgment": { "correct": 11, "attempts": 12, "correct_rate": 91.66666666666667, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 3.31175841699983 }, "predictions": { "brier": { "metric": 0.06913299877777777, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06913299877777777, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4276202532618688, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4276202532618688, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 10.906350084000268 }, "cost": 6.5320423529411755, "average_task_cost_estimate": 0.06169151111111111, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 10.906350084000268, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-terra@low", "model_id": "openai/gpt-5.6-terra", "model_label": "GPT-5.6 Terra", "effort": "low", "label": "GPT-5.6 Terra \u00b7 Low", "short_label": "GPT-5.6 Terra Low", "score": 85.55555555555556, "passed": 77, "attempts": 90, "tokens": 15669.111111111111, "ci": [ 76.67, 93.33 ], "categories": { "Documents": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Takeoffs": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Analysis": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Knowledge": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Apps": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 9, "attempts": 12, "correct_rate": 75.0, "false_stops": 3, "solvable_attempts": 90, "false_stop_rate": 3.3333333333333335, "median_correct_stop_s": 2.938163832999766 }, "predictions": { "brier": { "metric": 0.06842101569444445, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06842101569444445, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4422145684544501, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4422145684544501, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 8.616859749999945 }, "cost": 5.112142857142857, "average_task_cost_estimate": 0.04373722222222222, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 8.616859749999945, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-terra@max", "model_id": "openai/gpt-5.6-terra", "model_label": "GPT-5.6 Terra", "effort": "max", "label": "GPT-5.6 Terra \u00b7 Max", "short_label": "GPT-5.6 Terra Max", "score": 98.88888888888889, "passed": 89, "attempts": 90, "tokens": 40101.72222222222, "ci": [ 96.67, 100.0 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 4.9153228125002935 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.06962447270244437, "baseline_full": 0.06768977925381477, "valid_batches": 10, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 2, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.06927988083550167, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.5231404162600766, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5231404162600766, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 89, "codex-cli-subscription-v2": 1 }, "mixed_harness": true, "latency_by_harness": { "bidlo-api": 20.02264543750009, "codex-cli-subscription-v2": 271.194591875 }, "cost": 14.98376404494382, "average_task_cost_estimate": 0.14817277777777776, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 20.281218375000172, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-terra@medium", "model_id": "openai/gpt-5.6-terra", "model_label": "GPT-5.6 Terra", "effort": "medium", "label": "GPT-5.6 Terra \u00b7 Medium", "short_label": "GPT-5.6 Terra Medium", "score": 86.66666666666667, "passed": 78, "attempts": 90, "tokens": 17737.433333333334, "ci": [ 77.78, 94.44 ], "categories": { "Documents": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Knowledge": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Apps": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Complex": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 } }, "judgment": { "correct": 9, "attempts": 12, "correct_rate": 75.0, "false_stops": 1, "solvable_attempts": 90, "false_stop_rate": 1.1111111111111112, "median_correct_stop_s": 3.558902499999851 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.06513735513100806, "baseline_full": 0.06768977925381477, "valid_batches": 11, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 1, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.06812799253030304, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.4121465637860785, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4121465637860785, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 9.376397604000289 }, "cost": 5.9398820512820505, "average_task_cost_estimate": 0.051478977777777775, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 9.376397604000289, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-terra@none", "model_id": "openai/gpt-5.6-terra", "model_label": "GPT-5.6 Terra", "effort": "none", "label": "GPT-5.6 Terra \u00b7 Off", "short_label": "GPT-5.6 Terra Off", "score": 65.55555555555556, "passed": 59, "attempts": 90, "tokens": 13513.977777777778, "ci": [ 53.33, 78.89 ], "categories": { "Documents": { "passed": 8, "attempts": 18, "rate": 44.44444444444444 }, "Takeoffs": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Analysis": { "passed": 7, "attempts": 18, "rate": 38.888888888888886 }, "Knowledge": { "passed": 10, "attempts": 18, "rate": 55.55555555555556 }, "Apps": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 } }, "difficulty": { "Routine": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Expert": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Frontier": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Stress": { "passed": 6, "attempts": 15, "rate": 40.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 2.044341083000065 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.07859125554029071, "baseline_full": 0.06768977925381477, "valid_batches": 8, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 4, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.09526208333333333, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.6672198057010744, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.6672198057010744, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 4.674105959000066 }, "cost": 6.099383050847458, "average_task_cost_estimate": 0.039984844444444445, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 4.674105959000066, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-5.6-terra@xhigh", "model_id": "openai/gpt-5.6-terra", "model_label": "GPT-5.6 Terra", "effort": "xhigh", "label": "GPT-5.6 Terra \u00b7 XHigh", "short_label": "GPT-5.6 Terra XHigh", "score": 87.77777777777777, "passed": 79, "attempts": 90, "tokens": 19751.71111111111, "ci": [ 78.89, 94.44 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Analysis": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Frontier": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 10, "attempts": 12, "correct_rate": 83.33333333333333, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 3.4898537500002424 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.06513735513100806, "baseline_full": 0.06768977925381477, "valid_batches": 11, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 1, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.06500877325757576, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.5423540791917535, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5423540791917535, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 11.536145499999636 }, "cost": 7.875832911392405, "average_task_cost_estimate": 0.06913231111111111, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 11.536145499999636, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-astra@high", "model_id": "openai/gpt-6-astra", "model_label": "GPT-6 Astra", "effort": "high", "label": "GPT-6 Astra \u00b7 High", "short_label": "GPT-6 Astra High", "score": 100.0, "passed": 90, "attempts": 90, "tokens": 13920.977777777778, "ci": [ 100.0, 100.0 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 5.019656312499894 }, "predictions": { "brier": { "metric": 0.06758012848611111, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06758012848611111, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5387447252675678, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5387447252675678, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 18.920124417000217 }, "cost": 20.876177777777777, "average_task_cost_estimate": 0.2087617777777778, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 18.920124417000217, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-astra@low", "model_id": "openai/gpt-6-astra", "model_label": "GPT-6 Astra", "effort": "low", "label": "GPT-6 Astra \u00b7 Low", "short_label": "GPT-6 Astra Low", "score": 100.0, "passed": 90, "attempts": 90, "tokens": 11621.144444444444, "ci": [ 100.0, 100.0 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 3.8947036460002415 }, "predictions": { "brier": { "metric": 0.06719941922222222, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06719941922222222, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5716322164394324, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5716322164394324, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 10.73800843799999 }, "cost": 14.90238888888889, "average_task_cost_estimate": 0.1490238888888889, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 10.73800843799999, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-astra@max", "model_id": "openai/gpt-6-astra", "model_label": "GPT-6 Astra", "effort": "max", "label": "GPT-6 Astra \u00b7 Max", "short_label": "GPT-6 Astra Max", "score": 100.0, "passed": 90, "attempts": 90, "tokens": 17597.966666666667, "ci": [ 100.0, 100.0 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 7.341566145499703 }, "predictions": { "brier": { "metric": 0.06754189144444445, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06754189144444445, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5635214721417133, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5635214721417133, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 33.67382472899999 }, "cost": 31.994544444444443, "average_task_cost_estimate": 0.31994544444444445, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 33.67382472899999, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-astra@medium", "model_id": "openai/gpt-6-astra", "model_label": "GPT-6 Astra", "effort": "medium", "label": "GPT-6 Astra \u00b7 Medium", "short_label": "GPT-6 Astra Medium", "score": 100.0, "passed": 90, "attempts": 90, "tokens": 12847.888888888889, "ci": [ 100.0, 100.0 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100.0 } }, "judgment": { "correct": 11, "attempts": 12, "correct_rate": 91.66666666666667, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 4.049042833999963 }, "predictions": { "brier": { "metric": 0.06728328408333333, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06728328408333333, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5399095273638022, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5399095273638022, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 12.788350708500715 }, "cost": 17.281533333333332, "average_task_cost_estimate": 0.17281533333333335, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 12.788350708500715, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-astra@xhigh", "model_id": "openai/gpt-6-astra", "model_label": "GPT-6 Astra", "effort": "xhigh", "label": "GPT-6 Astra \u00b7 XHigh", "short_label": "GPT-6 Astra XHigh", "score": 100.0, "passed": 90, "attempts": 90, "tokens": 15382.488888888889, "ci": [ 100.0, 100.0 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 6.402764979000203 }, "predictions": { "brier": { "metric": 0.0674106542361111, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.0674106542361111, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5531448214814492, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5531448214814492, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 29.14351893700022 }, "cost": 25.076266666666665, "average_task_cost_estimate": 0.2507626666666667, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 29.14351893700022, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-luna@high", "model_id": "openai/gpt-6-luna", "model_label": "GPT-6 Luna", "effort": "high", "label": "GPT-6 Luna \u00b7 High", "short_label": "GPT-6 Luna High", "score": 78.88888888888889, "passed": 71, "attempts": 90, "tokens": 15844.211111111112, "ci": [ 68.89, 88.89 ], "categories": { "Documents": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Takeoffs": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Analysis": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Complex": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Expert": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 3, "solvable_attempts": 90, "false_stop_rate": 3.3333333333333335, "median_correct_stop_s": 3.7464250415000135 }, "predictions": { "brier": { "metric": 0.06804867916666667, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06804867916666667, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5875202703677846, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5875202703677846, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 9.951377582999879 }, "cost": 0.30853281690140844, "average_task_cost_estimate": 0.002433981111111111, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 9.951377582999879, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-luna@low", "model_id": "openai/gpt-6-luna", "model_label": "GPT-6 Luna", "effort": "low", "label": "GPT-6 Luna \u00b7 Low", "short_label": "GPT-6 Luna Low", "score": 70.0, "passed": 63, "attempts": 90, "tokens": 8449.888888888889, "ci": [ 56.67, 83.33 ], "categories": { "Documents": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Takeoffs": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Analysis": { "passed": 9, "attempts": 18, "rate": 50.0 }, "Knowledge": { "passed": 10, "attempts": 18, "rate": 55.55555555555556 }, "Apps": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Expert": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Frontier": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Stress": { "passed": 5, "attempts": 15, "rate": 33.333333333333336 } }, "judgment": { "correct": 11, "attempts": 12, "correct_rate": 91.66666666666667, "false_stops": 10, "solvable_attempts": 90, "false_stop_rate": 11.11111111111111, "median_correct_stop_s": 3.589884166999953 }, "predictions": { "brier": { "metric": 0.09586666666666667, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.09586666666666667, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.4176012456588276, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.4176012456588276, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 6.8721844159998 }, "cost": 0.17018126984126986, "average_task_cost_estimate": 0.0011912688888888888, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 6.8721844159998, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-luna@max", "model_id": "openai/gpt-6-luna", "model_label": "GPT-6 Luna", "effort": "max", "label": "GPT-6 Luna \u00b7 Max", "short_label": "GPT-6 Luna Max", "score": 92.22222222222223, "passed": 83, "attempts": 90, "tokens": 24644.266666666666, "ci": [ 83.33, 98.89 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 } }, "difficulty": { "Routine": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Expert": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 12, "attempts": 15, "rate": 80.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 6.078004895500141 }, "predictions": { "brier": { "metric": 0.06870528068055556, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06870528068055556, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5425132827351865, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5425132827351865, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 18.15507424999995 }, "cost": 0.5238959036144578, "average_task_cost_estimate": 0.004831484444444444, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 18.15507424999995, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-luna@medium", "model_id": "openai/gpt-6-luna", "model_label": "GPT-6 Luna", "effort": "medium", "label": "GPT-6 Luna \u00b7 Medium", "short_label": "GPT-6 Luna Medium", "score": 78.88888888888889, "passed": 71, "attempts": 90, "tokens": 15802.122222222222, "ci": [ 68.89, 88.89 ], "categories": { "Documents": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Takeoffs": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Analysis": { "passed": 9, "attempts": 18, "rate": 50.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 } }, "difficulty": { "Routine": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Multi-step": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Complex": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Expert": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Frontier": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Stress": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 } }, "judgment": { "correct": 10, "attempts": 12, "correct_rate": 83.33333333333333, "false_stops": 3, "solvable_attempts": 90, "false_stop_rate": 3.3333333333333335, "median_correct_stop_s": 3.465215187499882 }, "predictions": { "brier": { "metric": 0.06885169861111111, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06885169861111111, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.44579599858238617, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.44579599858238617, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 9.610592874999973 }, "cost": 0.2836556338028169, "average_task_cost_estimate": 0.0022377277777777776, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 9.610592874999973, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-luna@none", "model_id": "openai/gpt-6-luna", "model_label": "GPT-6 Luna", "effort": "none", "label": "GPT-6 Luna \u00b7 Off", "short_label": "GPT-6 Luna Off", "score": 56.666666666666664, "passed": 51, "attempts": 90, "tokens": 20710.577777777777, "ci": [ 42.22, 71.11 ], "categories": { "Documents": { "passed": 8, "attempts": 18, "rate": 44.44444444444444 }, "Takeoffs": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Analysis": { "passed": 6, "attempts": 18, "rate": 33.333333333333336 }, "Knowledge": { "passed": 7, "attempts": 18, "rate": 38.888888888888886 }, "Apps": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 } }, "difficulty": { "Routine": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Multi-step": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Complex": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Expert": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Frontier": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Stress": { "passed": 1, "attempts": 15, "rate": 6.666666666666667 } }, "judgment": { "correct": 9, "attempts": 12, "correct_rate": 75.0, "false_stops": 12, "solvable_attempts": 90, "false_stop_rate": 13.333333333333334, "median_correct_stop_s": 1.7103761670002713 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.06915372239178526, "baseline_full": 0.06768977925381477, "valid_batches": 8, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 4, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.1089188, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.3771298981519376, "baseline_full": 0.3567919219293358, "valid_batches": 10, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 2, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.4471104988691634, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 7.9663701249999 }, "cost": 0.4401490196078431, "average_task_cost_estimate": 0.0024941777777777777, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 7.9663701249999, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-luna@xhigh", "model_id": "openai/gpt-6-luna", "model_label": "GPT-6 Luna", "effort": "xhigh", "label": "GPT-6 Luna \u00b7 XHigh", "short_label": "GPT-6 Luna XHigh", "score": 88.88888888888889, "passed": 80, "attempts": 90, "tokens": 22999.68888888889, "ci": [ 80.0, 96.67 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Analysis": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 } }, "difficulty": { "Routine": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 12, "attempts": 15, "rate": 80.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 3, "solvable_attempts": 90, "false_stop_rate": 3.3333333333333335, "median_correct_stop_s": 4.5776513334997 }, "predictions": { "brier": { "metric": 0.06814458626388889, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06814458626388889, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5285964073207607, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5285964073207607, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 11.530144562500006 }, "cost": 0.392978, "average_task_cost_estimate": 0.0034931377777777774, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 11.530144562500006, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-sol@high", "model_id": "openai/gpt-6-sol", "model_label": "GPT-6 Sol", "effort": "high", "label": "GPT-6 Sol \u00b7 High", "short_label": "GPT-6 Sol High", "score": 98.88888888888889, "passed": 89, "attempts": 90, "tokens": 16102.844444444445, "ci": [ 96.67, 100.0 ], "categories": { "Documents": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 4.895087416999973 }, "predictions": { "brier": { "metric": 0.06731681666666667, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06731681666666667, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.6168871885831155, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.6168871885831155, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 11.909863749999554 }, "cost": 4.929483146067416, "average_task_cost_estimate": 0.04874711111111111, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 11.909863749999554, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-sol@low", "model_id": "openai/gpt-6-sol", "model_label": "GPT-6 Sol", "effort": "low", "label": "GPT-6 Sol \u00b7 Low", "short_label": "GPT-6 Sol Low", "score": 94.44444444444444, "passed": 85, "attempts": 90, "tokens": 12034.477777777778, "ci": [ 87.78, 100.0 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Stress": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 3, "solvable_attempts": 90, "false_stop_rate": 3.3333333333333335, "median_correct_stop_s": 3.6567647920001765 }, "predictions": { "brier": { "metric": 0.07098764166666667, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.07098764166666667, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.6055948636176703, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.6055948636176703, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 7.382528458000161 }, "cost": 3.108778823529412, "average_task_cost_estimate": 0.02936068888888889, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 7.382528458000161, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-sol@max", "model_id": "openai/gpt-6-sol", "model_label": "GPT-6 Sol", "effort": "max", "label": "GPT-6 Sol \u00b7 Max", "short_label": "GPT-6 Sol Max", "score": 98.88888888888889, "passed": 89, "attempts": 90, "tokens": 26355.133333333335, "ci": [ 96.67, 100.0 ], "categories": { "Documents": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 7.002121104000136 }, "predictions": { "brier": { "metric": 0.06746843273611111, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06746843273611111, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.6134672153994195, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.6134672153994195, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 16.565193333999954 }, "cost": 8.475231460674157, "average_task_cost_estimate": 0.08381062222222223, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 16.565193333999954, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-sol@medium", "model_id": "openai/gpt-6-sol", "model_label": "GPT-6 Sol", "effort": "medium", "label": "GPT-6 Sol \u00b7 Medium", "short_label": "GPT-6 Sol Medium", "score": 94.44444444444444, "passed": 85, "attempts": 90, "tokens": 13382.555555555555, "ci": [ 86.67, 100.0 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 3, "solvable_attempts": 90, "false_stop_rate": 3.3333333333333335, "median_correct_stop_s": 4.662467749999836 }, "predictions": { "brier": { "metric": 0.06821722916666667, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06821722916666667, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.631205072494237, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.631205072494237, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 10.411654500000004 }, "cost": 3.7387811764705883, "average_task_cost_estimate": 0.03531071111111111, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 10.411654500000004, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-sol@none", "model_id": "openai/gpt-6-sol", "model_label": "GPT-6 Sol", "effort": "none", "label": "GPT-6 Sol \u00b7 Off", "short_label": "GPT-6 Sol Off", "score": 83.33333333333333, "passed": 75, "attempts": 90, "tokens": 17675.166666666668, "ci": [ 73.33, 92.22 ], "categories": { "Documents": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Complex": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Expert": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Frontier": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Stress": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 3, "solvable_attempts": 90, "false_stop_rate": 3.3333333333333335, "median_correct_stop_s": 2.7702606459999224 }, "predictions": { "brier": { "metric": 0.07068222222222222, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.07068222222222222, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.627929352824778, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.627929352824778, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 7.889356167000253 }, "cost": 5.024962666666666, "average_task_cost_estimate": 0.041874688888888886, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 7.889356167000253, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "openai/gpt-6-sol@xhigh", "model_id": "openai/gpt-6-sol", "model_label": "GPT-6 Sol", "effort": "xhigh", "label": "GPT-6 Sol \u00b7 XHigh", "short_label": "GPT-6 Sol XHigh", "score": 98.88888888888889, "passed": 89, "attempts": 90, "tokens": 18151.577777777777, "ci": [ 96.67, 100.0 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 5.234220208000043 }, "predictions": { "brier": { "metric": 0.06779464816666667, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06779464816666667, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.6001520720351002, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.6001520720351002, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 14.054134125000099 }, "cost": 5.700166292134831, "average_task_cost_estimate": 0.05636831111111111, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 14.054134125000099, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "spacexai/grok-4.20-reasoning@high", "model_id": "spacexai/grok-4.20-reasoning", "model_label": "Grok 4.20", "effort": "high", "label": "Grok 4.20 \u00b7 High", "short_label": "Grok 4.20 High", "score": 77.77777777777777, "passed": 70, "attempts": 90, "tokens": 39175.2, "ci": [ 66.64, 87.78 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Knowledge": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Apps": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Complex": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Expert": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Frontier": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Stress": { "passed": 9, "attempts": 15, "rate": 60.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 9, "solvable_attempts": 90, "false_stop_rate": 10.0, "median_correct_stop_s": 8.05594635450002 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.0582050592863808, "baseline_full": 0.06768977925381477, "valid_batches": 5, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 7, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.15786275676666667, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.3556956978124052, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.3556956978124052, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 39.118376000500746 }, "cost": 7.426714285714286, "average_task_cost_estimate": 0.05776333333333334, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 39.118376000500746, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "spacexai/grok-4.20-reasoning@none", "model_id": "spacexai/grok-4.20-reasoning", "model_label": "Grok 4.20", "effort": "none", "label": "Grok 4.20 \u00b7 Off", "short_label": "Grok 4.20 Off", "score": 35.55555555555556, "passed": 32, "attempts": 90, "tokens": 72067.82222222222, "ci": [ 24.44, 46.67 ], "categories": { "Documents": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Takeoffs": { "passed": 5, "attempts": 18, "rate": 27.77777777777778 }, "Analysis": { "passed": 5, "attempts": 18, "rate": 27.77777777777778 }, "Knowledge": { "passed": 8, "attempts": 18, "rate": 44.44444444444444 }, "Apps": { "passed": 2, "attempts": 18, "rate": 11.11111111111111 } }, "difficulty": { "Routine": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 }, "Multi-step": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Complex": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Expert": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 }, "Frontier": { "passed": 2, "attempts": 15, "rate": 13.333333333333334 }, "Stress": { "passed": 3, "attempts": 15, "rate": 20.0 } }, "judgment": { "correct": 5, "attempts": 12, "correct_rate": 41.666666666666664, "false_stops": 1, "solvable_attempts": 90, "false_stop_rate": 1.1111111111111112, "median_correct_stop_s": 1.2093355000000447 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.04310500624877206, "baseline_full": 0.06768977925381477, "valid_batches": 5, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 7, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.1657752359, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.2755607886678828, "baseline_full": 0.3567919219293358, "valid_batches": 6, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 6, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.2755607886678829, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 12.009604312499985 }, "cost": 31.82805078125, "average_task_cost_estimate": 0.11316640277777779, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 12.009604312499985, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "spacexai/grok-4.7@high", "model_id": "spacexai/grok-4.7", "model_label": "Grok 4.7", "effort": "high", "label": "Grok 4.7 \u00b7 High", "short_label": "Grok 4.7 High", "score": 92.22222222222223, "passed": 83, "attempts": 90, "tokens": 74868.04444444444, "ci": [ 85.56, 98.89 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 15, "attempts": 15, "rate": 100.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 9.843619124999968 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.07510872751197764, "baseline_full": 0.06768977925381477, "valid_batches": 9, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 3, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.07560355539621111, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.35679192192933584, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.35679192192933584, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 71.54865137499944 }, "cost": 13.493713734939758, "average_task_cost_estimate": 0.12444202666666666, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 71.54865137499944, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "spacexai/grok-4.7@low", "model_id": "spacexai/grok-4.7", "model_label": "Grok 4.7", "effort": "low", "label": "Grok 4.7 \u00b7 Low", "short_label": "Grok 4.7 Low", "score": 97.77777777777777, "passed": 88, "attempts": 90, "tokens": 42300.37777777778, "ci": [ 94.44, 100.0 ], "categories": { "Documents": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 18, "attempts": 18, "rate": 100.0 } }, "difficulty": { "Routine": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Expert": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Frontier": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Stress": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 5.150795146000048 }, "predictions": { "brier": { "metric": 0.06884936194444444, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.06884936194444444, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.35679192192933584, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.35679192192933584, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 30.365771833999993 }, "cost": 6.657272727272726, "average_task_cost_estimate": 0.06509333333333334, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 30.365771833999993, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "spacexai/grok-4.7@medium", "model_id": "spacexai/grok-4.7", "model_label": "Grok 4.7", "effort": "medium", "label": "Grok 4.7 \u00b7 Medium", "short_label": "Grok 4.7 Medium", "score": 91.11111111111111, "passed": 82, "attempts": 90, "tokens": 57711.6, "ci": [ 85.56, 95.56 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Analysis": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 } }, "difficulty": { "Routine": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Multi-step": { "passed": 15, "attempts": 15, "rate": 100.0 }, "Complex": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Expert": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Frontier": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Stress": { "passed": 12, "attempts": 15, "rate": 80.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 13.061131104500033 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.07285720452495058, "baseline_full": 0.06768977925381477, "valid_batches": 8, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 4, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.07410864086325208, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.35679192192933584, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.35679192192933584, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 46.95067316650017 }, "cost": 10.280318048780488, "average_task_cost_estimate": 0.09366511999999999, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 46.95067316650017, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "spacexai/grok-4.7@xhigh", "model_id": "spacexai/grok-4.7", "model_label": "Grok 4.7", "effort": "xhigh", "label": "Grok 4.7 \u00b7 XHigh", "short_label": "Grok 4.7 XHigh", "score": 86.66666666666667, "passed": 78, "attempts": 90, "tokens": 102732.33333333333, "ci": [ 81.11, 91.11 ], "categories": { "Documents": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Takeoffs": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Analysis": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Knowledge": { "passed": 18, "attempts": 18, "rate": 100.0 }, "Apps": { "passed": 7, "attempts": 18, "rate": 38.888888888888886 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Expert": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Frontier": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Stress": { "passed": 12, "attempts": 15, "rate": 80.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 21.056023520999997 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.05681988881897858, "baseline_full": 0.06768977925381477, "valid_batches": 3, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 9, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.056130295277777784, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.35679192192933584, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.35679192192933584, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 66.5564459995 }, "cost": 19.370169230769232, "average_task_cost_estimate": 0.1678748, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 66.5564459995, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "spacexai/grok-build-0.1@fixed", "model_id": "spacexai/grok-build-0.1", "model_label": "Grok Build 0.1", "effort": "fixed", "label": "Grok Build 0.1 \u00b7 Fixed", "short_label": "Grok Build 0.1 Fixed", "score": 54.44444444444444, "passed": 49, "attempts": 90, "tokens": 73046.4, "ci": [ 42.22, 66.67 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 6, "attempts": 18, "rate": 33.333333333333336 }, "Analysis": { "passed": 11, "attempts": 18, "rate": 61.111111111111114 }, "Knowledge": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Apps": { "passed": 5, "attempts": 18, "rate": 27.77777777777778 } }, "difficulty": { "Routine": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 }, "Multi-step": { "passed": 6, "attempts": 15, "rate": 40.0 }, "Complex": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Expert": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 }, "Frontier": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Stress": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 } }, "judgment": { "correct": 10, "attempts": 12, "correct_rate": 83.33333333333333, "false_stops": 6, "solvable_attempts": 90, "false_stop_rate": 6.666666666666667, "median_correct_stop_s": 11.952241853999787 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.07511831470544343, "baseline_full": 0.06768977925381477, "valid_batches": 5, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 7, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.18430540666666667, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": null, "baseline_matched": 0.3548349036483021, "baseline_full": 0.3567919219293358, "valid_batches": 11, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 1, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.3548349036483021, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 60.94710854099999 }, "cost": 16.301110204081635, "average_task_cost_estimate": 0.08875048888888888, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 60.94710854099999, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "zai/glm-5.3@high", "model_id": "zai/glm-5.3", "model_label": "GLM 5.3", "effort": "high", "label": "GLM 5.3 \u00b7 High", "short_label": "GLM 5.3 High", "score": 76.66666666666667, "passed": 69, "attempts": 90, "tokens": 33932.12222222222, "ci": [ 65.53, 87.78 ], "categories": { "Documents": { "passed": 15, "attempts": 18, "rate": 83.33333333333333 }, "Takeoffs": { "passed": 8, "attempts": 18, "rate": 44.44444444444444 }, "Analysis": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Knowledge": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 }, "Apps": { "passed": 16, "attempts": 18, "rate": 88.88888888888889 } }, "difficulty": { "Routine": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Multi-step": { "passed": 14, "attempts": 15, "rate": 93.33333333333333 }, "Complex": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Expert": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Frontier": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Stress": { "passed": 8, "attempts": 15, "rate": 53.333333333333336 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 3.9176439375001935 }, "predictions": { "brier": { "metric": 0.07636218680555555, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.07636218680555555, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.5293912471811874, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.5293912471811874, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 53.8532805420002 }, "cost": 9.323078840579711, "average_task_cost_estimate": 0.07147693777777778, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 53.8532805420002, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "zai/glm-5.3@low", "model_id": "zai/glm-5.3", "model_label": "GLM 5.3", "effort": "low", "label": "GLM 5.3 \u00b7 Low", "short_label": "GLM 5.3 Low", "score": 64.44444444444444, "passed": 58, "attempts": 90, "tokens": 49479.78888888889, "ci": [ 54.44, 75.56 ], "categories": { "Documents": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Takeoffs": { "passed": 7, "attempts": 18, "rate": 38.888888888888886 }, "Analysis": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 }, "Knowledge": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Apps": { "passed": 12, "attempts": 18, "rate": 66.66666666666667 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 11, "attempts": 15, "rate": 73.33333333333333 }, "Complex": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Expert": { "passed": 10, "attempts": 15, "rate": 66.66666666666667 }, "Frontier": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Stress": { "passed": 7, "attempts": 15, "rate": 46.666666666666664 } }, "judgment": { "correct": 11, "attempts": 12, "correct_rate": 91.66666666666667, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 3.024182625000365 }, "predictions": { "brier": { "metric": 0.15270434305555555, "baseline_matched": 0.06768977925381477, "baseline_full": 0.06768977925381477, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.15270434305555555, "reporting_note": "Complete 12-batch comparison." }, "price": { "metric": 0.6267189557636639, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.6267189557636639, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 20.259806208500056 }, "cost": 13.672231724137932, "average_task_cost_estimate": 0.08810993777777777, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 20.259806208500056, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 }, { "id": "zai/glm-5.3@max", "model_id": "zai/glm-5.3", "model_label": "GLM 5.3", "effort": "max", "label": "GLM 5.3 \u00b7 Max", "short_label": "GLM 5.3 Max", "score": 71.11111111111111, "passed": 64, "attempts": 90, "tokens": 45760.688888888886, "ci": [ 60.0, 82.22 ], "categories": { "Documents": { "passed": 14, "attempts": 18, "rate": 77.77777777777777 }, "Takeoffs": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Analysis": { "passed": 13, "attempts": 18, "rate": 72.22222222222223 }, "Knowledge": { "passed": 17, "attempts": 18, "rate": 94.44444444444444 }, "Apps": { "passed": 7, "attempts": 18, "rate": 38.888888888888886 } }, "difficulty": { "Routine": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Multi-step": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Complex": { "passed": 12, "attempts": 15, "rate": 80.0 }, "Expert": { "passed": 13, "attempts": 15, "rate": 86.66666666666667 }, "Frontier": { "passed": 9, "attempts": 15, "rate": 60.0 }, "Stress": { "passed": 6, "attempts": 15, "rate": 40.0 } }, "judgment": { "correct": 12, "attempts": 12, "correct_rate": 100.0, "false_stops": 0, "solvable_attempts": 90, "false_stop_rate": 0.0, "median_correct_stop_s": 5.5258954999998675 }, "predictions": { "brier": { "metric": null, "baseline_matched": 0.058798703772410324, "baseline_full": 0.06768977925381477, "valid_batches": 7, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 5, "execution_error_batches": 0, "metric_complete": false, "valid_output_metric": 0.05866610514285714, "reporting_note": "No full-set error score: some attempts did not produce a valid forecast." }, "price": { "metric": 0.40707370130775383, "baseline_matched": 0.3567919219293358, "baseline_full": 0.3567919219293358, "valid_batches": 12, "attempted_batches": 12, "expected_batches": 12, "invalid_output_batches": 0, "execution_error_batches": 0, "metric_complete": true, "valid_output_metric": 0.40707370130775383, "reporting_note": "Complete 12-batch comparison." } }, "harness_counts": { "bidlo-api": 90 }, "mixed_harness": false, "latency_by_harness": { "bidlo-api": 73.34288087450025 }, "cost": 14.4224996875, "average_task_cost_estimate": 0.10255999777777777, "cost_basis": "standard_uncached_api_equivalent_estimate", "latency": 73.34288087450025, "latency_basis": "observed_harness_time_for_successful_attempts", "cost_known_attempts": 90, "tokens_known_attempts": 90, "unresolved_served_identity_count": 0 } ], "historical_comparisons": { "source": "/benchmarks/bid-bench-1.1-2026-09-30.json", "run_date": "September 28\u201330, 2026 UTC", "configurations": 80, "complete": false, "note": "September 28\u201330 measurements are descriptive comparisons. Different API/CLI orchestration, budgets, context and serving conditions prevent controlled latency or token comparisons." }, "limitations": [ "All five settings reach the delivery ceiling on this small authored set; unseen-task reliability is not established.", "CLI session-selected model and effort verified; no API served-model receipt is exposed.", "Forecast cohort and candidate-panel limitations match the Bid Bench 1.1 method.", "Token and latency comparisons to historical configurations reflect different harnesses." ] }