{
  "generated_at": "2026-08-15T09:58:33.889347+00:00",
  "task": "construction quantity takeoff — fill a 27-line Mexican budget from DXF drawings and a soil study, leaving a cell blank rather than inventing a figure",
  "metrics": {
    "invented": "a quantity with NO cited source — a fabricated number",
    "scope_violations": "a REAL, correctly-measured figure applied to the WRONG EXTENT (e.g. the site boundary used for a zone-limited concept). More dangerous than an invented value: the citation is genuine, so every other check passes.",
    "filled": "lines given a quantity. NOT a quality score on its own — a blank is frequently the correct answer when the drawings lack scale. Ranking candidates by this field is the single easiest way to reach the WRONG conclusion here.",
    "validator_rejected": "the engine's own validator refused the answer (e.g. amount != quantity x unit_price). DISQUALIFYING and invisible in every other field: such a run can show the highest `filled` with invented=0 and scope_violations=0."
  },
  "verdict_rule": "any invented value, scope violation, or validator rejection disqualifies, regardless of cost or fill count",
  "reading_guide": "Do not rank by `filled`. Disqualify first on invented / scope_violations / validator_rejected, and only then compare cost and latency among survivors. A candidate with filled=0 has not passed — it abstained, which is untested rather than safe. Agreement figures are only as strong as the reference run: check the matching entry in `reference_runs` for how many lines the reference itself filled before trusting a high agreement score.",
  "reference_runs": [
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 4,
      "blank": 23,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 13,
      "input_tokens": 2,
      "output_tokens": 11739,
      "latency_ms": 131873,
      "ran_at": "2026-08-05T17:03:57.387596+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 4,
      "blank": 23,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 33,
      "input_tokens": 2,
      "output_tokens": 14516,
      "latency_ms": 163750,
      "ran_at": "2026-08-05T16:18:42.621713+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 4,
      "blank": 23,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 19,
      "input_tokens": 2,
      "output_tokens": 15589,
      "latency_ms": 191872,
      "ran_at": "2026-08-05T14:26:31.476552+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 4,
      "blank": 23,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 14,
      "input_tokens": 2,
      "output_tokens": 13274,
      "latency_ms": 146024,
      "ran_at": "2026-08-05T13:23:05.251373+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 4,
      "blank": 23,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 16,
      "input_tokens": 2,
      "output_tokens": 13979,
      "latency_ms": 155303,
      "ran_at": "2026-08-05T05:10:47.058385+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 5,
      "blank": 22,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 16,
      "input_tokens": 2,
      "output_tokens": 15839,
      "latency_ms": 180318,
      "ran_at": "2026-08-05T03:48:40.850028+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 3,
      "blank": 24,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 18,
      "input_tokens": 2,
      "output_tokens": 15612,
      "latency_ms": 175695,
      "ran_at": "2026-08-04T18:31:03.384756+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 4,
      "blank": 23,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 13,
      "input_tokens": 2,
      "output_tokens": 13626,
      "latency_ms": 159370,
      "ran_at": "2026-08-04T16:02:59.068231+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 1,
      "blank": 26,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 12,
      "input_tokens": 2,
      "output_tokens": 10874,
      "latency_ms": 125541,
      "ran_at": "2026-08-04T15:46:46.414975+00:00"
    },
    {
      "model": "claude-opus-5",
      "provider": "agent-sdk",
      "cost_basis": "subscription",
      "filled": 1,
      "blank": 26,
      "invented": 0,
      "scope_violations": 0,
      "validator_rejected": false,
      "findings": 16,
      "input_tokens": 2,
      "output_tokens": 11799,
      "latency_ms": 134371,
      "ran_at": "2026-08-04T15:01:43.301824+00:00"
    }
  ],
  "bakeoffs": [
    {
      "experiment": "1b0fcbac",
      "label": "bakeoff 2026-08-15 15:27",
      "ran_at": "2026-08-15T08:30:03.582399+00:00",
      "candidates": [
        {
          "model": "Qwen/Qwen3-235B-A22B-Instruct-2507",
          "provider": "deepinfra",
          "cost_basis": "api",
          "filled": 1,
          "blank": 26,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 7,
          "input_tokens": 46846,
          "output_tokens": 4530,
          "latency_ms": 163213,
          "ran_at": "2026-08-15T08:30:03.582399+00:00"
        }
      ]
    },
    {
      "experiment": "37c6aa0d",
      "label": "bakeoff 2026-08-15 15:23",
      "ran_at": "2026-08-15T08:27:14.024572+00:00",
      "candidates": [
        {
          "model": "openai/gpt-oss-120b",
          "provider": "together",
          "cost_basis": "api",
          "filled": 4,
          "blank": 23,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 25,
          "input_tokens": 39366,
          "output_tokens": 9865,
          "latency_ms": 237935,
          "ran_at": "2026-08-15T08:27:14.024572+00:00"
        }
      ]
    },
    {
      "experiment": "d2d767e1",
      "label": "bakeoff 2026-08-15 15:18",
      "ran_at": "2026-08-15T08:23:06.012758+00:00",
      "candidates": [
        {
          "model": "Qwen/Qwen3-235B-A22B-Instruct-2507",
          "provider": "deepinfra",
          "cost_basis": "api",
          "filled": 1,
          "blank": 26,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 28,
          "input_tokens": 46846,
          "output_tokens": 6118,
          "latency_ms": 273829,
          "ran_at": "2026-08-15T08:23:06.012758+00:00"
        }
      ]
    },
    {
      "experiment": "1da6fe6f",
      "label": "bakeoff 2026-08-15 15:17",
      "ran_at": "2026-08-15T08:18:32.37994+00:00",
      "candidates": [
        {
          "model": "openai/gpt-oss-120b",
          "provider": "together",
          "cost_basis": "api",
          "filled": 4,
          "blank": 23,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 25,
          "input_tokens": 39368,
          "output_tokens": 9289,
          "latency_ms": 69205,
          "ran_at": "2026-08-15T08:18:32.37994+00:00"
        },
        {
          "model": "Qwen/Qwen3-235B-A22B-Instruct-2507",
          "provider": "deepinfra",
          "cost_basis": "api",
          "filled": 3,
          "blank": 24,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 27,
          "input_tokens": 46846,
          "output_tokens": 6025,
          "latency_ms": 270010,
          "ran_at": "2026-08-15T08:23:03.240788+00:00"
        }
      ]
    },
    {
      "experiment": "14cff688",
      "label": "bakeoff 2026-08-15 15:17",
      "ran_at": "2026-08-15T08:18:29.762128+00:00",
      "candidates": [
        {
          "model": "openai/gpt-oss-120b",
          "provider": "together",
          "cost_basis": "api",
          "filled": 0,
          "blank": 27,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 30,
          "input_tokens": 39368,
          "output_tokens": 8555,
          "latency_ms": 56040,
          "ran_at": "2026-08-15T08:18:29.762128+00:00"
        },
        {
          "model": "Qwen/Qwen3-235B-A22B-Instruct-2507",
          "provider": "deepinfra",
          "cost_basis": "api",
          "filled": 1,
          "blank": 26,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 15,
          "input_tokens": 46846,
          "output_tokens": 5142,
          "latency_ms": 333091,
          "ran_at": "2026-08-15T08:24:03.704013+00:00"
        }
      ]
    },
    {
      "experiment": "91c41442",
      "label": "bakeoff 2026-08-15 15:17",
      "ran_at": "2026-08-15T08:18:25.451655+00:00",
      "candidates": [
        {
          "model": "openai/gpt-oss-120b",
          "provider": "together",
          "cost_basis": "api",
          "filled": 0,
          "blank": 27,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 29,
          "input_tokens": 39368,
          "output_tokens": 5964,
          "latency_ms": 40549,
          "ran_at": "2026-08-15T08:18:25.451655+00:00"
        }
      ]
    },
    {
      "experiment": "d3561dc3",
      "label": "bakeoff 2026-08-15 15:11",
      "ran_at": "2026-08-15T08:12:59.316667+00:00",
      "candidates": [
        {
          "model": "openai/gpt-oss-120b",
          "provider": "together",
          "cost_basis": "api",
          "filled": 8,
          "blank": 19,
          "invented": 0,
          "scope_violations": 1,
          "validator_rejected": false,
          "findings": 7,
          "input_tokens": 39368,
          "output_tokens": 10924,
          "latency_ms": 81024,
          "ran_at": "2026-08-15T08:12:59.316667+00:00"
        }
      ]
    },
    {
      "experiment": "e43431c5",
      "label": "bakeoff 2026-08-15 15:07",
      "ran_at": "2026-08-15T08:08:23.321055+00:00",
      "candidates": [
        {
          "model": "openai/gpt-oss-120b",
          "provider": "together",
          "cost_basis": "api",
          "filled": 3,
          "blank": 24,
          "invented": 0,
          "scope_violations": 2,
          "validator_rejected": false,
          "findings": 26,
          "input_tokens": 39368,
          "output_tokens": 10037,
          "latency_ms": 63063,
          "ran_at": "2026-08-15T08:08:23.321055+00:00"
        },
        {
          "model": "Qwen/Qwen3-235B-A22B-Instruct-2507",
          "provider": "deepinfra",
          "cost_basis": "api",
          "filled": 0,
          "blank": 27,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 3,
          "input_tokens": 46846,
          "output_tokens": 4116,
          "latency_ms": 189756,
          "ran_at": "2026-08-15T08:11:33.961101+00:00"
        }
      ]
    },
    {
      "experiment": "2757f77b",
      "label": "bakeoff 2026-08-15 12:57",
      "ran_at": "2026-08-15T05:59:20.928116+00:00",
      "candidates": [
        {
          "model": "Qwen/Qwen3-235B-A22B-Instruct-2507",
          "provider": "deepinfra",
          "cost_basis": "api",
          "filled": 4,
          "blank": 23,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 7,
          "input_tokens": 46846,
          "output_tokens": 5116,
          "latency_ms": 125677,
          "ran_at": "2026-08-15T05:59:20.928116+00:00"
        }
      ]
    },
    {
      "experiment": "2e9024c3",
      "label": "bakeoff 2026-08-15 12:55",
      "ran_at": "2026-08-15T05:56:37.829957+00:00",
      "candidates": [
        {
          "model": "openai/gpt-oss-120b",
          "provider": "together",
          "cost_basis": "api",
          "filled": 3,
          "blank": 24,
          "invented": 0,
          "scope_violations": 1,
          "validator_rejected": false,
          "findings": 27,
          "input_tokens": 39368,
          "output_tokens": 11078,
          "latency_ms": 79715,
          "ran_at": "2026-08-15T05:56:37.829957+00:00"
        }
      ]
    },
    {
      "experiment": "84783ac2",
      "label": "bakeoff 2026-07-29 17:01",
      "ran_at": "2026-07-29T10:04:16.45296+00:00",
      "candidates": [
        {
          "model": "claude-opus-5",
          "provider": "anthropic",
          "cost_basis": "api",
          "filled": 1,
          "blank": 26,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 14,
          "input_tokens": 18987,
          "output_tokens": 12468,
          "latency_ms": 138657,
          "ran_at": "2026-07-29T10:04:16.45296+00:00"
        }
      ]
    },
    {
      "experiment": "e1d83298",
      "label": "bakeoff 2026-07-26 17:55",
      "ran_at": "2026-07-26T10:56:44.646411+00:00",
      "candidates": [
        {
          "model": "gemini-3.6-flash",
          "provider": "gemini",
          "cost_basis": "api",
          "filled": 1,
          "blank": 26,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 6,
          "input_tokens": 10870,
          "output_tokens": 4217,
          "latency_ms": 46692,
          "ran_at": "2026-07-26T10:56:44.646411+00:00"
        },
        {
          "model": "gpt-5.4-2026-03-05",
          "provider": "openai",
          "cost_basis": "api",
          "filled": 5,
          "blank": 22,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": true,
          "findings": 27,
          "input_tokens": 9595,
          "output_tokens": 5562,
          "latency_ms": 40961,
          "ran_at": "2026-07-26T10:57:26.368922+00:00"
        },
        {
          "model": "llama-3.3-70b-versatile",
          "provider": "groq",
          "cost_basis": "api",
          "filled": 0,
          "blank": 27,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 3,
          "input_tokens": 9911,
          "output_tokens": 2894,
          "latency_ms": 5705,
          "ran_at": "2026-07-26T10:57:33.050066+00:00"
        },
        {
          "model": "kimi-for-coding",
          "provider": "kimi-cli",
          "cost_basis": "subscription",
          "filled": 4,
          "blank": 23,
          "invented": 0,
          "scope_violations": 3,
          "validator_rejected": false,
          "findings": 13,
          "input_tokens": null,
          "output_tokens": null,
          "latency_ms": 110735,
          "ran_at": "2026-07-26T10:59:24.225664+00:00"
        }
      ]
    },
    {
      "experiment": "0434d68f",
      "label": "bakeoff 2026-07-26 12:32",
      "ran_at": "2026-07-26T05:33:28.830314+00:00",
      "candidates": [
        {
          "model": "gemini-3.6-flash",
          "provider": "gemini",
          "cost_basis": "api",
          "filled": 1,
          "blank": 26,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 23,
          "input_tokens": 10870,
          "output_tokens": 5958,
          "latency_ms": 51491,
          "ran_at": "2026-07-26T05:33:28.830314+00:00"
        },
        {
          "model": "gpt-5.4-2026-03-05",
          "provider": "openai",
          "cost_basis": "api",
          "filled": 4,
          "blank": 23,
          "invented": 0,
          "scope_violations": 3,
          "validator_rejected": false,
          "findings": 19,
          "input_tokens": 9595,
          "output_tokens": 4450,
          "latency_ms": 30995,
          "ran_at": "2026-07-26T05:34:01.252442+00:00"
        },
        {
          "model": "llama-3.3-70b-versatile",
          "provider": "groq",
          "cost_basis": "api",
          "filled": 0,
          "blank": 27,
          "invented": 0,
          "scope_violations": 0,
          "validator_rejected": false,
          "findings": 29,
          "input_tokens": 9911,
          "output_tokens": 4514,
          "latency_ms": 8446,
          "ran_at": "2026-07-26T05:34:10.429802+00:00"
        }
      ]
    },
    {
      "experiment": "15dce07e",
      "label": "bakeoff 2026-07-26 12:11",
      "ran_at": "2026-07-26T05:13:12.87269+00:00",
      "candidates": [
        {
          "model": "kimi-for-coding",
          "provider": "kimi-cli",
          "cost_basis": "subscription",
          "filled": 4,
          "blank": 23,
          "invented": 0,
          "scope_violations": 3,
          "validator_rejected": false,
          "findings": 12,
          "input_tokens": null,
          "output_tokens": null,
          "latency_ms": 101481,
          "ran_at": "2026-07-26T05:13:12.87269+00:00"
        }
      ]
    }
  ]
}