Search Plus UI Blocks

Find a block, a category or a site

Model benchmarks with latency and calibrationbenchmarks-calibration-table

Use in Plus UI
npx shadcn@latest add https://blocks.plusui.com/r/benchmarks-calibration-table.json

Stats · 13 variants

← → to step through · + on a thumbnail to compare side by side (up to 3)

/preview/benchmarks-calibration-table
Open ↗

Default content

{
  "eyebrow": "Benchmarks",
  "title": "Measured on decisions, not on essays",
  "description": "Arbitrel's own decision suite, run in September 2026. The other columns are general chat models prompted to answer with one label and parsed; we publish the prompts and the data.",
  "models": [
    {
      "name": "arbitrel-d2",
      "ours": true
    },
    {
      "name": "arbitrel-d2-mini",
      "ours": true
    },
    {
      "name": "Chat model A",
      "ours": false
    },
    {
      "name": "Chat model B",
      "ours": false
    }
  ],
  "tasks": [
    {
      "name": "Support ticket routing (12 queues)",
      "metric": "Accuracy",
      "scores": [
        93.8,
        91.2,
        89.4,
        86.1
      ]
    },
    {
      "name": "Policy violation, yes/no",
      "metric": "Macro F1",
      "scores": [
        95.1,
        93.7,
        92.8,
        90.2
      ]
    },
    {
      "name": "Search relevance, 1–5",
      "metric": "Exact match",
      "scores": [
        71.4,
        67.9,
        63,
        58.7
      ]
    },
    {
      "name": "Claim vs. source",
      "metric": "Accuracy",
      "scores": [
        90.6,
        87.3,
        88.1,
        83.9
      ]
    },
    {
      "name": "Schema-valid outputs",
      "metric": "% valid",
      "scores": [
        100,
        100,
        97.2,
        94.6
      ]
    }
  ],
  "latency": {
    "title": "Latency per decision",
    "p50": [
      84,
      52,
      610,
      940
    ],
    "p95": [
      210,
      130,
      1480,
      2300
    ]
  },
  "calibration": {
    "title": "Calibration",
    "description": "When arbitrel-d2 says 0.8, it is right about 80% of the time. Chat models asked for a confidence are overconfident at the top.",
    "bins": [
      0.55,
      0.65,
      0.75,
      0.85,
      0.95
    ],
    "curves": [
      [
        0.54,
        0.66,
        0.74,
        0.86,
        0.94
      ],
      [
        0.52,
        0.63,
        0.76,
        0.83,
        0.93
      ],
      [
        0.49,
        0.55,
        0.62,
        0.7,
        0.81
      ],
      [
        0.45,
        0.5,
        0.57,
        0.64,
        0.74
      ]
    ],
    "ece": [
      0.018,
      0.027,
      0.094,
      0.141
    ]
  },
  "labels": {
    "task": "Task",
    "metric": "Metric",
    "ours": "Ours",
    "p50": "Median",
    "p95": "p95",
    "ms": "ms",
    "predicted": "Stated confidence",
    "observed": "Observed accuracy",
    "perfect": "Perfect calibration",
    "ece": "ECE"
  },
  "methodology": "Arbitrel decision suite v3: 18,400 labelled items from five public datasets and two donated by customers, relabelled by three annotators. Single run, temperature 0, measured from a client in Frankfurt on 2026-09-22. Prompts and raw outputs are on GitHub.",
  "action": {
    "label": "Download the full report",
    "href": "#",
    "variant": "secondary"
  }
}