{
  "id": 18448878,
  "title": "Benchmarks as Software: Terminal-Bench and the Case for Continuous Benchmarks",
  "url": "https://urgent.news/2026/11/05/benchmarks-as-software-terminal-bench-and-the-case-for-continuous",
  "topic": "tech",
  "section": "Tech",
  "published": "2026-11-05T18:18:39.000Z",
  "source": {
    "name": "arXiv cs.AI",
    "slug": "arxiv-cs-ai",
    "url": "https://arxiv.org/abs/2611.08417v1"
  },
  "original_language": "en",
  "account": null,
  "summary": "Coding agents are among the most economically consequential deployments of LLMs to date, making reliable evaluation increasingly important. Yet even benchmarks that are carefully validated at release can become stale. As models improve, they can discover valid solutions that existing verifiers reject, while changes to evaluation infrastructure can invalidate assumptions embedded in tasks and…",
  "key_points": [],
  "editors_take": null,
  "illustration": null,
  "coverage": {
    "outlets": 1,
    "also_reported_by": []
  },
  "ai_generated": true,
  "disclaimer": "Summaries, key points and the editor’s take are written by software from other outlets’ reporting and may contain errors — always check the linked original."
}