1:I[195232,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"LoadingBoundaryProvider"] 2:"$Sreact.fragment" 3:I[869117,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"NetworkActivity"] 4:I[814062,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"SessionProvider"] 5:I[951529,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"Shell"] 6:I[195232,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"default"] 7:I[371653,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js","/_next/static/immutable/chunks/1eimg0j34ed-k.js"],"default"] 8:I[16439,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"default"] 10:I[966765,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"default",1] :HL["/_next/static/immutable/chunks/1g7csgrqtlfkz.css","style"] :HL["/_next/static/immutable/chunks/3k5f1v0v_00uw.css","style"] :HL["/_next/static/immutable/chunks/3aohrjh7bn_av.css","style"] :HL["/_next/static/immutable/media/aaeb013c77132fba.p.11vj41fawvu--.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/_next/static/immutable/media/caa3a2e1cccd8315-s.p.0zr6hhvz-h9nw.woff2","font",{"crossOrigin":"","type":"font/woff2"}] e:X 0:{"P":null,"c":["","developers","docs","changelog"],"q":"","i":false,"f":[[["",{"children":["developers",{"children":["docs",{"children":["changelog",{"children":["__PAGE__",{},"$undefined","$undefined",4096]},"$undefined","$undefined",4096]},"$undefined","$undefined",4096]},"$undefined","$undefined",4096]},"$undefined","$undefined",4116],[["$","$L1",null,{"loading":[["$","div","l",{"role":"status","aria-label":"Loading page","className":"w-full min-w-0","data-page-skeleton":"detail","children":[["$","span",null,{"className":"sr-only","children":"Loading page"}],["$","div",null,{"aria-hidden":"true","className":"space-y-6 py-4","children":[["$","div",null,{"className":"space-y-2","children":[["$","div",null,{"data-slot":"skeleton","className":"animate-pulse rounded-md bg-muted motion-reduce:animate-none h-8 w-48 max-w-full"}],["$","div",null,{"data-slot":"skeleton","className":"animate-pulse rounded-md bg-muted motion-reduce:animate-none h-4 w-80 max-w-full"}]]}],[["$","div",null,{"className":"grid grid-cols-1 gap-4 md:grid-cols-3","children":[["$","div","0",{"data-slot":"skeleton","className":"animate-pulse bg-muted motion-reduce:animate-none h-28 rounded-none"}],["$","div","1",{"data-slot":"skeleton","className":"animate-pulse bg-muted motion-reduce:animate-none h-28 rounded-none"}],["$","div","2",{"data-slot":"skeleton","className":"animate-pulse bg-muted motion-reduce:animate-none h-28 rounded-none"}]]}],["$","div",null,{"data-slot":"skeleton","className":"animate-pulse bg-muted motion-reduce:animate-none h-56 w-full rounded-none"}]]]}]]}],[],null],"children":["$","$2","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/immutable/chunks/1g7csgrqtlfkz.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/immutable/chunks/3k5f1v0v_00uw.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/_next/static/immutable/chunks/19rwqzjcor83k.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/_next/static/immutable/chunks/0b31e3qldb1c1.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/_next/static/immutable/chunks/0s81a39zwttql.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/_next/static/immutable/chunks/42lqb6gnfp-ec.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/_next/static/immutable/chunks/0hwxphe_gn9vz.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/_next/static/immutable/chunks/3qzarik02d-ve.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/_next/static/immutable/chunks/3oq2rvn05ot3q.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/_next/static/immutable/chunks/2s7e2fbltj51p.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/_next/static/immutable/chunks/3sqy9_inm3v1n.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/_next/static/immutable/chunks/22opv1dsilvnc.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/_next/static/immutable/chunks/1tmi36srrzjjw.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/_next/static/immutable/chunks/1g8gja0xe2spr.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/_next/static/immutable/chunks/0_-kt8naw9vzn.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":[["$","head",null,{"children":[["$","script",null,{"dangerouslySetInnerHTML":{"__html":"(function(){var p=location.pathname.replace(/\\/+$/,\"\")||\"/\";if(p!==\"/\"&&p!==\"/suppliers\"&&[\"/home\",\"/labs\",\"/enterprise\",\"/pricing\"].indexOf(p)<0)document.documentElement.classList.add(\"app-theme\")})()"}}],["$","link",null,{"rel":"alternate","type":"text/markdown","href":"/agents.md","title":"EvalRouter for agents"}],["$","link",null,{"rel":"alternate","type":"text/plain","href":"/llms.txt","title":"llms.txt"}],["$","link",null,{"rel":"service-desc","type":"application/json","href":"/.well-known/openapi.json"}],["$","link",null,{"rel":"service-doc","type":"text/html","href":"/developers/docs"}]]}],["$","body",null,{"className":"geist_deef94d5-module__Sms4YG__variable newsreader_ee39d331-module__s6CX4q__variable journal","children":[["$","$L3",null,{}],["$","$L4",null,{"children":["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","error":"$7","errorStyles":[],"errorScripts":[["$","script","script-0",{"src":"/_next/static/immutable/chunks/1eimg0j34ed-k.js","async":true}]],"template":["$","$L8",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$L9","forbidden":"$undefined","unauthorized":"$undefined"}]}]}]]}]]}]]}]}],{"children":["$La",{"children":["$Lb",{"children":["$Lc",{"children":["$Ld",{},null,false,null]},null,false,"$e"]},null,false,null]},null,false,null]},null,false,null],"$Lf",false]],"m":"$undefined","G":["$10",["$L11","$L12"]],"S":false,"h":null,"r":"$undefined","s":"$undefined","a":"$undefined","l":"$undefined","p":"$undefined","d":"$undefined"} 13:I[53258,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],""] 14:I[126441,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js","/_next/static/immutable/chunks/09zowaxycz7bm.js"],"DeveloperFrame"] 15:I[610238,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js","/_next/static/immutable/chunks/09zowaxycz7bm.js","/_next/static/immutable/chunks/1s_5by4_edxu0.js"],"DocumentationSite"] 2a:I[904771,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"OutletBoundary"] 2b:"$Sreact.suspense" 2d:I[904771,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"ViewportBoundary"] 2f:I[904771,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"MetadataBoundary"] 9:[["$","div",null,{"className":"empty","children":[["$","h1",null,{"children":"There’s nothing at this address."}],["$","p",null,{"children":"Return to the catalog to explore benchmarks and interactive environments."}],["$","$L13",null,{"href":"/catalog","children":"Catalog","data-slot":"button","className":"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive bg-primary text-primary-foreground hover:bg-primary/90 h-9 px-4 py-2 has-[>svg]:px-3","ref":null}]]}],[]] a:["$","$2","c",{"children":[[["$","script","script-0",{"src":"/_next/static/immutable/chunks/09zowaxycz7bm.js","async":true,"nonce":"$undefined"}]],["$","$L14",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L8",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]}]]}] 16:T4a5, EvalRouter Run models and agents on benchmarks. See what each completed, what it cost, and the evidence behind the result. You can also turn your own data into a private benchmark. - Catalog: Find benchmarks, models and agents that can run together. - Runs: Quote a benchmark with a model or agent, then start the run. You can cap what it may charge; without a cap, your available credit bounds it. Follow it, inspect scores and errors, and compare results on the same benchmark, tasks and protocol. - Credits: Runs charge recorded usage against workspace credit, never more than the cap the quote stated. If a run reaches its cap, its completed work stays available. Choose a workflow - [Run your first evaluation from the CLI](/developers/docs/quickstart). - [Read and export a result](/developers/docs/results). Where private imports are enabled, [import a Hugging Face dataset](/developers/docs/huggingface) or [upload a file](/developers/docs/byob). These benchmarks stay private to your workspace. [Model compatibility](/developers/docs/selection) explains which combinations can run. [Credits and billing](/developers/docs/quotes-budgets) explains holds and actual charges. 17:Tb2d, Run a model evaluation Choose a benchmark and a model, review a quote, and start the run. An optional spending cap limits what it may charge; without one, your workspace credit bounds it. A model provider you connect may bill inference separately. For a free introduction, open [onboarding](/onboarding). Its recorded comparison consumes no credits. You need workspace credit to start real work. Install and sign in Install [uv](https://docs.astral.sh/uv/getting-started/installation/), then install the CLI: sh uv tool install evalrouter evalrouter --help uv sets up the CLI and a compatible Python. If your shell cannot find evalrouter, run uv tool update-shell and restart your terminal. Update later with evalrouter upgrade. Sign in through your browser: sh evalrouter login evalrouter whoami The CLI selects a workspace after sign-in. See [CLI sign-in](/developers/docs/cli-login) for remote terminals and CI. Installing the package does not add credit. A new account gets one $5 welcome credit in its workspace. Run in one step sh evalrouter run gpqa-diamond --model gpt-4o-mini --sample 25 The CLI quotes the run, shows its maximum charge, asks you to confirm, starts it and follows it to the results. In a script, pass --yes; without it nothing is quoted or charged (exit status 5). Add --max-cost USD to cap the run: it starts only if the quote's maximum charge fits under that cap. Quote, then start Find benchmark profile and model IDs: sh evalrouter benchmarks list --json evalrouter models list --json Save a quote request as quote.json, replacing the example IDs with catalog values. To cap the run, add max_charge_microusd in micro-USD ("2000000" is $2); without it the run is bounded by your workspace credit: json { "model": {"kind": "managed", "route_id": "MODEL_ROUTE_ID"}, "selection": {"profile_ids": ["BENCHMARK_PROFILE_ID"]}, "coverage": {"mode": "sample", "sample_count": 3, "seed": 42} } Review the quote, then start it and wait for the result: sh evalrouter quotes create --body quote.json --json evalrouter runs create --quote QUOTE_ID --yes --wait --json evalrouter runs results RUN_ID evalrouter runs export RUN_ID --file-format html,json A quote checks compatibility, coverage and the maximum charge without making model calls. Starting the same quote again returns the same run, so retrying after a dropped connection never starts a second one. See [CLI commands](/developers/docs/cli) for every command, output formats and exit statuses. Use HTTP Create a [workspace API key](/developers/docs/authentication) with eval:read and eval:write. Send the same body to POST /v1/quotes, then start the returned quote with POST /v1/runs, restating its cap. Follow GET /v1/runs/{run_id} and read GET /v1/runs/{run_id}/results. The [public API schema](/developers/docs/openapi.json) lists all fields. 18:T652, Sign-in and automation keys For terminal use, run evalrouter login and approve sign-in in your browser. The CLI remembers your session and workspace. You do not need an API key. See [CLI sign-in](/developers/docs/cli-login). Workspace keys for CI and services For CI or a service, [create an account](/register), verify your email, and create a [workspace API key](/settings/api-keys) with only the permissions it needs. A device sign-in can read the credit balance, but it cannot create keys, add credit, or change billing settings. Set these through your secret manager or a private shell environment: - EVALROUTER_API_KEY: the workspace key's secret value. - EVALROUTER_WORKSPACE_ID: that workspace's ID. - EVALROUTER_BASE_URL: https://api.evalrouter.ai, without /v1. Keep credentials out of command arguments, URLs, source control, screenshots, and shared logs. A key belongs to its workspace, not to the person who created it, so it keeps working if they leave or change role. The API keys page shows who created each key and when it was last used; revoke keys you no longer need. To replace a key, roll it: you get a new key with the same permissions while the old one keeps working for up to seven days. You are emailed a week before a key expires, and a key found in public code is revoked automatically. A key grants only its workspace permissions. It does not add credit or make an unavailable evaluation runnable. If a key fails, check its status and workspace before retrying. Do not start an evaluation to diagnose authentication. Continue with the [CLI quickstart](/developers/docs/quickstart). 19:T16fb, Sign in from the CLI Sign in from your terminal. If you are new to EvalRouter, you can create your account in the browser it opens; verify your email when asked. sh evalrouter login When the browser can return to this computer, the CLI offers to open it. Press Enter, choose your account, and finish sign-in at auth.kimpton.ai. The browser returns to the waiting CLI; there is no device code. The browser page then shows the result: the email you signed in as, or what went wrong. If sign-in wasn't completed, choose Try again. To sign in with a one-time code instead, run: sh evalrouter login --no-browser The CLI prints a code and URL. Open the URL on a device with a browser and approve the code. Keep the command running while you do this. The CLI also uses this flow if the browser callback is unavailable, and says so. For a device-code sign-in, check that the account and code are yours, then choose Continue to Kimpton sign-in. The next page is auth.kimpton.ai: 1. Confirm your terminal code. 2. Sign in if asked, even if EvalRouter already recognizes your account. 3. Keep the tab open until it says Device connected. Confirming the code alone does not finish sign-in. Keep the terminal command running. When sign-in finishes, it shows your email and workspace. It selects your only workspace, the one named with --workspace ID_OR_NAME, or one you pick. Connecting costs nothing; paid runs need credit. If sign-in fails, the browser page or the terminal says why. Fix that, then run evalrouter login again. A failed attempt leaves older CLI credentials in place. If you visit /device directly during a code flow, run evalrouter login first and enter the code it gives you. If the code expires or you close the browser before completion, cancel with Ctrl+C and run evalrouter login again. The CLI issues one code per command and exits on expiry. Your account and workspace are preserved. Use Use another account if the browser shows the wrong identity. Older CLI versions may start with a device code even when the browser callback is available, or wait for account setup after device approval. Update with uv tool upgrade evalrouter for the current flow. On a remote machine, open the code URL on your own device. For unattended agents and CI, use a scoped workspace API key. The CLI never asks for your password. It saves and refreshes sessions locally. Run evalrouter whoami to check your account and workspace. | Command | Purpose | | --- | --- | | login [--workspace ID_OR_NAME] [--no-browser] | Sign this device in and choose a workspace. | | login --with-api-key | Store a workspace API key instead, read from stdin or a hidden prompt. | | whoami | Show the signed-in account, workspace, role and scopes. | | workspaces list | List your workspaces and which ones this device is signed in to. | | workspaces switch [WORKSPACE] | Use another workspace, by ID or name, without signing in again. At a terminal, omit it to choose from a list. | | logout [--all] | Revoke this device's sign-in and delete its stored credentials, including a stored API key. | Use any of your sign-in methods Signing in with Google or with email and password opens the same EvalRouter account when both use the same verified email. This works on the website and in the terminal. The terminal always shows the email you signed in as, so check that it is the account you expect. Settings > Account > Sign-in methods lists the methods connected to your account. Store a workspace API key Where browser sign-in is not an option, store a [workspace key](/developers/docs/authentication) with the same credential storage. The CLI never takes the key as a command argument. In a terminal it asks with hidden input; otherwise it reads the key from stdin: sh evalrouter login --with-api-key EvalRouter API key (input hidden): printf '%s' "$EVALROUTER_KEY_FROM_SECRET_MANAGER" | evalrouter login --with-api-key The CLI checks the key with the API before saving it and never prints it. One server keeps either a device sign-in or a stored key: storing a key signs this device out (revoking its sign-in), and evalrouter login replaces a stored key. evalrouter logout deletes it. Precedence and automation The CLI uses the first of these it finds: EVALROUTER_API_KEY, then a key stored with login --with-api-key, then a stored sign-in. EVALROUTER_API_KEY always takes precedence, so scripts and CI keep using workspace keys unchanged. In a non-interactive shell the CLI never prompts; without a key or a stored sign-in it fails with an error asking you to run evalrouter login or set EVALROUTER_API_KEY. --json keeps the result on stdout and the sign-in code on stderr. Access and revocation Each workspace a device uses is a separate signed-in device entry. It grants evaluation reads and read-only access to the workspace credit balance and billing records, plus evaluation writes when your workspace role allows them. Your current workspace membership and role are checked on every request: if you are removed from a workspace, the device stops working there immediately. A CLI sign-in cannot create API keys, add credit or change billing settings. A device signed in before credit reads were available is asked to run evalrouter login once more; it is never upgraded silently. Review and revoke signed-in devices in [account settings](/settings/account). Revocation takes effect on the device's next request. evalrouter logout revokes the device on the server and ends the sign-in before deleting local credentials. Local credential storage The CLI uses an available operating-system keychain, falling back to a private file readable only by your user. No extra installation is needed for sign-in. A system permission prompt may appear when the keychain is used. Tokens are never printed. logout removes the stored session as well as revoking device access. 1a:T4d57, CLI and interactive workspace Install the evalrouter command with [uv](https://docs.astral.sh/uv/getting-started/installation/): sh uv tool install evalrouter evalrouter uv sets up the CLI and a compatible Python in its own environment. If your shell cannot find evalrouter, run uv tool update-shell and restart your terminal. Update later with evalrouter upgrade (or uv tool upgrade evalrouter). The interactive workspace checks for newer releases in the background. The check never installs anything or delays startup; set EVALROUTER_NO_UPDATE_CHECK=1 to turn it off. Scripts, --json and pinned installations skip it. [Sign in with evalrouter login](/developers/docs/cli-login), then follow the [quickstart](/developers/docs/quickstart). Command shape Every resource command is evalrouter , and each one maps to one API operation: sh evalrouter benchmarks list evalrouter models list evalrouter quotes create gsm8k --model gpt-5 evalrouter runs create --quote QUOTE_ID --yes --wait evalrouter runs list evalrouter runs get RUN_ID evalrouter runs results RUN_ID evalrouter runs export RUN_ID --file-format html,json A few shortcuts stand on their own: run (quote, confirm, start and follow in one step), agents list, login, logout, whoami, open, upgrade, completion and api. evalrouter api PATH sends one raw request (-X METHOD, --body FILE) for anything without its own command. Run evalrouter alone for the interactive workspace, evalrouter --help for the commands by purpose, and any command with --help for its flags and examples. The [command reference](command-reference) below lists every resource command. A command from an earlier release (status, wait, catalog, quote, export, compare, resume and so on) prints one line naming its replacement and exits 2. Start a run run quotes the run, shows the price, asks you to confirm, starts the run and follows it: sh evalrouter run gsm8k --model gpt-5 evalrouter run gsm8k --model gpt-5 --dry-run the quote; nothing is charged evalrouter run gsm8k --model gpt-5 --yes --json evalrouter run gsm8k --model gpt-5 --max-cost 5 with a spending cap evalrouter run --from RUN_ID the same setup again, as a new run Benchmarks and models are names: a slug or an unambiguous name (gsm8k@2 pins a version); exact IDs work too. At a terminal, a missing or ambiguous name is a choice. --max-cost USD is the most the run may charge: the run starts only when the quote's maximum charge fits under it. It is optional; without it the run is bounded by your wallet balance, and the confirmation says so. Your API key's spend ceiling bounds everything. Without a terminal (scripts, CI and agents), a paid start needs --yes; without it nothing is quoted or charged and the command exits 5. --dry-run prints the quote (maximum charge and plan), starts nothing and needs no confirmation. Starting a quote is idempotent: repeating the same command after a dropped connection returns the same run, never a second one. The two-step path keeps the quote reviewable on its own: sh evalrouter quotes create gsm8k --model gpt-5 --json evalrouter run --quote QUOTE_ID --yes --json Several models in one run Repeat --model to run two to eight models on the same tasks in one run, under one spending cap or your wallet: sh evalrouter run gsm8k --model gpt-5 --model claude-sonnet-5 --dry-run one quote for every model evalrouter run gsm8k --model gpt-5 --model claude-sonnet-5 --benchmark gpqa-diamond evalrouter runs compare RUN_ID rank, model, score, 95% range evalrouter runs continue RUN_ID finish an interrupted run The quote lists every model, and each one's share of the cap when the cap is split. --reasoning applies to every model. runs compare compares the models on the tasks every model finished: text gsm8k@6 · Accuracy · 1,319 tasks claude-sonnet-5-5 92.7% vs gpt-4.1-mini 84.2% on 1,319 tasks · difference 8.5 percentage points (6.9–10.1) 1 claude-sonnet-5-5 92.7% (91.3–94.0) 2 gpt-4.1-mini 84.2% (82.2–86.1) Difference 8.5 percentage points (6.9–10.1) The second line is the top two rows in one line ("(N models)" is added when there are more). Each row is rank (1= for a tie), model, score and 95% range. Difference (the top model minus the other) appears with exactly two models. A run compared on fewer tasks than planned says "300 of 1,319 tasks". --json adds every pair. A run stopped by its spending cap, your credit or a cancellation shows as interrupted with the reason. runs continue quotes only its unfinished tasks, asks you to confirm and starts the run that completes it; finished tasks are reused at no charge. The original run then shows as continued, and runs compare on it compares the completed run. Evaluate an agent --agent takes one value; --model is the model the agent uses through the run's metered gateway: sh evalrouter agents list ACP Registry agents and their integrity evalrouter run gsm8k --model gpt-5 --agent opencode the pinned snapshot's version evalrouter run gsm8k --model gpt-5 --agent opencode@1.18.34 evalrouter run gsm8k --model gpt-5 --agent "local:my-agent --acp" evalrouter run gsm8k --model gpt-5 --agent remote:my-agent a remote a2a agent, by connection name; it calls the model through EvalRouter evalrouter run gsm8k --model gpt-5 --agent opencode --hosted run hosted by EvalRouter (or --agent hosted:opencode) Registry and local agents run on this machine: EvalRouter delivers each task, meters every model call and grades the submission. agents list reads the ACP Registry snapshot bundled with the CLI (no API call) and labels each agent pinned (a binary with a published sha256), package-registry (an exact npm or PyPI package) or unverified-upstream. An unverified-upstream agent runs only with --allow-unverified; its observed hash is recorded. Scripts and agents | Setting | Effect | | --- | --- | | --format human\|json\|ids (EVALROUTER_FORMAT) | json: one JSON document on stdout. ids: one ID per line. | | --json | Same as --format json. | | --no-input | Never prompt; where a choice is needed, fail with exit status 5. | | --profile NAME (EVALROUTER_PROFILE) | A named sign-in and API origin, kept separately (evalrouter login --profile NAME). | | --base-url, --workspace-id, --timeout | API origin, workspace and request deadline for this command. | A flag wins over its environment variable, which wins over the profile. evalrouter options lists them all; they go before or after the command. With --json, stdout carries exactly one JSON document and nothing else. Progress goes to stderr, and only on a terminal. An error is one object on stderr, with stdout left empty: json {"error": {"code": "confirmation_required", "message": "…", "hint": "…", "retryable": false, "request_id": null, "details": null}} --json, CI=true and known agent environments imply --no-input. Colour is off without a terminal, with NO_COLOR, or with TERM=dumb. Upgrades The API serves published clients from 0.5.0 on. An older CLI or SDK gets client_upgrade_required (HTTP 426) on every command, before anything runs or is charged; the CLI prints the upgrade command for how it was installed, such as uv tool upgrade evalrouter. When a newer release is available, an interactive CLI offers to upgrade before it runs the command. Exit status | Status | Meaning | | --- | --- | | 0 | Success. | | 1 | The API refused the request, or another error. | | 2 | Invalid command line or input; nothing was sent. | | 3 | The run ended without completing (failed, partial or cancelled). | | 4 | Not signed in, or the credential was rejected. | | 5 | A confirmation or choice was needed and not given, or the quote exceeds --max-cost; nothing started. | | 6 | The API could not be reached. A run you were watching keeps going on the server. | | 124 | --wait-timeout passed while waiting; the server work continues. | | 130 | Interrupted (Ctrl-C). Submitted server work is not cancelled. | Lists and pages List commands read up to --limit rows (default 50). --all reads every page, and --cursor continues after an earlier page. With --json the document includes next_cursor; it is null on the last page. In a terminal, lists print a table sized to the window; --columns, --sort, --filter KEY=VALUE, --wide and --jq EXPR shape it. benchmarks list -i, models list -i and runs list -i browse interactively. Public catalog reads (benchmarks list, benchmarks get, models list) can reuse a recent local copy when the server allows it, and say so. --refresh reads now; --offline makes no request. Quotes always check current availability and prices. Follow, read and export a run sh evalrouter runs wait RUN_ID --progress auto evalrouter runs events list RUN_ID --follow evalrouter runs results RUN_ID evalrouter runs export RUN_ID --file-format html,json --open evalrouter runs cancel RUN_ID evalrouter open RUN_ID runs wait retries temporary API failures with growing delays, and stops watching with exit status 6 after --disconnect-timeout (default 90 seconds) without an answer. --progress json writes one evalrouter.progress.v1 record per line on stderr. runs events list prints one JSON event per line; --follow reads until the run ends and exits like runs wait. Exports report the full path of every file saved. Without --output, files are named evalrouter-. in the current directory, and an existing name gets a -2 suffix. --file-format bundle saves the evidence ZIP. runs export RUN_ID --include-resumed saves the chain result of a run and its completed successors. comparisons report RUN_A RUN_B --file-format html downloads one descriptive comparison archive from saved results without running inference; --evidence also downloads each full export. comparisons create compares exact result versions under a release policy on the server. result-groups create --name NAME --run RUN --run RUN saves a group of existing runs for one collective report. open RUN_ID opens the run's live report page and prints its address. It never opens a browser in CI, over SSH, without a display, or with --no-browser. Reasoning and output format Reasoning defaults belong to the exact model route. --reasoning sets an explicit effort; without it the route's own default applies, and the quote shows the effective choice. A route that cannot run a requested effort is refused before any charge. Benchmark versions that grade a JSON answer may offer --output-format default|leaderboard|prompt_json|native_json|json_schema on run and quotes create. Results record the format, and scores from different formats are never ranked together. Neither setting can change on an existing quote. Connections and remote agents connections create registers your own model endpoint. The CLI never takes a key as an argument: at a terminal it asks with hidden input; scripts use --key-stdin or --key-env NAME. sh printf '%s' "$PROVIDER_KEY" | evalrouter connections create --name NAME --endpoint URL --model MODEL --key-stdin evalrouter connections check CONNECTION_ID Use a checked connection as a quote's model, {"kind": "connection", "connection_id": "…"}, or run BENCHMARK --model connection:ID. A remote agent is a connection with --protocol a2a. Its credentials come from the environment or stdin, never argv: sh evalrouter connections create --protocol a2a --name my-agent --endpoint https://agent.example.com \ --header-env Authorization=AGENT_TOKEN evalrouter run BENCHMARK --model MODEL --agent remote:my-agent --yes Headers use --header-env NAME=ENVVAR (repeatable, up to four) or --header-stdin NAME; OAuth2 client credentials use --oauth2-token-url, --oauth2-client-id, --oauth2-client-secret-env ENVVAR and optional --oauth2-scope/--oauth2-audience. --card-path names a non-default agent card. On run and quotes create, --agent-parameters @file.json, --agent-gateway-api chat_completions and --agent-max-concurrent-tasks N tune the agent. Your provider may bill model or agent requests separately. API keys and workspaces sh evalrouter api-keys create --label ci --scope eval:read --scope eval:write --spend-ceiling 20 evalrouter api-keys update KEY_ID --spend-ceiling none evalrouter api-keys roll KEY_ID evalrouter workspaces switch NAME_OR_ID A key's --spend-ceiling USD bounds what that key can spend in total; none removes it. workspaces switch registers this device for another workspace without a browser. Tab completion Add eval "$(evalrouter completion zsh)" to ~/.zshrc (after compinit) or eval "$(evalrouter completion bash)" to your Bash startup file, then open a new terminal. For fish, run evalrouter completion fish > ~/.config/fish/completions/evalrouter.fish. Tab completes commands, flags and their choices, benchmark names (type @ for versions), model names, ACP Registry agents for --agent, remote: connection names, profiles and recent run IDs. Commands, flags and agents come from the installed CLI; the other names come from a local cache, ~/.cache/evalrouter/complete.json, kept per profile, API origin and workspace and cleared by login, logout and workspaces switch. A background process refreshes it with read-only list requests when it is more than five minutes old. Pressing Tab never waits for the API and works offline. Preview commands recommend, releases, traces, datasets, improvements and tune are preview: they are marked (preview) in evalrouter --help, are outside the stability promise and may change or be removed in any release. Scripts should rely only on the commands in the reference below. Command reference | Command | What it does | |---|---| | evalrouter api-keys create | Create an API key; its secret is shown once. | | evalrouter api-keys delete KEY_ID | Revoke an API key. | | evalrouter api-keys list | List this workspace's API keys. | | evalrouter api-keys roll KEY_ID | Replace an API key; the old one keeps working for the overlap. | | evalrouter api-keys update KEY_ID | Change an API key's spend ceiling. | | evalrouter benchmarks environments get FAMILY | Show one environment family. | | evalrouter benchmarks environments list | List environment families. Pages with --limit, --all and --cursor. | | evalrouter benchmarks evals get REF | Show one evaluation reference. | | evalrouter benchmarks evals list | List evaluation references. Pages with --limit, --all and --cursor. | | evalrouter benchmarks get SLUG | Show one benchmark and its versions. | | evalrouter benchmarks imports capabilities | Show which imports this workspace can use. | | evalrouter benchmarks imports github | Import a file from a GitHub repository as a private source. | | evalrouter benchmarks imports github-preview | Preview the files of a GitHub repository revision. | | evalrouter benchmarks imports github-refs | List the branches and tags of a GitHub repository. | | evalrouter benchmarks imports huggingface | Import a Hugging Face dataset as a private source. | | evalrouter benchmarks list | List maintained benchmarks. Pages with --limit, --all and --cursor. | | evalrouter benchmarks private archive PROFILE_ID | Stop new runs of a private benchmark. | | evalrouter benchmarks private create | Create a private benchmark from a data file and a small manifest. | | evalrouter benchmarks private get PROFILE_ID | Show one private benchmark. | | evalrouter benchmarks private list | List this workspace's private benchmarks. | | evalrouter benchmarks uploads archive UPLOAD_ID | Archive an uploaded source to free a slot. | | evalrouter benchmarks uploads create | Upload local data files as a source private to this workspace. | | evalrouter benchmarks uploads get UPLOAD_ID | Show one uploaded source. | | evalrouter benchmarks uploads list | List uploaded sources and the slots they hold. | | evalrouter comparisons create | Compare exact result versions under a release policy. | | evalrouter comparisons get COMPARISON_ID | Show a comparison. | | evalrouter comparisons list | List comparisons. Pages with --limit, --all and --cursor. | | evalrouter connections check CONNECTION_ID | Test a connection with one small request. | | evalrouter connections create | Connect an OpenAI-compatible endpoint. | | evalrouter connections delete CONNECTION_ID | Stop using a connection. | | evalrouter connections list | List endpoint connections. Pages with --limit, --all and --cursor. | | evalrouter connections update CONNECTION_ID | Change a connection or rotate its key. | | evalrouter datasets approve DATASET_ID | Approve pending examples. | | evalrouter datasets benchmark DATASET_ID VERSION_ID | Build a private benchmark from a dataset version. | | evalrouter datasets import-traces DATASET_ID | Add recent production traces as held-out examples; running it confirms the workspace may evaluate on them. | | evalrouter datasets versions create DATASET_ID | Freeze the approved examples as a new version. | | evalrouter datasets versions list DATASET_ID | List a dataset's versions. | | evalrouter improvements get PROJECT_ID | Show an agent project. | | evalrouter improvements list | List agent projects. Pages with --limit, --all and --cursor. | | evalrouter improvements prepare PROJECT_ID | Fetch the source and detect its runtime. | | evalrouter improvements reconcile-runtime-fees PROJECT_ID CYCLE_ID CANDIDATE_ID | Owner or admin only: check completed work against an existing reservation; no model runs start. | | evalrouter models list | List models and their prices. Pages with --limit, --all and --cursor. | | evalrouter quotes create | Price a run and freeze its setup without starting it. | | evalrouter quotes get QUOTE_ID | Show a quote. | | evalrouter result-groups create | Group existing runs' results for reuse. | | evalrouter result-groups export GROUP_ID | Save a result group export to a file. | | evalrouter result-groups get GROUP_ID | Show a result group. | | evalrouter result-groups list | List result groups. Pages with --limit, --all and --cursor. | | evalrouter runs attempts list RUN_ID | List a run's attempts and their outcomes. Pages with --limit, --all and --cursor. | | evalrouter runs cancel RUN_ID | Request cancellation of a run. | | evalrouter runs compare RUN_ID | Compare a multi-model run's models on the tasks every model finished. | | evalrouter runs create | Start a run from a quote under a spending cap (--max-cost). | | evalrouter runs events list RUN_ID | Print a run's events as JSON lines. | | evalrouter runs export RUN_ID | Save a run's versioned export to a file. | | evalrouter runs get RUN_ID | Show one run. | | evalrouter runs list | List recent runs. Pages with --limit, --all and --cursor. | | evalrouter runs receipt RUN_ID | Show a run's frozen receipt. | | evalrouter runs results RUN_ID | Show a run's scores, coverage and uncertainty. | | evalrouter runs samples RUN_ID | List a run's per-task samples (--all pages through every one). Pages with --limit, --all and --cursor. | | evalrouter runs wait RUN_ID | Wait until it ends; interrupting never cancels the server work. | | evalrouter traces connect | Create a trace connection and print its ingest key once. | | evalrouter traces connections list | List trace connections. | | evalrouter traces list | List recent complete traces. Pages with --limit, --all and --cursor. | | evalrouter workspaces create | Create a workspace. | | evalrouter workspaces delete WORKSPACE_ID | Delete a workspace (confirm with its name). | | evalrouter workspaces get WORKSPACE_ID | Show a workspace. | | evalrouter workspaces list | List workspaces you belong to. | | evalrouter workspaces update WORKSPACE_ID | Rename a workspace. | 1b:Tdfe, Results and exports Check whether a run finished and how much work it covered before you compare scores: sh evalrouter runs get RUN_ID --json evalrouter runs results RUN_ID --json The result records the benchmark version, completed and missing work, scores, errors, and billing state. A sample is not a full benchmark result. Mark partial and failed runs as such in comparisons. In a terminal, evalrouter runs results RUN lists failed tasks under Why tasks failed. It uses each sample's error code from GET /v1/runs/{run_id}/samples?status=error. A task without a code is unclassified. The terminal uses plain metric names such as Accuracy for acc. JSON and exports keep the original keys and error codes. Use the failed-samples endpoint or an export to inspect individual tasks. If a benchmark offers output formats, selection.output_format and each sample record the chosen format: prompt_json, native_json, or json_schema. Compare only matching formats. Otherwise, comparisons return output_format_differs with no delta, and leaderboard protocol hashes differ. In the browser, open Evaluations → Group runs. The list includes models started together and groups created from existing runs. Select a group, then use Runs to inspect its members or Comparison to read their results together. Choose Create group from existing runs to select two to four finished results, with one baseline and up to three candidates. The group saves frozen result references, including the version, checksum and model index where applicable. It never reruns a model. Pass/fail checks are optional; without them, the comparison makes no pass/fail decision. Incompatible pairs explain their differences and keep scores available individually, without direct score deltas or release conclusions. To start new work, New evaluation opens Run a benchmark for one to eight managed models. One model starts an ordinary run; two to eight start a group on the same benchmark selection. A started-together group's Comparison tab reads all its members' latest available results without running inference or saving another comparison. It reports when results are pending or unavailable; resumed-group comparison is currently unavailable in the browser. Save a versioned report Use the result's version when you need a repeatable export: sh evalrouter runs export RUN_ID --version RESULT_VERSION --file-format json --output result.json evalrouter runs export RUN_ID --version RESULT_VERSION --file-format csv --output result.csv evalrouter runs export RUN_ID --version RESULT_VERSION --file-format html --output result.html Use the integer version returned by the result. Exports need a destination and refuse to overwrite a file by default. Keep the version and checksum with any downstream report. Your workspace permissions and the evaluation's rights still govern what you can export. HTML single-run bundles and CLI comparisons show All benchmark metrics, with each recorded value and its available uncertainty. Hover a metric name for its definition and exact key, or expand the glossary. Weighted variants stay separate. SOB source scores appear only when the export includes them; source coverage alone is not a score. Missing values are never inferred from pooled averages. Upgrading the CLI does not change an existing archive: generate a new report, and request a new API export when the saved data lacks the source metrics. If a download fails, retry the export for the same run and version. For a run still in progress, use evalrouter runs wait RUN_ID --wait-timeout 3600 --json. 1c:T864, Availability and limits Run a maintained benchmark from the catalog. Where private imports are enabled, you can also make a benchmark from a file or Hugging Face dataset. Private imports use EvalRouter's built-in graders. Maintained benchmarks use their version's recorded grader. Benchmarks and models A ready benchmark may still reject your model. Some tasks need continuation likelihoods that a chat-only model cannot provide. Reasoning settings, output format, and context limit must also fit. EvalRouter checks compatibility again before a run starts. Check [the catalog](/catalog) or run evalrouter benchmarks list and evalrouter models list. If imports are enabled, [import your data](/catalog/import) after reading the [formats and limits](/developers/docs/byob). Validation checks the data and settings. A small sample run checks the chosen model. Credits and execution Runs need workspace credit. Each run starts under a spending cap: the --max-cost you set, or your available balance when you set none (see [credits and billing](/developers/docs/quotes-budgets)). New accounts receive a one-time $5 welcome credit in their workspace. A model connection may also incur separate provider charges. Results separate wrong answers from system errors and unfinished work. A terminal timeout does not stop the server. Keep the run ID so you can check status without starting another run. Current self-service limits Private imports accept JSON Lines, CSV, and Parquet with a built-in grader. They cannot run uploaded code, custom graders, or arbitrary evaluation repositories. [Public catalog additions](/developers/docs/bring-your-own-benchmarkrequest-an-addition) need separate review. Workload imports and Tune Hosted source connections, reviewed workload graders, and Tune are enabled separately from catalog evaluations. A visible interface or SDK method does not establish deployment availability. The API refuses disabled operations. Production activation requires an exact-source qualification bundle and independent approval. See [workload imports](/developers/docs/workload-imports) and [Tune](/developers/docs/tune). 1d:T26c2, Bring your own benchmark You can run your own data as a private benchmark or ask us to add a maintained benchmark to the public catalog. The choice depends on whose grading rules you need. Options and status | What you want to do | Availability | Start here | | --- | --- | --- | | Run a maintained benchmark | Available | Choose a compatible model in the [catalog](/catalog), or run evalrouter benchmarks list. | | Upload your own dataset | Available | [Upload a file](/catalog/import?source=file): JSON Lines, CSV or Parquet. Define prompts and scoring; no manifest is needed in the browser. | | Import a Hugging Face dataset | Available for supported files and licenses | [Import from Hugging Face](/catalog/import). Map the columns and choose a built-in grader. [Requirements](/developers/docs/huggingface). | | Import a gated or private Hugging Face dataset | Requires authorized source access and a supported license | Provide a Hugging Face token with access to the dataset. The imported benchmark stays private to your workspace. | | Keep a benchmark private to your workspace | Available | All [self-service imports](/developers/docs/byob) are workspace-private. | | Use built-in scoring | Available | Choose exact match, multiple choice, numeric, regular expression or JSON fields. See [graders](built-in-graders). | | Add a public benchmark to the maintained catalog | Reviewed by request | [Send a catalog request](request-an-addition) with the source, evaluation definition and license. | | Import a GitHub data file | Enabled workspaces only | [Import a CSV, JSONL or Parquet file](/developers/docs/byobfrom-a-github-data-file), up to 16 MiB, from a public repository or a selected private GitHub App repository. | | Import directly from another HTTPS URL | Catalog request only | Include the source URL in your request. For a supported data file you already have, use file upload. | | Run custom grader code or an arbitrary evaluation repository | Not available through self-service import | Describe the required grader in a catalog request. Uploads do not execute code. | | Supply data for one run without retaining a benchmark | Not available | Create a private benchmark, then archive it to stop future runs. Archiving is not immediate deletion. | A catalog addition is visible to every workspace. A private import stays in yours and does not change your data's license. What a public catalog addition needs - Public data pinned exactly. The data can be in one of these places: - a Hugging Face Hub dataset, identified by its full 40-character commit hash; - a GitHub repository, identified by its full 40-character commit hash. Files stored with Git LFS are fine; - a GitHub release asset, named by the release tag and the file name. The tag only finds the file; its sha256 is what we check; - a public HTTPS URL for a single file. It must not need a login or a signed link, and it cannot point to a private or internal network address. For Hugging Face and GitHub repositories, branch names, tags, main and latest are not accepted, because their contents can change. If you only know a tag or branch, send it and we will pin the commit it points to at review time. Whatever the host, we download each file once and keep our own copy. If the host changes the file later, the catalog version keeps the copy we checked. - Every file listed with its sha256 and size. That includes companion files the task loader needs, such as other splits. We refuse a file whose bytes do not match. - An existing task. The benchmark must match a task in [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) or [inspect_evals](https://github.com/UKGovernmentBEIS/inspect_evals) at the revision EvalRouter's runners pin. We run the author's task code. If that code must change to use your data, the benchmark cannot be added yet. For data on GitHub or at an HTTPS URL, we give the framework's file loader our verified copy of your files. It supports Parquet, JSON Lines, JSON, or CSV; Inspect's Hugging Face reader supports JSON Lines, JSON, or CSV only. Prompts, answer extraction, and metrics stay as the author wrote them. The new benchmark version records the changed data location and every file's sha256. This path works for one lm-evaluation-harness task that reads one plain Hugging Face dataset, or an Inspect task already in our catalog that uses Inspect's Hugging Face reader. Name each file for its split, such as train.jsonl or test-00000-of-00001.parquet; the main file is the scored split. We decline task groups, loading scripts, custom dataset code, row shuffling on read, and tasks that require their exact Hub dataset. Those datasets need to be on the Hub. - The author's grader and metrics. Scoring uses the task's native metric at that same pinned revision. You cannot supply your own scoring code. - An allowed data license. One of MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, CC-BY-4.0, CC-BY-SA-4.0 or CC0-1.0. The license the Hugging Face Hub or GitHub reports at the pinned commit or release must agree with it. A plain HTTPS URL reports no license, so send a link to where the license is granted; we review it and keep a copy of the notice. Anything else needs a separate license review and may be declined. Example manifest We record each catalog benchmark in a manifest. This is a trimmed version of the one for COPA: yaml slug: copa name: "COPA" category: Commonsense description: All 100 SuperGLUE validation questions, scored with native causal-continuation likelihoods. source: uri: hf://aps/super_glue@3de24cf8022e94f4ee4b9d55a6f539891524d646/copa/validation-00000-of-00001.parquet sha256: 247a7f586c5cc706eb18a15ad20f1a5b543b9c7af0df4161df138a25edc95d67 bytes: 11975 adapter: runner: lm-eval entrypoint: super_glue/copa/default.yaml split: validation capability: continuation_loglikelihood grader: uri: github://EleutherAI/lm-evaluation-harness@b954108c9baaaa934b4ad842033b31a97ee30816/lm_eval/tasks/super_glue/copa/default.yaml kind: author-native metrics: [acc] sampling: generation: greedy max_output_tokens: 0 attempts_per_task: 1 license: data: BSD-2-Clause code: MIT url: https://asgordon.github.io/copa.html qualification: parity: {mode: full, rows: 100} failure_checks: [invalid_output, timeout, unsupported_endpoint, grader_error] | Block | Meaning | | --- | --- | | source | The data file, pinned to a commit (or a GitHub release asset or HTTPS URL), with its sha256 and size. | | adapter | The runner (lm-eval or Inspect), the existing task, the split that is scored and the kind of model call it needs. | | grader | The task author's grader, pinned to a commit, and the metrics it reports. | | sampling | How the model is called: generation mode, output token limit and attempts per task. | | license | The data and code licenses and where the grant is published. | | qualification | How much of the benchmark must reproduce the native task's results, and which failure cases are exercised. | You do not write a version number. We assign one. If any of the data, task, grader, sampling or license changes, the benchmark gets a new version and the previous version keeps working, so earlier results stay comparable to what they measured. Built-in graders Self-service [private benchmarks](/developers/docs/byob) use the five graders below. Maintained catalog benchmarks can use specialized graders from their evaluation definitions, so this list is not a limit on every benchmark in EvalRouter. Use built-in graders for a JSON Lines, CSV, or Parquet file with one question per row. Map the question, choices, answer, and optional row ID. In the browser, write a prompt with {question} and, for multiple choice, {choices}. CLI manifests can map more fields. Validation refuses missing or mistyped columns, empty rows, and duplicate IDs. | Grader | What counts as correct | | --- | --- | | multiple_choice | The model's letter (or the exact text of one choice) matches the answer, given as a letter, an index or the choice text. | | exact_match | The model's answer equals a reference answer after the normalization you declare: letter case, whitespace and punctuation. | | regex_match | Your declared pattern matches the model's answer, anchored to the whole answer by default, and the captured text equals the reference. | | numeric | The number extracted from the answer equals the reference, within the absolute or relative tolerance you declare. | | json_field_match | The answer parses as JSON and each field you name equals the reference. | Each row scores 1 or 0. Results show accuracy and standard error. The label names EvalRouter's built-in grader, not an author's grader. Compare only runs with the same grader, mapping, and prompt. What happens after you ask 1. We review the request: pins, license and whether an existing task expresses it. 2. We fetch the pinned files and refuse any whose size or sha256 does not match. 3. We store the verified files. Runs read those stored copies, not the live source. 4. We qualify it: our run must reproduce the native task's results on the stated rows, and invalid output, timeouts, unsupported model endpoints and grader errors must be recorded as failures rather than scores. 5. Once qualified and published, it appears as a versioned benchmark in the catalog. Only compatible models can run it. A request can be declined at any step. We tell you which requirement it did not meet. Request an addition Email [info@kimpton.ai](mailto:info@kimpton.ai) with: - where the data is (Hugging Face dataset, GitHub repository or release, or HTTPS URL) and the commit, tag or URL you want; - the lm-eval or Inspect task it matches, and the split to score; - the data license and a link to where it is granted. Once it is listed, run it like any other benchmark from the [CLI quickstart](/developers/docs/quickstart). 1e:T3f72, Benchmarks from Hugging Face datasets Use a Hugging Face dataset as benchmark data. EvalRouter resolves the dataset to an exact commit, checks the files and license, and stores a copy. Runs read that copy, so later Hub changes cannot alter an earlier benchmark version. "Public" and "private" below describe who can read the dataset on Hugging Face. They do not change who can see the benchmark: a benchmark you import is always private to your workspace. Status | Path | Status | | --- | --- | | Catalog benchmark from a Hugging Face dataset | By request, reviewed. See [Bring your own benchmark](/developers/docs/bring-your-own-benchmark). | | Import a public dataset into your workspace | Available through [the browser](/catalog/import) or evalrouter benchmarks imports huggingface hf://.... Supported files and licenses are checked before creation. | | Import a gated or private dataset | Needs a Hugging Face token that can read it and a supported license. The benchmark stays private to your workspace. | An import does not reproduce the dataset author's benchmark by itself. You choose prompts, reference answers, and a built-in grader. For the author's established protocol, use a maintained catalog benchmark. Naming a dataset | Form | Example | Where | | --- | --- | --- | | Dataset page URL | https://huggingface.co/datasets/cais/mmlu | Browser | | Dataset ID | cais/mmlu | Browser | | Pinned reference | hf://cais/mmlu@main or hf://cais/mmlu@<40-character commit> | Browser, CLI and API | Only a dataset page is accepted. A model page, a link to one file, a dataset viewer link or a URL with a query or fragment is refused before anything is fetched. A pinned reference names a revision: a full commit, a tag or a branch. A URL or ID has no revision, so the browser uses the dataset's main branch. The CLI and the API always need @. Whatever you name, EvalRouter resolves it to one commit when it fetches the data, and stored files and receipts name only that commit. Importing a moved branch or tag later creates a new source instead of changing the old one. You then choose which file to use: - a config (when the dataset has several) and a split, for example --config abstract_algebra --split test. We use the files the dataset card assigns to that split, or the Hub's usual layout (/-00000-of-00001.parquet, data/-...); - or explicit file paths in the repository. A matches within one folder. Only Parquet, JSON Lines and CSV files are accepted. Nothing else in the repository is downloaded, and nothing from it is executed: no dataset loading script, no custom code and no grader from the repository. Import in the browser Open [Import a benchmark](/catalog/import) and choose Hugging Face. Search by dataset or publisher name, or enter a dataset ID or page URL in the same input. Select the public dataset to pin its commit and browse its folders. Choose one .jsonl, .csv or .parquet file; Next and Previous move through large folders, and the breadcrumbs return to parent folders. The filter searches the current page. Typing a folder path ending in / opens that folder. Use Preview selected file to inspect CSV or JSONL columns and sample rows at the displayed commit. For Parquet, enter column names in Scoring and review the server's sample during validation. Source and access options contains optional config, split and exact-file inputs, plus a read token for private or gated data. Public discovery does not use that token; private or gated sources use the manual source and file path. Map the prompt and answer columns, choose a grader, then click Import and validate. Review the returned rows, shown exactly as the model receives them, and which columns are never sent, before Create benchmark. The browser needs no EvalRouter API key or manifest. Public browsing and samples help choose a file; dataset access, file integrity, license and every row are checked by the server when you validate (see [Two checks](two-checks-source-then-rows)). Example: 100 MMLU questions Use this public dataset to try a small multiple-choice benchmark: | Setting | Value | | --- | --- | | Dataset | https://huggingface.co/datasets/cais/mmlu, cais/mmlu, or hf://cais/mmlu@main | | Config and split | abstract_algebra, test | | File | abstract_algebra/test-00000-of-00001.parquet | | Columns | prompt question, choices choices, answer answer | | Grader | multiple_choice, answers are zero-based indexes | For the exact data used in this example, enter hf://cais/mmlu@c30699e8356da336a370243923dbaf21066bb9fe. This split contains 100 rows. EvalRouter's built-in multiple-choice grader scores the answers; this setup does not reproduce the original MMLU evaluation protocol. For an established protocol, choose a maintained catalog benchmark that matches the evaluation you need. Gated and private datasets A gated or private dataset needs a Hugging Face token that can read it. For a gated dataset, first accept its terms on huggingface.co with the account that owns the token. Put your own data in a private dataset 1. On huggingface.co, create a new dataset repository (New Dataset) under your account or organization and set its visibility to Private. 2. Upload one .csv, .jsonl or .parquet file, with one row per question and a column for the reference answer: Files and versions → Add file → Upload files, or hf upload / ./test.jsonl test.jsonl --repo-type dataset. 3. Add a license to the dataset card (README.md), in its YAML header, for example: yaml --- license: cc-by-4.0 --- It must be one of the [accepted licenses](licenses). A private dataset needs one too. Then import it as https://huggingface.co/datasets//, / or hf:///@main, and name the uploaded file's path (for example test.jsonl). Create a read token for that repository Use the smallest token that works. In your Hugging Face Settings → Access Tokens, create a fine-grained token. Under repository permissions, select only this dataset repository and give it read access to its contents. It needs no write permission, and no access to other repositories, models, Spaces or your organization settings. Revoke it on Hugging Face when you no longer need it. Use the token - Browser: paste the token when asked. It is sent with that import only. It is not saved in your draft, so enter it again if you reload the page. - CLI: pass it without putting it on the command line: bash evalrouter benchmarks imports huggingface hf:///@main --split test --hf-token-env HF_TOKEN printf '%s\n' "$HF_TOKEN" | evalrouter benchmarks imports huggingface hf:///@main --split test --hf-token-stdin EvalRouter uses the token only to read that dataset from huggingface.co, for that one import. We never store it, and it never appears in the receipt, the benchmark, our logs or an error message. Without a token, a gated dataset is refused with token_required (403). A private dataset is invisible to the Hub without one, so it is refused as not found, access_refused (404), exactly like a dataset that does not exist. A token that cannot read the dataset, for example because its terms were not accepted, also gets access_refused. Licenses The dataset card must report a license at the resolved commit, and every license it reports must be one of MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, CC-BY-4.0, CC-BY-SA-4.0 or CC0-1.0. A card that reports no license, other, unknown, or a non-commercial or no-derivatives license is refused with license_refused, and the message lists what is accepted. This applies to private datasets too: a token grants access, not a license. The catalog also accepts a few other permissive licenses, and it can accept a card that reports other only after a separate license review for those exact files. What can be evaluated An imported file becomes a benchmark only if its rows fit a [built-in grader](/developers/docs/bring-your-own-benchmarkbuilt-in-graders): one prompt from text columns, one reference answer, one model reply per row, scored by exact_match, multiple_choice, numeric, regex_match or json_field_match. A generic import keeps the data, not the author's evaluation. It does not preserve a reinforcement-learning or agent environment, the dataset's original or custom graders, reward functions or verifiers, a loading script, or any other code in the repository. Every imported benchmark is scored by one of the five built-in graders. Mapped prompt columns must hold text or numbers in every row. A column holding a list of chat messages or a nested object is refused, and a dotted path such as reward_model.ground_truth does not reach inside one. Images, audio, multi-turn conversations, tool use, agent environments, model-graded scoring and a dataset's own reward functions or verifiers are not supported by self-service import. Datasets are judged by their rows and the evaluation they need, never by their name or a tag. Some reinforcement-learning datasets store text questions with plain reference answers and import as usual. Others are built for agent environments: their rows hold chat-message prompts, a reward-model block and environment metadata, and they are scored by the author's custom verifiers. Such a file can pass the source import, but its rows are refused at validation, for example row 1 column 'prompt' must be text or a number, not list. Flattening it into a question and an answer would not reproduce the author's evaluation, so it cannot be run as that benchmark here. Describe it in a [catalog request](/developers/docs/bring-your-own-benchmark) instead. Two checks: source, then rows An import is not yet a runnable benchmark. It passes two separate checks. 1. Source import. Resolve the revision, select the files, check access and the license, then download and verify every file. These refusals happen here, before any row is read: | Code | Status | Meaning | | --- | --- | --- | | invalid_source, invalid_selection | 422 | The dataset reference, config, split or file path is malformed. | | revision_unresolved | 422 | The branch, tag or commit does not exist. | | config_required, unknown_config, split_required, unknown_split, incomplete_split | 422 | The config or split is missing, unknown, or its files are incomplete. | | file_not_found, unsupported_format | 422 | The named file does not exist, is not Parquet, JSON Lines or CSV, or the selection mixes formats. | | file_refused | 422 | The Hub lists no size or hash for a selected file, so it cannot be verified. | | too_many_files, import_too_many_files | 422 | The selection has more files than one import or your workspace allows. | | license_refused | 422 | The card's license is missing or not accepted. | | token_required | 403 | A gated dataset (or one of its files) needs a token and none was given. | | access_refused | 404 | The dataset does not exist, is private and no token was given, the token cannot read it, or it is disabled on the Hub. | | too_large | 413 | The selection exceeds your workspace's upload limits. | | hash_mismatch, hub_unavailable | 502 | A file did not match the Hub's hash, or huggingface.co could not be reached. Retry later. | 2. Row validation. EvalRouter checks every row, not just the preview, against your column mapping and grader. With a mapping, evalrouter benchmarks imports huggingface refuses a problem as invalid_mapping. evalrouter benchmarks private create --dry-run, POST /v1/private-benchmarks with "dry_run": true and the browser's Import and validate refuse it as invalid_private_benchmark. Both are 422 and name the row and the reason. Examples: a missing column, an empty reference, a duplicate ID, a numeric answer that is not a number, a choice list that is not a list, or a prompt column holding a list. A file with more rows than allowed is refused with import_row_limit or upload_row_limit, and a file over 64 MiB with private_benchmark_too_large. Nothing is created and no model is called until both checks pass and you create the benchmark. Once created, it runs like any benchmark, using available workspace credit. After you create it Create benchmark adds the benchmark to your workspace's private catalog. It appears in [the catalog](/catalog) and in the benchmark list for new evaluations, marked as belonging to your workspace, and only members of this workspace can see or run it. It is never published, listed for other workspaces or shared with them, even when the Hugging Face dataset is public. The page shows the benchmark ID and row count. Choose Configure an evaluation to run it on a compatible chat model. The benchmark version never changes: it keeps the pinned commit, file hash, column mapping and grader. To change any of them, create a new benchmark. Results are labeled as scored by an EvalRouter built-in grader and are comparable only to other runs of the same benchmark ID. Choose Source → Created by me in the catalog to find your imports in the current workspace. Same-name imports remain separate benchmarks. Delete asks you to confirm the exact benchmark ID, then archives it and frees its benchmark slot while keeping earlier runs and results. See [managing private benchmarks](/developers/docs/byobcreate-and-run-it). In the catalog A catalog manifest can name the dataset directly: yaml source: uri: hf://cais/mmlu@c30699e8356da336a370243923dbaf21066bb9fe config: abstract_algebra split: test files: - path: abstract_algebra/test-00000-of-00001.parquet sha256: 92338cdcdb43d2871691ef9a0e125e633ca57335ed635a635fe6dead94f224c2 bytes: 9964 The manifest must pin the full commit and list every file of the split with its sha256 and size, so the review sees exactly what will run. Before anything is downloaded, we check the list against the Hub at that commit: each file must match, and a file of that split that the manifest leaves out is refused. The rest of the manifest (task, grader, license, qualification) is the same as for any catalog benchmark. Into your workspace with the CLI bash evalrouter benchmarks imports huggingface hf:///@main --config --split test \ --prompt-field question --answer-field answer --grader exact_match The import runs on our side: the CLI uploads nothing. It prints the commit the revision resolved to, the license and each stored file's sha256. With --prompt-field, --answer-field and --grader (and --choices-field for multiple_choice), we also run the row validation above. Then name the stored file in a manifest to create the benchmark; see [From a Hugging Face dataset](/developers/docs/byobfrom-a-hugging-face-dataset). Results are labeled as scored by an EvalRouter built-in grader, not the benchmark author's. Imported data is private to your workspace and is never shared with, or deduplicated against, another workspace. What we check and keep 1. The commit. The Hub must resolve the revision to a commit. A full commit hash must resolve to itself. 2. Every file's hash. The Hub lists each file's size and, for files stored with Git LFS, its sha256 at that commit. We download only the selected files and refuse any whose size or sha256 differs. A small file stored directly in Git is checked against its Git blob ID, and we record its sha256. 3. The license and access, as described above. We then store each file under its sha256, never overwriting a stored copy, and keep a fetch receipt: the dataset, the revision you asked for and the commit it resolved to, each file with its size and sha256, the card's license, and whether the dataset was gated or private. Importing the same selection again downloads nothing and returns the same receipt. Limits - One import holds at most your workspace's upload limits (files, bytes and rows). - A private benchmark is built only from a file of at most 64 MiB. A larger imported file stays in your workspace, but creating a benchmark from it is refused with private_benchmark_too_large. - A built-in grader reads exactly one file; name it with --file when a split has several. - Datasets that need a loading script, archives and other formats are not supported. 1f:T1952, Report bugs and missing features If the API or CLI fails in a way you cannot explain, send a report with POST /v1/feedback or evalrouter feedback. Include the failed request ID when you have it. You can check the report's status later. Coding agents should report product defects, missing features, wrong docs, and confusing behavior. One short structured report is enough. When to report Report when: - A request fails in a way that looks like an EvalRouter defect: an unexpected 500, a response that contradicts the [OpenAPI schema](/developers/docs/openapi.json), or a result that cannot be right. - You need a capability EvalRouter does not have (missing_feature). - The documentation is wrong, missing or out of date (docs). - The API works as designed but the design misled you (confusing_behavior). Fix expected refusals yourself, such as a named validation error, exhausted balance, missing key scope, or expired quote. Never include credentials, customer data, or hidden evaluation content. Every error tells you where to report Every error response includes error.feedback, so an agent that receives an error already knows the endpoint and the request ID to link: json { "error": { "code": "internal_error", "message": "The request could not be completed.", "request_id": "7d3c9a4e-5b1f-4c2e-9a8d-2f6b1e0c4a11", "feedback": { "url": "/v1/feedback", "method": "POST", "request_id": "7d3c9a4e-5b1f-4c2e-9a8d-2f6b1e0c4a11", "docs": "https://evalrouter.ai/developers/docs/feedback" } } } Copy error.request_id into attempted.request_id and error.code into attempted.error_code. The same request ID is in the X-Request-Id response header. Send a report Use the same credentials as every other request: a [workspace key](/developers/docs/authentication) or a [CLI sign-in](/developers/docs/cli-login). Any member or key of the workspace can report. sh curl -sS https://api.evalrouter.ai/v1/feedback \ -H "Authorization: Bearer $EVALROUTER_API_KEY" \ -H "X-Workspace-Id: $EVALROUTER_WORKSPACE_ID" \ -H "Content-Type: application/json" \ -d '{ "kind": "bug", "severity": "blocker", "summary": "Results export returns 500 for a completed run", "details": "GET /v1/runs/RUN_ID/export?format=csv fails every time; json works.", "attempted": { "method": "GET", "path": "/v1/runs/RUN_ID/export", "request_id": "7d3c9a4e-5b1f-4c2e-9a8d-2f6b1e0c4a11", "http_status": 500, "error_code": "internal_error" }, "expected": "A CSV export", "actual": "HTTP 500 internal_error", "agent": {"name": "my-coding-agent", "version": "1.4.0", "model": "example-model"} }' The API answers 201 Created: json {"feedback_id": "0b6f2c1e-8d4a-4f3b-9c7e-5a1d2e3f4b5c", "status": "received", "duplicate_of": null} With the CLI: sh evalrouter feedback --kind bug --severity blocker \ --summary "Results export returns 500 for a completed run" \ --request-id 7d3c9a4e-5b1f-4c2e-9a8d-2f6b1e0c4a11 --error-code internal_error Longer details from a file or another command: evalrouter feedback --kind docs --severity nice_to_have \ --summary "Quote docs do not say caps are decimal strings" --details - < notes.txt Add --json for one machine-readable result. Other options: --http-status, --method, --path, --agent-name, --agent-version, --agent-model and --contact-ok. The CLI names itself as the client automatically. Fields | Field | Required | Description | | --- | --- | --- | | kind | yes | bug, missing_feature, docs, confusing_behavior or other. | | severity | yes | blocker (you cannot continue), degraded (a workaround costs time or quality) or nice_to_have. | | summary | yes | One line, 1 to 200 characters. | | details | no | Steps, context and what you already tried; up to 5,000 characters. | | attempted | no | The failed request: method, path, request_id, http_status, error_code. The path is stored as its route template, for example /v1/runs/{run_id}/export, without query strings. | | expected, actual | no | Up to 2,000 characters each. | | agent | no | name, version, model of the reporting agent (identifiers, not prose). | | client | no | name, version. Filled from the User-Agent header when omitted. | | contact_ok | no | Whether EvalRouter may contact the workspace owner about the report. Default false. | Unknown fields are rejected with 422 validation_error. Check the outcome sh curl -sS https://api.evalrouter.ai/v1/feedback/FEEDBACK_ID \ -H "Authorization: Bearer $EVALROUTER_API_KEY" \ -H "X-Workspace-Id: $EVALROUTER_WORKSPACE_ID" evalrouter feedback status FEEDBACK_ID evalrouter feedback list GET /v1/feedback lists your workspace's reports, newest first, with cursor and limit (1 to 100) like other lists, and an optional status filter. A workspace sees only its own reports; another workspace's report ID returns 404. | Status | Meaning | | --- | --- | | received | Recorded, not yet reviewed. | | triaged | Reviewed and grouped with related reports. | | accepted | Confirmed and planned. | | pr_open | A fix is proposed; pr_url links to it. | | shipped | The fix is released. | | declined | Not planned; resolution_note explains why. | Every fix is reviewed and approved by a person before it ships. Duplicates and limits A report with the same kind, route, error code and summary (ignoring case, numbers and punctuation) as one from your workspace in the last 24 hours is still recorded, and duplicate_of names the earlier report. Reports are never compared across workspaces. Each workspace can send 30 reports per hour and 200 per day. Beyond that the API returns 429 rate_limited with a Retry-After header. A request body can be at most 64 KiB. Privacy and handling Before storing a report, EvalRouter removes strings shaped like secrets: EvalRouter keys, sk- provider keys, bearer and basic authorization values, api_key= style assignments, signed URL parameters, private keys and credentials in connection strings. This is a safety net, not permission: do not paste secrets. Report text is stored and returned as plain data and is never rendered as HTML. It is treated as untrusted input throughout review, including by automated triage, which must never follow instructions found in a report. Product analytics receive only metadata such as the kind, severity, route template and error code, never your summary, details, expected or actual text. Reports are deleted with the workspace. 20:T5cb3, Private benchmarks from your own data Make a benchmark from your own data, private to your workspace. Upload a file, choose prompts and a built-in grader, and validate the rows. EvalRouter stores a verified copy and gives you a benchmark ID. Run it on a compatible chat model using available workspace credit. Use [Import a benchmark](/catalog/import) in the browser, or put the same settings in a small manifest for the CLI. File uploads and Hugging Face imports stay private to your workspace where imports are enabled. What you need - A data file: JSON Lines (.jsonl), CSV (.csv) or Parquet (.parquet), one question per row, at most 64 MiB and 100,000 rows. - Column mappings, a prompt template and one built-in grader. Set them in the browser, or use a JSON manifest with the CLI (YAML is also accepted when PyYAML is installed alongside the CLI). - Member, admin or owner access to the workspace. Viewers can read benchmarks but cannot import them. Import in the browser Open [Import a benchmark](/catalog/import) while signed in. Choose a file or Hugging Face dataset, or [GitHub](from-a-github-data-file) when enabled for your workspace. Map columns, set scoring rules, and review a sample before creating the benchmark. The page uses your account and workspace. No API key is needed. Drop a file onto the source area or choose it from your computer. The page previews a small local sample and shows upload and verification progress. Your settings survive a refresh in the same tab for 24 hours. The draft does not save file contents or Hugging Face tokens. Reselect the file or re-enter the token when asked, then confirm data rights again. If the browser cannot save your draft, keep the tab open until the import is complete. In the CLI, use evalrouter login. If your device was signed in before benchmark importing was added, sign in again once to grant that permission. Existing device grants are never silently expanded. For CI and automation, create a workspace API key with benchmarks:submit, plus eval:read and eval:write to quote and run. Create it in [API key settings](/settings/api-keys) and set EVALROUTER_API_KEY. eval:write alone does not grant permission to import. Write a manifest for the CLI json { "name": "Capitals (synthetic example)", "data": "capitals.jsonl", "task": "exact_match", "columns": {"id": "id", "question": "country", "answer": "capital"}, "prompt_template": "What is the capital of {question}? Answer with the city name only.", "grader_options": {"extract": "last_line"}, "max_output_tokens": 16, "data_license": "LicenseRef-Workspace-Private", "rights_attestation": true } | Key | Meaning | | --- | --- | | name, description | What people in your workspace see. | | data or source | data is a local file, relative to the manifest; the CLI uploads it first. source is an existing upload, workspace:///, from evalrouter benchmarks uploads create, or a Hugging Face file your workspace imported, hf:///@/ (see [From a Hugging Face dataset](from-a-hugging-face-dataset)). | | source_sha256, source_bytes | Required with an hf:// source: the file's sha256 and size from the import receipt. | | format | Optional: jsonl, csv or parquet. Inferred from the file name. | | task | The grader: multiple_choice, exact_match, regex_match, numeric or json_field_match. | | columns | Maps roles to your column names. answer is required. id (a unique row ID) and group (a sampling stratum) are optional. choices is required for multiple_choice only. Every other role is a prompt placeholder. | | prompt_template | The user message, with {role} placeholders. Every mapped prompt column must appear; id, answer and group never can. {choices} renders lettered lines. {{ and }} are literal braces. | | system_prompt | Optional system message. | | grader_options | Options for the chosen grader (below). Omitted options take their defaults, which are recorded in the benchmark. | | max_output_tokens | Output limit per answer, 1 to 4096 (default 256). | | split | Optional label for the scored rows (default test). | | data_license | A license identifier for your data, recorded with results (default LicenseRef-Workspace-Private). | | rights_attestation | Must be true. See [Rights and terms](rights-and-terms). | How we interpret your dataset This is the whole contract between your file and the model. The dry run, the created benchmark (delivery in the API response) and every quote for it state the same thing for your benchmark. What the model receives, per row: one or two messages, nothing else. 1. Your system_prompt, verbatim, as a system message, if you set one. 2. Your prompt_template as the user message, with each {role} replaced by that row's value from the column you mapped to the role, and {choices} (multiple choice only) replaced by lettered lines A. text, B. text, and so on. EvalRouter adds no instructions, no answer-format hint, no examples and no other text. A mapped prompt value must be text or a number; a list or object is refused. What the model never receives. - answer: read to grade the row, never delivered. The template cannot contain {answer}. - id and group: used to identify and sample rows, never delivered. - Every column you did not map: not read at all and never delivered. It stays only in the stored copy of your file. Answer guard. A mapped prompt column is delivered, whatever its name. If a row's delivered messages contain that row's own answer, the benchmark is refused with byob_answer_in_input, naming the column (for example a solution or explanation column mapped as {hint}). The comparison uses Unicode NFKC, case folding and collapsed whitespace, and covers answers of at least 20 characters after that. Shorter answers, such as numbers, letters or "yes", occur in prompts by chance and are not checked, so make sure no mapped column carries them. A multiple-choice answer is a letter and is never checked: delivering the correct choice's text is the format. How each row is run. No few-shot examples (0 shots). One attempt per row, greedy, at temperature 0, with no automatic retry. Each answer is cut at max_output_tokens. No tools are offered. A rendered user message over 100,000 characters is refused at validation. How each row is scored. One [grader](graders) decides 1 or 0 from the model's text output alone. Its options, including the defaults you did not set, are recorded with the benchmark and cannot change. Graders Self-service uploads use the five graders below. They support datasets with reference answers. They never execute uploaded Python code or arbitrary lm-eval or Inspect repositories. Maintained catalog benchmarks can use their own specialized grading implementations. Every grader is built into EvalRouter, deterministic and scores each row 1 or 0. Results report accuracy with its standard error and are labeled as scored by an EvalRouter built-in grader, not by any benchmark author's own grader. They are comparable only to other runs of the same benchmark ID. | Grader | Options (defaults) | How it extracts and matches | | --- | --- | --- | | multiple_choice | answer_format: letter, index or text (letter); index_base: 0 or 1 (0) | In order: the whole output is one letter, optionally in parentheses and followed by ., ) or : (any case); otherwise the last answer, optional is, optional : or -, then an uppercase letter (answer: B, The answer is (C); answer is b is not read); otherwise the one choice whose normalized text equals the whole output. Correct when that letter is the reference. A letter outside the row's choices scores 0. | | exact_match | extract: full, last_line or answer_prefix (full); prefix (ANSWER:); normalize | full takes the whole output, last_line the last non-blank line, answer_prefix the text after the last prefix (any case) up to the end of that line. Correct when it equals a reference, both normalized. answer may be a list of accepted answers (in CSV, a JSON list of strings). | | regex_match | pattern (required, up to 512 characters); anchored (true); ignore_case, dot_all (false); normalize | The pattern must match the whole stripped output (anchored) or anywhere in it. It takes the group named answer, else the only group, else the whole match. Correct when it equals a reference, both normalized. | | numeric | extract: last_number, first_number, full or answer_prefix (last_number); prefix (ANSWER:, for answer_prefix); abs_tol, rel_tol (0) | last_number/first_number take the last or first number in the output. full takes the whole output and answer_prefix the text after the last prefix; both then drop trailing periods and one trailing % and must be a single number. Numbers may have a sign, comma thousands separators, decimals and an exponent. Correct when equal as decimals, or within abs_tol/rel_tol when set. | | json_field_match | fields (required dotted paths); extract: strict or first_object (strict); normalize_strings (false) | strict parses the whole output; first_object the first fenced json block, else the first decodable JSON object or array. Correct when every named field equals the reference's value. Booleans never equal numbers. Strings are compared exactly unless normalize_strings. | normalize defaults to Unicode NFKC, case-folded and whitespace-collapsed comparison. Punctuation is kept unless you set strip_punctuation (punctuation becomes spaces). An unparseable or unmatched answer scores 0 with a recorded reason. Timeouts and provider errors are errors, never scores. Tune rubric grading. The API also accepts "grader_protocol": "tune_rubric_v1" with "task": "exact_match", an evaluator_id (a rubric version in your workspace) and a managed judge_route_id. Delivery to the model is unchanged. Instead of the table above, a separately metered managed judge reads the rubric, the delivered user message, the reference answer and the output, and returns pass or fail; an abstention leaves the row inconclusive. Create and run it 1. Check the manifest against every row without creating anything. The data is uploaded and verified. Nothing is charged: sh evalrouter benchmarks private create manifest.json --dry-run The response shows the row count, [how the dataset is interpreted](how-we-interpret-your-dataset) (delivered and never-delivered columns, grader and match rule, examples, attempts and output limit), the first three rows exactly as the model receives them (system and user messages, untruncated), and any warnings. Answers are not shown, because they are never delivered. A mapping problem is refused with the row and reason, for example a missing column, a duplicate ID, an answer that is not a number, or a mapped column that carries the answer (byob_answer_in_input). 2. Create the benchmark: sh evalrouter benchmarks private create manifest.json It prints the benchmark ID, byob-@1. 3. Run it like any benchmark, by its ID (or through the [quickstart request](/developers/docs/quickstart), replacing BENCHMARK_PROFILE_ID with it): sh evalrouter run byob-@1 --model openai/gpt-4.1-mini-2025-04-14 --full --max-cost 1 Use --dry-run to see the quote without starting. For a private benchmark, the quote's warnings repeat what the model receives (system prompt, template and mapped columns), what it never receives, and the grader's match rule. Read the scores with evalrouter runs results RUN and export them with evalrouter runs export RUN, as described in [results](/developers/docs/results). 4. List your private benchmarks with evalrouter benchmarks private list, and stop new runs of one with evalrouter benchmarks private archive byob-@1. Earlier runs and results are kept. In [the catalog](/catalog), choose Source → Created by me to see the private benchmarks you created in the current workspace. Imports with the same name have separate benchmark IDs. Their creation dates help distinguish them. Members, admins and owners can choose Delete on a private benchmark. Check its ID and creation date in the confirmation, then choose Delete benchmark. This archives only that benchmark, removes it from the active catalog and frees its benchmark slot. Existing runs and results remain available. The uploaded source is not erased and its source slot is not freed. Free an uploaded source slot Every completed upload is an active source until you archive it, whether or not a benchmark uses it, and a workspace has 1,000 slots. Each benchmarks private create with a local data file (including --dry-run) uploads a new source. When they are all in use, a new upload is refused with private_benchmark_limit and "limit_kind": "upload_sources". Archiving a private benchmark does not free a source slot. sh evalrouter benchmarks uploads list --status completed evalrouter benchmarks uploads archive benchmarks uploads list lists each upload's ID, status, size, rows, date and files, and how many of the 1,000 slots are used. benchmarks uploads archive accepts the upload ID or its workspace:/// source. Archiving is permanent: the source can no longer be named in a new manifest. Nothing is deleted: its files stay stored (and still count toward storage), and private benchmarks already created from it, and their runs, keep working. To reuse a source you still need instead of uploading it again, name it in the manifest as "source": "workspace:///" in place of data. A benchmark version never changes. To change the data, template or grader, create a new benchmark. From a Hugging Face dataset Instead of uploading a file, you can build a private benchmark from a Hugging Face dataset your workspace has imported (see [Benchmarks from Hugging Face datasets](/developers/docs/huggingface)). This needs Hugging Face imports to be enabled where you run it. Otherwise an hf:// source is refused. In the browser, enter the dataset page URL (https://huggingface.co/datasets//), its ID (/) or hf:///@. A URL or ID uses the main branch. The CLI and API take hf:///@. That page also explains private datasets and read tokens. 1. Import it. The import pins the revision to a commit, checks each file's hash and the dataset's license, and keeps a copy private to your workspace: sh evalrouter benchmarks imports huggingface hf:///@main --split test --json The response lists each file as hf:///@/ and its sha256 and bytes in the receipt. With a mapping (--grader, --prompt-field, --answer-field), it also returns a declaration. Its default template is {prompt}. For multiple_choice it is {prompt}, a blank line, {choices}, a blank line, then Answer with one letter as "ANSWER: ". Whatever template you put in the manifest is delivered verbatim, including that line if you keep it. 2. Name that file in the manifest instead of data: json { "name": "Imported QA (example)", "source": "hf:///@/data/test-00000-of-00001.jsonl", "source_sha256": "", "source_bytes": 12345, "task": "exact_match", "columns": {"question": "question", "answer": "answer"}, "prompt_template": "{question}\nAnswer briefly.", "rights_attestation": true } 3. Create and run it exactly as above, with evalrouter benchmarks private create manifest.json. Only a full 40-character commit is accepted, and only a file your own workspace imported: another workspace's import, a different sha256 or size, or a tag or branch name is refused. The same size, row, rights and privacy rules apply as for uploads. From a GitHub data file GitHub imports are available only to enabled workspaces. When unavailable, the GitHub source is hidden. A saved draft or direct link keeps your settings and offers file upload or Hugging Face instead. Publishing this guide does not enable access. 1. In [Import a benchmark](/catalog/import?source=github), choose GitHub. Use Public for a repository URL or owner/repository, or Private to select one connected private repository. Owners and admins connect or disconnect accounts in [Settings → Integrations](/settings/integrations). Set repository permissions on GitHub. Connecting an account makes its granted repositories available in the importer. No personal access token is needed. 2. Files load automatically from the default branch. Expand folders in the file tree to select one CSV, JSONL or Parquet file, or filter by typing a path. You can also paste an exact file path. Use the arrow keys to move through files and expand or collapse folders, and Enter or Space to select a file. To use another revision, choose a branch, tag or recent commit from the dropdown. You can also enter a reference. Files can be up to 16 MiB and within your workspace limits. The displayed commit is frozen for the import, even if the branch later moves. 3. Map the columns, choose one of the [built-in graders](graders), confirm your data rights, then validate and create the benchmark. The verified file and benchmark stay private to your workspace, including data from a public repository. This imports data for your chosen prompts and scoring rules. It does not run repository code, install dependencies, or reproduce an author's custom evaluation protocol. JSON arrays, Git LFS pointers, symlinks and submodules are not supported. Upload the underlying supported data file instead. evalrouter benchmarks imports capabilities (or the capability endpoint below) shows whether GitHub imports are available to your workspace. Where they are, evalrouter benchmarks imports github-refs, github-preview and github follow the same steps as the browser. Privacy and isolation - A private benchmark and its upload are visible only inside your workspace. Anyone else asking for its ID or upload gets "not found", never "forbidden", so an ID reveals nothing. - It never appears in the shared catalog, cannot be added to it, and its results cannot be shared through a public link. - Your file is stored once per upload under your workspace's private storage and read back by its SHA-256 before every run. Identical files in two workspaces are never deduplicated or compared. - Rows are sent to the model provider of each run you start, like any benchmark prompt. Rights and terms In the browser, choose the dataset’s existing license. Use Private or proprietary for data governed by your own private terms, a named license when it applies, or Other license to enter its identifier. This records the existing terms. It does not change them or make the benchmark public. The CLI records this choice in data_license. Hugging Face imports preserve the verified source license. By setting rights_attestation: true you confirm: "This workspace has the right to use this data for evaluation and to send its rows to the model providers selected for each run. EvalRouter keeps it private to this workspace and never publishes, shares or reuses it." The statement and your data_license are recorded with the benchmark and shown in every result's notices. Do not upload data you are not allowed to process this way, and do not upload personal data you have no basis to send to model providers. Limits and safety | Limit | Value | | --- | --- | | File size for a private benchmark | 64 MiB | | GitHub source file | 16 MiB | | Rows | 100,000 | | Rendered prompt | 100,000 characters per row | | Private benchmarks created per workspace per day | 10 | | Active private benchmarks per workspace | 1,000 (archive one to free a slot) | | Active uploaded sources per workspace | 1,000 (archive one with evalrouter benchmarks uploads archive) | | Uploads per workspace per day | 10, at most 2 open at a time | | Minimum available balance to upload | $5 | The 64 MiB limit applies to the file a benchmark is built from. An upload or a Hugging Face import can be larger, up to your workspace's upload limit, but creating a private benchmark from a file over 64 MiB is refused with private_benchmark_too_large. Uploads and imported GitHub and Hugging Face files share your workspace's storage allowance. A GitHub import reserves its bytes before storage is written. Retrying the same import after a storage failure reuses that reservation. Archiving a benchmark or completed upload does not free those stored bytes. Uploaded content is treated as untrusted data. Nothing in it is executed: prompts are rendered with plain placeholders and answers are scored by the built-in graders only. Previews and error messages replace credential-shaped strings, such as API keys, tokens and private keys, with [redacted] and warn you. They are still part of your data and are still sent to the model, so remove real credentials from the file. HTTP API | Operation | Purpose | | --- | --- | | GET /v1/benchmark-imports/capabilities | Return { "workspace_id": "", "github": true } or github: false for the authorized workspace, with Cache-Control: no-store. Use the same authentication and workspace headers as importing; clients should hide GitHub actions on false or an unavailable endpoint. | | POST /v1/benchmark-uploads | Create an upload, then send each file to its returned upload URL with PUT (what evalrouter benchmarks uploads create does). | | POST /v1/benchmark-uploads/{id}/complete | Complete the upload and verify the data file. | | GET /v1/benchmark-uploads | List your uploads (status, cursor, limit up to 100; with active_sources and active_source_limit). | | POST /v1/benchmark-uploads/{id}/archive | Archive a completed upload to free a source slot. | | POST /v1/benchmark-imports/huggingface | Import a pinned Hugging Face dataset into your workspace (what evalrouter benchmarks imports huggingface does). | | POST /v1/benchmark-imports/github/refs | When enabled, list the default branch, up to 100 branches, 100 tags and 20 recent commits for a repository. Branches and tags include name and commit_sha; commits include commit_sha. branches_limited and tags_limited indicate lists that reached the limit. Private repositories require both workspace connection/repository identifiers. Responses use Cache-Control: no-store. | | POST /v1/benchmark-imports/github/preview | When enabled, resolve a repository and revision and preview a selected data file. | | POST /v1/benchmark-imports/github | Import the selected file using the preview's full commit_sha as revision. Private repositories also require the workspace's selected github_connection_id and github_repository_id. | | POST /v1/private-benchmarks | Validate ("dry_run": true) or create a private benchmark from source (with source_sha256 and source_bytes for imported hf:// or github:// references) plus the manifest keys above. GitHub sources require current workspace availability and repository access. | | GET /v1/private-benchmarks | List your private benchmarks. | | GET /v1/private-benchmarks/{id} | Read a private benchmark. | | GET /v1/evals?include_private=true&catalog_view=true&private_only=true | Read the workspace catalog; uploaded private benchmark records include created_by (user ID, or null when unavailable) and created_at. | | POST /v1/private-benchmarks/{id}/archive | Stop new runs and free the benchmark slot while keeping earlier runs and results. The browser's Delete action uses this operation. | | POST /v1/runs | Run with "selection": {"profile_ids": ["byob-@1"]}. A spending cap is optional. | Not supported yet - Model-graded (judge) scoring other than [Tune rubric grading](graders), partial credit, few-shot examples and more than one attempt per row. - Custom grader code. 21:T1492, Models, environments, and compatibility For automated release checks, the [CI integration guide](https://github.com/kimpton-ai/kimpton-evalrouter/tree/main/examples/ci) includes a Python adapter and GitHub Actions workflow. It runs a pinned task suite against candidate and baseline models, then fails CI when evidence is incomplete or a configured score or regression threshold is violated. Choose a versioned evaluation and a model that can run it. Check the pair before you spend. GET /v1/evals lists evaluations from admitted providers. The public catalogs show native benchmarks and environments. Your workspace may also have private environments, which need workspace context to discover. Model routes and connections A managed route states its model, capabilities, rates, and limits. Choose the exact catalog entry. A familiar model name alone does not tell you whether the route supports the tools, reasoning, or output format a task needs. A workspace connection points to your provider endpoint, which bills you separately. Checking it makes a small provider request and may cost money outside EvalRouter. Task identity Use eval://provider/name/version to name the exact evaluation. Native aliases such as name@version resolve through the registry. Save the resolved identity with reports; a display name cannot identify the version later. The quote checks model capabilities, input and output limits, task availability, permissions, and resources. An unsupported pair fails before the run starts. Removing a requirement changes the evaluation protocol; it is not a compatibility fix. Coverage A sample freezes its tasks and seed. A full run targets the admitted task collection. Report completed, missing, failed, and excluded tasks separately. Scores from different subsets, prompts, graders, or splits may not be comparable, even under the same benchmark name. Use development results to iterate. Keep held-out tests separate. Remember that prior benchmark exposure or pretraining contamination can limit what a score means. Choose specific tasks To choose tasks yourself, open New evaluation → Benchmarks → Select tasks. Search task IDs and metadata, check tasks for each source, then review the exact IDs before starting. Required dependencies are included when available. The saved setup and repeated runs retain exact task selection. In the CLI, select Specific tasks during interactive evalrouter run, or repeat --task with IDs from that benchmark version: sh evalrouter run BENCHMARK --model ROUTE_ID --task TASK_ID_1 --task TASK_ID_2 --dry-run --task cannot be combined with --sample or --full. GET /v1/catalog/tasks?source_ref=EXACT_REF lists safe task metadata for an accessible source, with q, limit, and offset for search and pagination. The quote checks whether the selected tasks can execute with your model. An unavailable task selection does not fall back to sampling. Recommend tasks for an objective Recommendations are a preview. Save a task selection with the CLI: bash evalrouter recommend --objective "math and reasoning" --model ROUTE_ID --tasks 25 --json > recommendation.json The API equivalent is POST /v1/recommendations with objective, models, and task_limit. Review its selections, uncovered capabilities, exclusions, and warnings. Recommendation itself starts no work and creates no quote. To run the selected tasks of one benchmark, pass their IDs to evalrouter run with --task, as above. Tasks can come from multiple benchmarks and environments. Selection preserves their source versions and explicit task IDs. Declared task metadata determines relevance where available; source labels are a disclosed fallback. Required dependencies count toward the task limit. Results on a selected subset are not full benchmark scores. Optional --budget is historical-cost selection guidance only. Cost evidence can be missing or based on a source mean. It is not a guaranteed price or execution spending cap. Launch uses available workspace credit and actual usage billing. Reusable exact-task suites In New evaluation, choose tasks on the Benchmarks step and select Save selection as suite. A suite saves the exact source versions, splits and task IDs across benchmarks and environments. It does not save a model or start an evaluation. Open Suites to configure an evaluation from a saved version or create its next version. Earlier versions remain available. The API accepts task_selections in POST /v1/suites. Each selection contains source_ref, split and task_ids. Pass previous_version_id to create the next immutable version. Sources must be available to the workspace and selected tasks must include their dependencies. Run evalrouter suites create in a terminal to select sources and exact tasks, name the suite and save without a JSON file or a model run. Use evalrouter suites version --id SUITE_VERSION_ID to edit a saved selection into its next version. Use evalrouter run --suite SUITE_VERSION_ID --model MODEL to reuse a saved version. Add --dry-run to inspect preflight without launching. Exact-task suites use evaluation jobs; the legacy suite quote path rejects them. Suite operations are preview SDK capabilities. CLI availability requires a package containing this feature. 22:T9c6, Credits and billing A quote states the run's maximum EvalRouter charge before any model call. A spending cap is optional: with --max-cost USD (runs create or run) the run starts only when the quote's maximum charge fits under that cap; without one, the run is bounded by your workspace credit. At a terminal the CLI shows the quote and asks before anything is charged; scripts pass --yes, and without it nothing is quoted or charged. EvalRouter holds credit against the cap while work is in flight, charges recorded usage and releases the unused hold. A run that reaches its cap stops new work and keeps its completed results. Cancelling stops new work but cannot retract a call already sent. When EvalRouter's own infrastructure ends an attempt (a lost hosted or remote owner, a lost sandbox, an upstream provider failure, or EvalRouter stopping the run while the attempt was in progress), that attempt's model calls are not charged and its retry pays only for its own calls; calls to your own provider endpoint are billed by your provider and carry no EvalRouter fee. Actual spend and external costs Results show actual EvalRouter spend. A hold reduces available credit while work is in flight; it is not a completed charge. Billing may finish after a score appears. The billing page's Settled charges is everything settled against the wallet for evaluations. It has two parts. Usage is the metered model inference and compute your runs used. Platform fees are EvalRouter's per-evaluation fee, shown on the quote and charged as separate ledger entries, net of any platform fee credits. Credit purchase fees are paid at checkout and are not included. The GET /v1/billing/wallet response reports settled_usage_microusd, settled_platform_fees_microusd and their sum, settled_charges_microusd. A model endpoint or agent you connect may incur charges from its provider outside EvalRouter. EvalRouter's cap does not limit that external bill. Check the external-cost notice before launch. Cost forecasts appear only where that feature is published and qualified. An absent forecast is not a zero-cost estimate. API key ceilings An API key can carry its own spend ceiling across everything it starts: evalrouter api-keys create --spend-ceiling USD, or evalrouter api-keys update KEY_ID --spend-ceiling USD|none. A per-run cap still applies to each run. The API represents money as strings of integer micro-USD: "1000000" means $1. Usage and billing records remain the source for actual charges. 23:T85c, Import a workload Connect a Braintrust, Langfuse, or LangSmith project from [trace connections](/traces/connect). A workspace admin supplies the provider credentials and selects the project and region. EvalRouter encrypts those credentials and syncs traces in the background. The connection shows progress, errors, and controls to pause, resume, import history, or update credentials. For Braintrust and Langfuse datasets or experiments, open a [dataset](/datasets) and choose Import workload data. Select the connected project and resource, declare usage rights, and assign a split. Import subsequent pages until the import completes. Source identifiers and content hashes preserve provenance across retries. Historical model outputs and provider scores remain source evidence. They do not become correct answers or pass labels. Imported inputs seed a task draft for review. Review the instruction, inputs, tool behavior, reference answer, and grading rule before approving a task. Freeze an approved dataset version before using it in Tune. Use development tasks to choose configurations. Reserve separate held-out tasks for the final comparison. A copied task or a paraphrase of it should stay in the same split as its source. You can use reviewed deterministic rules in individual tasks or create a versioned grader in the [grader workspace](/evaluations/evaluators). Model-assisted drafting uses an explicit spending cap. Review its proposal, label calibration examples, and save the calibration against the exact dataset version. Calibration excludes held-out examples and never uses historical scores as human labels. Provider connections and Tune depend on deployment availability. See [availability](/developers/docs/availability). Uploaded executable scorer code is not run by these imports. You can also save a custom HTTPS scorer or pinned Braintrust function in [Evaluators](/evaluations/evaluators). Set its immutable version, dedicated credential, declared charge, and maximum calls per job. Then calibrate the resulting evaluator on reviewed labels. Imported historical scores alone do not make a scorer executable. 24:T19b8, Tune Tune is a preview: it is off unless it is enabled on the deployment, and it is not covered by the stability promise. Tune compares model or agent configurations on a reviewed workload. It records the source, task version, environment, grading configuration, model routes, and spending cap, if any, used in the comparison. Open [Tune](/evaluations/tune). For a complete agent, select a prepared repository and an approved dataset version containing development and held-out tasks. Choose the task grading rules or a saved calibration for that version. Set the objective, quality tolerance, and candidate limit. Review the quote before starting. Candidates run on development tasks. Tune selects one finalist for comparison with the current agent on held-out tasks. Incomplete execution cannot establish a successful replacement. Inspect the result's quality, cost, latency, and execution evidence before adopting a change. Export the tested configuration from the result. The export includes its source and workload commitments and the selected policy. A repository change can be reviewed in the linked improvement project and submitted as a pull request. Opening a pull request does not merge or deploy it. The CLI supports the same flow. See [Tune your repository's agent and open a pull request](tune-your-repositorys-agent-and-open-a-pull-request). To compare admitted ACP versions, select ACP agent versions and enter the environment, baseline, alternatives, and two declared disjoint splits. Each alternative needs its own valid admission. Tune exports the winning admission and native run evidence; it does not modify an admitted image. The CLI accepts --acp-environment, --baseline-agent, and repeated --candidate-agent arguments. For a single model call, choose that mode in the interface or run evalrouter tune start --repo owner/repository --max-draft 2. Review the generated task cases and expected answers before approving the workload. Repository drafting has its own explicit cap. Rubric approval requires a managed judge route (--judge-route in the CLI), whose inference is included in the native quote. Remote evaluator configurations show a maximum call count and declared provider charge estimate before an agent search. Those charges are billed by the scorer provider separately from EvalRouter. Recurring verification requires separate standing authority with a per-run cap and a total cap. It checks the selected configuration against fresh, approved validation tasks. It does not repeat the search or reuse held-out tasks. A scheduled check with no fresh workload records a skipped occurrence without starting a paid job. Queue a reviewed dataset version for a scheduled check, or submit it as an event for an event-triggered check. Versions that use a calibrated evaluator need a fresh calibration of that same evaluator. For ACP results, submit a new approved environment version and its declared split. The check runs the fixed baseline and selected admission on fresh tasks under the occurrence cap. You can pause the authority. Tune must be enabled on the deployment. Production activation requires qualification of the selected application and runtime plus independent approval. Local implementation and passing tests do not establish that activation. See [availability](/developers/docs/availability) and [budgets](/developers/docs/quotes-budgets). Repositories can declare model roles in evalrouter-model-policy.json: json { "schema_version": "evalrouter.model-policy.v1", "policies": { "default": { "planner": "MODEL_ID", "writer": "MODEL_ID" } } } The application must read those roles with the runtime helper: from evalrouter_model_policy import get_model in Python, or require('evalrouter-model-policy.cjs').getModel in Node. Pass the role name to select its model. Tune changes only declared roles, pins managed routes, and rejects unquoted model calls. The declaration alone does not change applications that use hard-coded model names. Tune your repository's agent and open a pull request A coding agent can run this flow from a local clone of a Python uv and LangGraph repository. EvalRouter tests the candidates and returns the selected change as a patch. The agent opens the pull request with local git and the GitHub CLI. 1. Connect GitHub once in the browser at [Integrations](/settings/integrations) and grant the GitHub App access to the repository. This is the only browser step. When the app cannot read the repository, evalrouter improvements create stops with the error code github_repository_unreadable and prints this link. 2. Create and prepare the improvement project. create pins the commit of --ref and uses the workspace's only active trace connection unless you pass --trace-connection. prepare fetches that commit and finds the LangGraph entrypoint from langgraph.json and uv.lock. sh evalrouter improvements create --repo owner/name --ref main --json evalrouter improvements prepare PROJECT_ID --json evalrouter improvements get PROJECT_ID --json 3. Start Tune with an approved dataset version that has development and held-out tasks, then quote the search. Without --max-cost the sweep is paid from wallet credit as it runs and stops when credit runs out. An agent passes --max-cost only with a cap the person gave. Show the quote; the person's request to tune is enough to start the sweep. sh evalrouter tune start --agent-project PROJECT_ID --dataset-version VERSION_ID --json evalrouter tune quote SESSION_ID --json evalrouter tune start-sweep SESSION_ID --json evalrouter tune status SESSION_ID --json evalrouter tune result SESSION_ID --json Repeat status until the session is complete. 4. Export the result as a patch. The patch applies at source_commit_sha in the export. When the result keeps the current agent, the export has no patch, the command stops with the error code tune_no_change and there is nothing to open. sh evalrouter tune export SESSION_ID --patch tune.patch --json 5. Open the pull request from the clone. Write the description from tune result: the held-out quality and cost of the current agent and the candidate, and the session ID. sh git fetch origin git switch -c evalrouter/tune-SESSION_ID SOURCE_COMMIT_SHA git apply --index tune.patch git commit -m "Apply EvalRouter Tune result SESSION_ID" git push -u origin evalrouter/tune-SESSION_ID gh pr create --title "Apply EvalRouter Tune result" --body-file tune-summary.md Opening a pull request does not merge or deploy it. 25:T1432, Save traces to a dataset Keep useful or failed traces together in a workspace-private dataset. Saving a trace copies the content that is still available at that moment. It does not change the source trace or generate a correct answer. Create a dataset Open Datasets, choose New dataset, enter a name, and create it. Members, admins and owners can create datasets and save traces; viewers can read retained snapshots. Add one trace Open a trace and choose Add to dataset. Search for an existing dataset or create one in the same dialog, then save. The saved snapshot includes available span inputs, outputs, errors, events, attributes and source provenance. Missing content stays marked as missing. The picker includes empty datasets and trace collections. Datasets containing workload uploads, curated examples or frozen versions stay separate so saving traces does not interrupt their review or versioning flow. Add several traces On Traces, select rows, then choose Add to dataset. You can select up to 50 traces at once. Search the dataset picker or create a dataset and save the selection. If part of the selection fails, retry the failed traces without saving the successful copies again. You can also open a dataset and choose Add traces. Filter for failed traces, select the ones to keep, and save them. Inspect saved evidence Open a dataset to see saved traces, recorded outcomes and input/output coverage. Choose View snapshot to inspect a copy without leaving the dataset. A dataset has its own retention period. Its saved copies remain available until that period ends, even if the original trace expires. Only content permitted by the source policy can be copied; redaction and workspace access still apply. This initial release supports up to 100 spans and 256 KiB of saved evidence per trace. If the trace changes during capture, retry the save to capture one consistent revision. These snapshots are observations. They do not become reference answers, exportable training data or runnable benchmarks automatically. Dataset imports, review workflows, version exports and benchmark generation are outside this initial Production UI. The existing Dev workload review and versioning screens remain available independently. Turn your app's traces into a benchmark A coding agent can run this whole flow from the CLI with a workspace API key or a CLI sign-in. It is available on Dev, where private benchmarks and workload evaluators are enabled. Add --json to any command for machine-readable output. 1. Create a trace connection. An API key needs the eval:write and traces:write scopes; a CLI sign-in needs the admin or owner role. The ingest key is shown once, so write it to an ignored env file and never commit it. sh evalrouter traces connect --name my-app --env-file .env.local 2. Capture traces with the SDK using EVALROUTER_TRACE_CONNECTION_ID and EVALROUTER_TRACE_KEY. Set the root span input to the user's request and the root span output to your app's final answer. Plain text works best. evalrouter traces list --connection CONNECTION_ID shows what has arrived, and evalrouter traces connections list lists the connections. 3. Create a dataset and import the most recent completed traces. Each trace becomes one held-out example: the root span input is the task and the root span output is the reference answer. Traces without both, traces whose root span failed and oversized traces are skipped and counted in skipped. Importing confirms that your workspace may evaluate models on this data. sh evalrouter datasets create --name "my-app production" evalrouter datasets import-traces DATASET_ID --connection CONNECTION_ID --limit 200 4. Review the examples, then approve them and freeze a version. --all approves every pending example at its current draft. Use --example ID to approve a reviewed subset instead. sh evalrouter datasets approve DATASET_ID --all evalrouter datasets versions create DATASET_ID 5. Build a private benchmark from the version. The match-production grader asks a managed judge model whether each answer is at least as correct, complete and helpful as your app's answer for the same input. Each grade is a metered judge call. Use --grader exact-match for exact string matching instead. An API key needs the benchmarks:submit scope for this step. The command returns benchmark_id. sh evalrouter datasets benchmark DATASET_ID 1 --grader match-production --judge-route JUDGE_ROUTE 6. Run each candidate model on the benchmark, group the runs, and rank them: sh evalrouter run BENCHMARK_ID --model MODEL_A --full --yes evalrouter run BENCHMARK_ID --model MODEL_B --full --yes evalrouter result-groups create --name "my-app models" --run RUN_A --run RUN_B evalrouter result-groups get GROUP_ID --rank --rank orders models by their headline score, then by cost, and marks the best model and the cheapest model within one standard error of it. The cheapest model within one standard error of the best score is the usual switch candidate. A judge that cannot decide leaves that case ungraded, so check coverage before you switch. 26:Te10, Changelog Released features, improvements, and fixes. EvalRouter v2 clients: several models in one run 2026-10-11 · Python and npm 0.5.0 · stable evalrouter and evalrouter-sdk on PyPI · @kimpton-ai/evalrouter on npm Changed - The CLI, Python SDK and npm package 0.4.x no longer work with the API. Every request from them is refused with HTTP 426 client_upgrade_required and the upgrade command, and is never charged. Upgrade to 0.5.0 or later: uv tool upgrade evalrouter (or pipx upgrade evalrouter) for the CLI, pip install --upgrade evalrouter in your project's environment for the Python SDK, npm install @kimpton-ai/evalrouter@latest for npm. - From 0.5.0 on, the CLI prints the exact upgrade command for your installation, and the SDKs raise ClientUpgradeRequired with that command. Added - Run two to eight models on the same tasks in one run under one spending cap: evalrouter run gsm8k --model A --model B, or client.runs.create(benchmark=..., subjects=[...]). - evalrouter runs compare RUN (run.compare()) shows each model's rank, score and 95% range on the tasks every model finished (1= for a tie), and the difference when there are two models. - A run stopped by its spending cap, your credit or a cancellation shows as interrupted. evalrouter runs continue RUN (run.continue_(), npm run.continue()) prices only its unfinished tasks and finishes it without charging the finished ones again. - client.wallet.get() reads your credit balance and client.agents.list() lists the ACP Registry agents you can run, as evalrouter credits and evalrouter agents list do. Removed - Preview SDK groups for development-only features: campaign_templates, environment_drafts, protocols, and the agent attempt protocol client_runs (agents on your machine run through runs.create(agent=...) and evalrouter.agents). One TypeScript SDK for evaluations and tracing 2026-10-06 · npm 0.4.1 · stable @kimpton-ai/evalrouter · npm latest Added - TraceCapture is included in @kimpton-ai/evalrouter and imported from @kimpton-ai/evalrouter/tracing. Install one npm package for the evaluation client and trace capture. - The tracing entry point includes TypeScript declarations and its OpenTelemetry dependencies for server-side capture on Node.js 22 or later. Python SDK and CLI 0.4.0 2026-10-06 · Python 0.4.0 · stable evalrouter · PyPI stable release · includes the CLI Added - Use evalrouter imports for the Python client. Install evalrouter[tracing] to capture agent, model and tool spans with TraceCapture from evalrouter.tracing. - Supported native single- and multi-model benchmarks start without a required quote or spending cap. You can still set an optional cap. Improved - Update the SDK and its included CLI with evalrouter upgrade or evalrouter --upgrade. Retained content in benchmark traces 2026-10-06 · October 6 update · production Workspaces with Traces enabled Fixed - Selecting a model span now loads its available input and output content. Access and retention rules continue to apply. Improved - New native benchmark model requests preserve available prompt inputs even when chat formatting is disabled. Content that was never recorded in older runs remains unavailable. More room in comparison reports 2026-10-02 · Python 0.3.23 · stable Python CLI · stable release Improved - Exported comparison reports have additional space above their charts. Generate a new report to use the updated layout; previously saved exports retain their original layout. A selection of verified releases. New updates will appear here as they ship. b:["$","$2","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/immutable/chunks/3aohrjh7bn_av.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/_next/static/immutable/chunks/1s_5by4_edxu0.js","async":true,"nonce":"$undefined"}]],["$","$L15",null,{"pages":[{"slug":"overview","title":"EvalRouter","section":"Get started","summary":"Run model benchmarks and inspect their results.","search":"$16"},{"slug":"quickstart","title":"Run a model evaluation","section":"CLI model evaluations","summary":"Install, sign in, choose a benchmark and model, run and export.","search":"$17"},{"slug":"authentication","title":"Account and workspace key","section":"CLI model evaluations","summary":"Create an account and configure a scoped workspace key for the CLI.","search":"$18"},{"slug":"cli-login","title":"Sign in from the CLI","section":"CLI model evaluations","summary":"Sign the CLI in with your EvalRouter account, switch workspaces and revoke devices.","search":"$19"},{"slug":"cli","title":"CLI commands","section":"CLI model evaluations","summary":"Discover models and benchmarks, manage model connections, quote, run, monitor and export.","search":"$1a"},{"slug":"results","title":"Results and exports","section":"CLI model evaluations","summary":"Read status, coverage and billing alongside native scores and versioned reports.","search":"$1b"},{"slug":"availability","title":"Availability and limits","section":"CLI model evaluations","summary":"Check runnable model/profile combinations and spending requirements.","search":"$1c"},{"slug":"model-api","title":"Model API reference","section":"API reference","summary":"Authentication, model discovery, quotes, bounded runs, status, results and exports.","search":"Model API reference Authentication and workspace Discover model routes and benchmark profiles Model connections Quote a model evaluation Submit once and follow status Results and exports Errors and recovery"},{"slug":"bring-your-own-benchmark","title":"Bring your own benchmark","section":"Benchmarks","summary":"Create a private benchmark with built-in scoring or request a maintained catalog addition.","search":"$1d"},{"slug":"huggingface","title":"Benchmarks from Hugging Face datasets","section":"Benchmarks","summary":"Pin a Hugging Face dataset to a commit; we check its hashes and license and keep our own copy.","search":"$1e"},{"slug":"feedback","title":"Report bugs and missing features","section":"CLI model evaluations","summary":"Report a defect, missing capability or docs gap from the API or CLI, then check its outcome.","search":"$1f"},{"slug":"byob","title":"Private benchmarks from your own data","section":"Benchmarks","summary":"Upload a dataset or import from Hugging Face, define scoring, and run a benchmark private to your workspace.","search":"$20"},{"slug":"selection","title":"Models, environments, and compatibility","section":"Evaluations","summary":"Select exact admitted identities and inspect compatibility before spending.","search":"$21"},{"slug":"quotes-budgets","title":"Quotes, budgets, and billing","section":"Evaluations","summary":"A quote bounds platform spending and makes external charges explicit.","search":"$22"},{"slug":"workload-imports","title":"Import a workload","section":"Workload optimization","summary":"Connect provider traces and datasets, review tasks, and freeze grading evidence.","search":"$23"},{"slug":"tune","title":"Tune models and agents","section":"Workload optimization","summary":"Compare configurations on a reviewed workload and export the tested change.","search":"$24"},{"slug":"trace-datasets","title":"Save traces to a dataset","section":"Traces and datasets","summary":"Save retained traces as evidence, or turn recent traces into a private benchmark.","search":"$25"},{"slug":"changelog","title":"Changelog","section":"Releases","summary":"Released app, CLI, and SDK updates, with explicit alpha and stable channels.","search":"$26"}],"children":["$L27","$L28"]}]]}] c:["$","$2","c",{"children":[null,["$","$L6",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L8",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}] d:["$","$2","c",{"children":["$L29",null,["$","$L2a",null,{"children":["$","$2b",null,{"name":"Next.MetadataOutlet","children":"$@2c"}]}]]}] f:["$","$2","h",{"children":[null,["$","$L2d",null,{"children":"$L2e"}],["$","div",null,{"hidden":true,"children":["$","$L2f",null,{"children":["$","$2b",null,{"name":"Next.Metadata","children":"$L30"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}] 11:["$","link","0",{"rel":"stylesheet","href":"/_next/static/immutable/chunks/1g7csgrqtlfkz.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] 12:["$","link","1",{"rel":"stylesheet","href":"/_next/static/immutable/chunks/3k5f1v0v_00uw.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}] e:C 31:T530,{"@context":"https://schema.org","@graph":[{"@type":"SoftwareApplication","@id":"https://evalrouter.ai/#software","name":"EvalRouter","url":"https://evalrouter.ai","description":"One API for thousands of benchmarks. Compare models without managing evaluation infrastructure. Route to any benchmark for grading, with an optional spending cap.","applicationCategory":"DeveloperApplication","operatingSystem":"macOS, Linux, Windows","softwareHelp":{"@type":"CreativeWork","url":"https://evalrouter.ai/developers/docs"},"installUrl":"https://pypi.org/project/evalrouter/","offers":{"@type":"Offer","price":"0","priceCurrency":"USD","description":"Free to install. Evaluations are charged in USD against prepaid workspace credit, within any spending cap you set. Connected providers bill inference separately."},"publisher":{"@type":"Organization","name":"Kimpton AI, Inc.","url":"https://kimpton.ai"},"sameAs":["https://pypi.org/project/evalrouter/","https://github.com/kimpton-ai/evalrouter-skills"]},{"@type":"WebAPI","@id":"https://evalrouter.ai/#api","name":"EvalRouter API","url":"https://api.evalrouter.ai","description":"HTTP API for benchmark discovery, spending-capped evaluations, status, results and exports.","documentation":"https://evalrouter.ai/developers/docs","provider":{"@id":"https://evalrouter.ai/#software"}}]}27:["$","script",null,{"type":"application/ld+json","dangerouslySetInnerHTML":{"__html":"$31"}}] 28:["$","$L6",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L8",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}] 29:["$","div",null,{"className":"documentation-changelog","children":[["$","header",null,{"className":"changelog-introduction","children":[["$","h1",null,{"children":"Changelog"}],["$","p",null,{"children":"What’s new, what’s improved, and what’s fixed."}],["$","nav",null,{"aria-label":"Changelog products","className":"changelog-products","children":[["$","a",null,{"href":"/developers/docs/changelog","aria-current":"page","children":"App"}],["$","a",null,{"href":"/developers/docs/changelog?product=packages","aria-current":"$undefined","children":"CLI & SDK"}]]}],["$","p",null,{"className":"changelog-latest","children":["Latest ",["$","a",null,{"href":"#app-2026-10-06","children":"October 6 update"}],["$","span",null,{"aria-hidden":true,"children":"·"}],["$","time",null,{"dateTime":"2026-10-06","children":"October 6, 2026"}]]}],["$","p",null,{"className":"changelog-channel-note","children":"Availability is noted for features with a limited rollout."}]]}],["$","div",null,{"className":"changelog-timeline","children":[["$","article","app-2026-10-06",{"id":"app-2026-10-06","className":"changelog-entry","aria-labelledby":"app-2026-10-06-title","children":[["$","div",null,{"className":"changelog-release-meta","children":[["$","time",null,{"dateTime":"2026-10-06","children":"October 6, 2026"}],["$","a",null,{"className":"changelog-version","href":"#app-2026-10-06","children":"October 6 update"}],["$","span",null,{"className":"changelog-channel","data-channel":"production","children":"Production"}]]}],["$","div",null,{"className":"changelog-release-content","children":[["$","h2",null,{"id":"app-2026-10-06-title","children":"Retained content in benchmark traces"}],["$","p",null,{"className":"changelog-availability","children":["Workspaces with Traces enabled"]}],[["$","section","Fixed",{"aria-label":"Fixed","children":[["$","h3",null,{"children":"Fixed"}],["$","ul",null,{"children":[["$","li","Selecting a model span now loads its available input and output content. Access and retention rules continue to apply.",{"children":["Selecting a model span now loads its available input and output content. Access and retention rules continue to apply."]}]]}]]}],["$","section","Improved",{"aria-label":"Improved","children":[["$","h3",null,{"children":"Improved"}],["$","ul",null,{"children":[["$","li","New native benchmark model requests preserve available prompt inputs even when chat formatting is disabled. Content that was never recorded in older runs remains unavailable.",{"children":["New native benchmark model requests preserve available prompt inputs even when chat formatting is disabled. Content that was never recorded in older runs remains unavailable."]}]]}]]}]],false]}]]}]]}],["$","p",null,{"className":"changelog-history-note","children":"A selection of verified releases. New updates will appear here as they ship."}]]}] 2e:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]] 32:I[281648,["/_next/static/immutable/chunks/19rwqzjcor83k.js","/_next/static/immutable/chunks/0b31e3qldb1c1.js","/_next/static/immutable/chunks/0s81a39zwttql.js","/_next/static/immutable/chunks/42lqb6gnfp-ec.js","/_next/static/immutable/chunks/0hwxphe_gn9vz.js","/_next/static/immutable/chunks/3qzarik02d-ve.js","/_next/static/immutable/chunks/3oq2rvn05ot3q.js","/_next/static/immutable/chunks/2s7e2fbltj51p.js","/_next/static/immutable/chunks/3sqy9_inm3v1n.js","/_next/static/immutable/chunks/22opv1dsilvnc.js","/_next/static/immutable/chunks/1tmi36srrzjjw.js","/_next/static/immutable/chunks/1g8gja0xe2spr.js","/_next/static/immutable/chunks/0_-kt8naw9vzn.js"],"IconMark"] 2c:null 30:[["$","title","0",{"children":"Changelog"}],["$","meta","1",{"name":"description","content":"Released features, improvements, and fixes for EvalRouter and its CLI and SDK packages."}],["$","meta","2",{"property":"og:title","content":"EvalRouter"}],["$","meta","3",{"property":"og:description","content":"One API for thousands of benchmarks. Compare models without managing evaluation infrastructure. Route to any benchmark for grading, with an optional spending cap."}],["$","meta","4",{"property":"og:site_name","content":"EvalRouter"}],["$","meta","5",{"property":"og:image","content":"https://evalrouter.ai/social/home.png"}],["$","meta","6",{"property":"og:image:width","content":"2400"}],["$","meta","7",{"property":"og:image:height","content":"1260"}],["$","meta","8",{"property":"og:image:alt","content":"EvalRouter. One API for thousands of benchmarks."}],["$","meta","9",{"property":"og:type","content":"website"}],["$","meta","10",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","11",{"name":"twitter:title","content":"EvalRouter"}],["$","meta","12",{"name":"twitter:description","content":"One API for thousands of benchmarks. Compare models without managing evaluation infrastructure. Route to any benchmark for grading, with an optional spending cap."}],["$","meta","13",{"name":"twitter:image","content":"https://evalrouter.ai/social/home.png"}],["$","meta","14",{"name":"twitter:image:alt","content":"EvalRouter. One API for thousands of benchmarks."}],["$","link","15",{"rel":"icon","href":"/favicon.ico?favicon.1kce_2ger0q4r.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","16",{"rel":"icon","href":"/icon.png?icon.2aiwe6upnfkk2.png","sizes":"64x64","type":"image/png"}],["$","link","17",{"rel":"apple-touch-icon","href":"/apple-icon.png?apple-icon.3jo8bqgaae458.png","sizes":"180x180","type":"image/png"}],["$","$L32","18",{}]]