Read a run's results over HTTP

This example finds an evaluation’s latest run, reads each cell’s average score and pages through each task’s result.

This example pulls run results into your own tools with a read-only key. It needs CONTEXT_APP_ORIGIN and CONTEXT_API_KEY from the quickstart, and the ID of an evaluation that has run: GET /api/v1/evals/benchmarks lists them (send workspaceScope as in the quickstart), and GET /api/v1/evals/benchmarks/{evaluationId}/runs shows whether one has run (or use context evals benchmark list, whose rows carry runCount).

export EVALUATION_ID="<evaluation-id>"

Find the latest run

The list of an evaluation’s runs isn’t in a set order, so sort it by createdAt. A run’s status ends as COMPLETED, FAILED or STOPPED; scores are final only once it has ended.

curl -sS --fail-with-body "$CONTEXT_APP_ORIGIN/api/v1/evals/benchmarks/$EVALUATION_ID/runs" \
  -H "Authorization: Bearer $CONTEXT_API_KEY" |
  jq -r 'sort_by(.createdAt) | reverse | .[] | [.id, .status, .createdAt] | @tsv'

import os

import requests

origin = os.environ["CONTEXT_APP_ORIGIN"]
headers = {"Authorization": f"Bearer {os.environ['CONTEXT_API_KEY']}"}
evaluation_id = os.environ["EVALUATION_ID"]

resp = requests.get(f"{origin}/api/v1/evals/benchmarks/{evaluation_id}/runs", headers=headers, timeout=60)
resp.raise_for_status()
for run in sorted(resp.json(), key=lambda r: r["createdAt"], reverse=True):
    print(run["id"], run["status"], run["createdAt"])

function env(name: string): string {
  const value = process.env[name]
  if (!value) throw new Error(`Set ${name} first`)
  return value
}

const url = new URL(`/api/v1/evals/benchmarks/${encodeURIComponent(env("EVALUATION_ID"))}/runs`, env("CONTEXT_APP_ORIGIN"))
const res = await fetch(url, { headers: { Authorization: `Bearer ${env("CONTEXT_API_KEY")}` } })
if (!res.ok) throw new Error(`${res.status} ${await res.text()}`)
const runs = (await res.json()) as Array<{ id: string; status: string; createdAt: string }>
runs.sort((a, b) => b.createdAt.localeCompare(a.createdAt))
for (const run of runs) console.log(run.id, run.status, run.createdAt)

export RUN_ID="<run-id>"

Read each cell's score

GET /runs/{runId}/status returns the run, with its cells, and a stats entry per cell. Join the two on cellId to see which model or agent each score belongs to. avgScore is from 0 to 1, and null until something has been graded.

curl -sS --fail-with-body "$CONTEXT_APP_ORIGIN/api/v1/evals/runs/$RUN_ID/status" \
  -H "Authorization: Bearer $CONTEXT_API_KEY" |
  jq '.run.cells as $cells | .stats[] | . as $s | {cellId, value: ($cells[] | select(.cellId == $s.cellId) | .value), avgScore, succeeded, failed}'

import os

import requests

origin = os.environ["CONTEXT_APP_ORIGIN"]
headers = {"Authorization": f"Bearer {os.environ['CONTEXT_API_KEY']}"}
run_id = os.environ["RUN_ID"]

resp = requests.get(f"{origin}/api/v1/evals/runs/{run_id}/status", headers=headers, timeout=60)
resp.raise_for_status()
status = resp.json()
cells = {cell["cellId"]: cell for cell in status["run"]["cells"]}
for stat in status["stats"]:
    cell = cells[stat["cellId"]]
    print(stat["cellId"], cell["value"], "repeat", cell["attempt"], "avgScore", stat["avgScore"])

function env(name: string): string {
  const value = process.env[name]
  if (!value) throw new Error(`Set ${name} first`)
  return value
}

type Cell = { cellId: string; value: unknown; attempt: number }
type Stat = { cellId: string; avgScore: number | null }

const url = new URL(`/api/v1/evals/runs/${encodeURIComponent(env("RUN_ID"))}/status`, env("CONTEXT_APP_ORIGIN"))
const res = await fetch(url, { headers: { Authorization: `Bearer ${env("CONTEXT_API_KEY")}` } })
if (!res.ok) throw new Error(`${res.status} ${await res.text()}`)
const status = (await res.json()) as { run: { cells: Cell[] }; stats: Stat[] }
for (const stat of status.stats) {
  const cell = status.run.cells.find((c) => c.cellId === stat.cellId)
  console.log(stat.cellId, JSON.stringify(cell?.value), "repeat", cell?.attempt, "avgScore", stat.avgScore)
}

export CELL_ID="<cell-id>"

Page through each task's result

GET /runs/{runId}/tasks takes a cellId and returns up to 50 tasks a page. Each row has the task’s score, its output preview, outputStatus (whether a recorded answer is available), costUsd, latencyMs, and the outcomes of its first 8 criteria under criteria.entries; criteria.totalCount says how many it has. Pass nextCursor back as cursor until it is null.

curl -sS --fail-with-body --get "$CONTEXT_APP_ORIGIN/api/v1/evals/runs/$RUN_ID/tasks" \
  -H "Authorization: Bearer $CONTEXT_API_KEY" \
  --data-urlencode "cellId=$CELL_ID" \
  --data-urlencode 'limit=50' |
  jq -r '.rows[] | [.taskId, .score, .outputStatus] | @tsv'

import os

import requests

origin = os.environ["CONTEXT_APP_ORIGIN"]
headers = {"Authorization": f"Bearer {os.environ['CONTEXT_API_KEY']}"}
run_id, cell_id = os.environ["RUN_ID"], os.environ["CELL_ID"]

params = {"cellId": cell_id, "limit": 50}
while True:
    resp = requests.get(f"{origin}/api/v1/evals/runs/{run_id}/tasks", headers=headers, params=params, timeout=60)
    resp.raise_for_status()
    page = resp.json()
    for row in page["rows"]:
        print(row["taskId"], row["score"], row["outputStatus"])
    if page["nextCursor"] is None:
        break
    params["cursor"] = page["nextCursor"]

function env(name: string): string {
  const value = process.env[name]
  if (!value) throw new Error(`Set ${name} first`)
  return value
}

type Row = { taskId: string; score: number | null; outputStatus: string }

let cursor: string | null = null
do {
  const url = new URL(`/api/v1/evals/runs/${encodeURIComponent(env("RUN_ID"))}/tasks`, env("CONTEXT_APP_ORIGIN"))
  url.searchParams.set("cellId", env("CELL_ID"))
  url.searchParams.set("limit", "50")
  if (cursor) url.searchParams.set("cursor", cursor)
  const res = await fetch(url, { headers: { Authorization: `Bearer ${env("CONTEXT_API_KEY")}` } })
  if (!res.ok) throw new Error(`${res.status} ${await res.text()}`)
  const page = (await res.json()) as { rows: Row[]; nextCursor: string | null }
  for (const row of page.rows) console.log(row.taskId, row.score, row.outputStatus)
  cursor = page.nextCursor
} while (cursor)

GET /runs/{runId}/tasks/{taskId}?cellId=... returns one task’s full answer and every criterion’s outcome with its reasoning.