{ "openapi": "3.1.0", "info": { "title": "Evals API", "version": "1.0.0", "description": "Read-only HTTP API for Evals: analytics, Evaluations, datasets, runs, traces, feedback, judges and scorecards. Every endpoint needs a task API key with the `evals:read` capability. The organization comes from the key. A key reads no more than its owner reads in the app. A member without Evals access reads only traces of agents they own. A key limited to specific agents reads only those agents' traces and is refused (403) on every other endpoint except `/feedback/enabled` and `/feedback/schemas`." }, "servers": [ { "url": "https://{deployment}/api/v1/evals", "description": "Your Context deployment", "variables": { "deployment": { "default": "context.example.com", "description": "The host your organization signs in to" } } } ], "security": [ { "TaskApiKey": [] }, { "TaskApiKeyHeader": [] } ], "tags": [ { "name": "Analytics" }, { "name": "Evaluations" }, { "name": "Datasets" }, { "name": "Runs" }, { "name": "Feedback" }, { "name": "Judges" }, { "name": "Scorecards" }, { "name": "Traces" } ], "components": { "securitySchemes": { "TaskApiKey": { "type": "http", "scheme": "bearer", "bearerFormat": "tsk__", "description": "A task API key with the `evals:read` capability. Create one in the app under Settings > API > Task keys." }, "TaskApiKeyHeader": { "type": "apiKey", "in": "header", "name": "X-API-Key", "description": "The same task API key, sent in the `X-API-Key` header instead." } }, "schemas": { "Error": { "type": "object", "required": [ "error" ], "properties": { "error": { "type": "string", "description": "What went wrong, in words." }, "code": { "type": "string", "description": "A stable code for some errors, for example `TASK_CAPABILITY_MISSING` or `INVALID_CURSOR`." }, "details": { "type": "array", "description": "On `400`: one entry per invalid parameter.", "items": { "type": "object", "required": [ "path", "message" ], "properties": { "path": { "type": "string", "description": "The parameter's name; a nested field is joined with dots, for example `filters.0.field`." }, "message": { "type": "string", "description": "What is wrong with it." } } } } } }, "RateLimited": { "type": "object", "required": [ "ok", "error", "scope", "retryAfterMs" ], "properties": { "ok": { "type": "boolean", "enum": [ false ] }, "error": { "type": "string", "enum": [ "rate_limited" ] }, "scope": { "type": "string", "enum": [ "principal" ] }, "retryAfterMs": { "type": "integer", "minimum": 0, "description": "Milliseconds until the key may send again." } } } } }, "paths": { "/analytics/records": { "get": { "tags": [ "Analytics" ], "summary": "List analytics records", "description": "Paginated analytics records (root tasks, sub-agent tasks, and dispatch runs), with filters.", "operationId": "listAnalyticsRecords", "parameters": [ { "in": "query", "name": "agentIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`agentIds=[\"a\",\"b\"]`) or repeat the key (`agentIds=a&agentIds=b`). A single plain value (`agentIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "workspaceIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`workspaceIds=[\"a\",\"b\"]`) or repeat the key (`workspaceIds=a&workspaceIds=b`). A single plain value (`workspaceIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "workspaceTagIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`workspaceTagIds=[\"a\",\"b\"]`) or repeat the key (`workspaceTagIds=a&workspaceTagIds=b`). A single plain value (`workspaceTagIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "sourceType", "required": false, "schema": { "type": "string", "enum": [ "manual", "scheduled", "api", "internal-api", "prewarmed", "local-device", "agent-brain", "subagent" ], "description": "Started from: keep only records that started on this source (a task source type, or `subagent` for tasks started by another task)." } }, { "in": "query", "name": "usedFrom", "required": false, "schema": { "type": "array", "items": { "type": "string", "enum": [ "web", "mobile", "cli", "api" ], "description": "An interface a task was used from: `web` (the web app and the desktop app), `mobile` (the native mobile app), `cli` (the `context-code` terminal on a paired machine), or `api` (the Task API with your own key). The platform's own automation is not an interface." }, "minItems": 1, "description": "Used from: keep only tasks a person acted on through ANY of these interfaces (a turn sent from the app, a run driven by a paired machine, or where the task started). Send a JSON list in one value (`usedFrom=[\"a\",\"b\"]`) or repeat the key (`usedFrom=a&usedFrom=b`). A single plain value (`usedFrom=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "userIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`userIds=[\"a\",\"b\"]`) or repeat the key (`userIds=a&userIds=b`). A single plain value (`userIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "email", "required": false, "schema": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Case-insensitive substring matched against the owning user's email, to narrow results to one person." } }, { "in": "query", "name": "countries", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`countries=[\"a\",\"b\"]`) or repeat the key (`countries=a&countries=b`). A single plain value (`countries=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "taskOpencodeBranchId", "required": false, "schema": { "type": "string", "minLength": 1 } }, { "in": "query", "name": "skillIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`skillIds=[\"a\",\"b\"]`) or repeat the key (`skillIds=a&skillIds=b`). A single plain value (`skillIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "workflowIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1, "maxItems": 50, "description": "Workflow ids. A record matches when its chat is classified into any of them. Send a JSON list in one value (`workflowIds=[\"a\",\"b\"]`) or repeat the key (`workflowIds=a&workflowIds=b`). A single plain value (`workflowIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "errorMessage", "required": false, "schema": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Case-insensitive substring matched against the error messages of calls in the record's own chat (the same filter the app's Traces view offers). A record with no chat never matches." } }, { "in": "query", "name": "from", "required": false, "schema": { "type": "string", "format": "date-time" } }, { "in": "query", "name": "to", "required": false, "schema": { "type": "string", "format": "date-time" } }, { "in": "query", "name": "minRating", "required": false, "schema": { "type": "number", "minimum": 1, "maximum": 5 } }, { "in": "query", "name": "maxRating", "required": false, "schema": { "type": "number", "minimum": 1, "maximum": 5 } }, { "in": "query", "name": "hasFeedback", "required": false, "schema": { "type": "boolean" } }, { "in": "query", "name": "search", "required": false, "schema": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Case-insensitive substring matched against the task title (a dispatch run matches on its task's title, otherwise its prompt). A task with no title of its own matches on its latest dispatch run's title or prompt, the same fallback the list shows." } }, { "in": "query", "name": "statuses", "required": false, "schema": { "type": "array", "items": { "type": "string", "enum": [ "completed", "failed", "running", "pending", "cancelled", "archived" ], "description": "A status from the status breakdown: `completed` (Completed or Succeeded), `failed`, `running` (Queued or Running), `pending`, `cancelled`, or `archived`." }, "minItems": 1, "description": "Keep only records whose current status (as shown in the status breakdown) is ANY of these. Send a JSON list in one value (`statuses=[\"a\",\"b\"]`) or repeat the key (`statuses=a&statuses=b`). A single plain value (`statuses=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "origins", "required": false, "schema": { "type": "array", "items": { "type": "string", "enum": [ "evaluation-run", "judge", "skill-learning-replay", "production" ], "description": "What started a chat: a task in an evaluation run, a judge grading one, a skill-learning replay, or the organization's own production work." }, "minItems": 1, "description": "Keep only records started by ANY of these kinds of work: `production`, `evaluation-run`, `judge`, or `skill-learning-replay`. The origin comes from the record's root task: whether an evaluation started it, then whether it runs as the organization's judge agent. Omit to keep every origin. Send a JSON list in one value (`origins=[\"a\",\"b\"]`) or repeat the key (`origins=a&origins=b`). A single plain value (`origins=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "cursor", "required": false, "schema": { "type": "string", "minLength": 1, "description": "Cursor from the previous page's `pageInfo.nextCursor`; stop when `pageInfo.hasNextPage` is false." } }, { "in": "query", "name": "limit", "required": false, "schema": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Maximum number of top-level records to return. Sub-agent rows shown under those records can make `data.length` larger.", "default": 25 } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "data": { "type": "array", "items": { "type": "object", "properties": { "recordId": { "type": "string", "minLength": 1 }, "recordKind": { "type": "string", "enum": [ "task", "dispatchRun" ] }, "recordType": { "type": "string", "enum": [ "rootTask", "subAgent", "dispatchRun" ], "description": "Kind of record: `rootTask`, `subAgent`, or `dispatchRun`." }, "taskId": { "type": [ "string", "null" ] }, "parentTaskId": { "type": [ "string", "null" ], "description": "Parent task ID when this row is a sub-agent task; null for root tasks and standalone dispatch runs." }, "runId": { "type": [ "string", "null" ] }, "taskOpencodeBranchId": { "type": [ "string", "null" ] }, "sourceType": { "type": "string", "enum": [ "manual", "scheduled", "api", "internal-api", "prewarmed", "local-device", "agent-brain", "subagent" ], "description": "Where the record started: one of the task source types, or `subagent` for child tasks started by another task." }, "title": { "type": "string", "minLength": 1 }, "createdAt": { "type": "string" }, "taskStatus": { "type": [ "string", "null" ], "enum": [ "active", "completed", "failed", "archived", null ] }, "latestRunStatus": { "type": [ "string", "null" ], "enum": [ "queued", "running", "succeeded", "failed", "cancelled", null ] }, "displayStatus": { "type": "string", "minLength": 1 }, "user": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 }, "email": { "type": "string", "format": "email" }, "country": { "type": [ "string", "null" ] } }, "required": [ "id", "label", "email" ], "additionalProperties": false }, "agent": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "workspace": { "type": [ "object", "null" ], "properties": { "id": { "type": "string", "minLength": 1 }, "name": { "type": "string", "minLength": 1 } }, "required": [ "id", "name" ], "additionalProperties": false }, "workspaceTag": { "anyOf": [ { "allOf": [ { "type": [ "object", "null" ], "properties": { "id": { "type": "string", "minLength": 1 }, "name": { "type": "string", "minLength": 1 } }, "required": [ "id", "name" ], "additionalProperties": false } ] }, { "type": "null" } ] }, "messageCount": { "type": [ "integer", "null" ], "minimum": 0 }, "userMessageCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Number of user-written messages in the task's latest archived conversation; null (not 0) when the task has no archived conversation." }, "skillsUsed": { "type": [ "array", "null" ], "items": { "type": "string" } }, "skillsUsedDetails": { "type": [ "array", "null" ], "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false } }, "connectorsAttached": { "type": [ "array", "null" ], "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "description": "MCP connectors attached to the record's agent (including the organization's custom connectors on the agent's list or covered by its All connectors setting), as connector names sorted by label; null when the agent has none. Attachments are not recorded per run, so this shows the agent's connectors at the time you read it, not when the record ran." }, "metrics": { "type": "object", "properties": { "runCount": { "type": "integer", "minimum": 0 }, "inputFiles": { "type": "integer", "minimum": 0 }, "outputFiles": { "type": "integer", "minimum": 0 }, "attempts": { "type": "integer", "minimum": 0 }, "totalTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Billed tokens (input + output + reasoning + cache write + cache read) summed across the record's LLM calls in trace data. Null when the record has no chat, there is no trace data for it, or trace data cannot be read." } }, "required": [ "runCount", "inputFiles", "outputFiles", "attempts", "totalTokens" ], "additionalProperties": false }, "taskViews": { "type": "object", "properties": { "viewCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Recorded opens of this task's conversation across every surface that records views. Null (not 0) when views were never measured for this record (see `unmeasuredReason`)." }, "unmeasuredReason": { "type": [ "string", "null" ], "enum": [ "no-view-key", "predates-instrumentation", null ], "description": "Why `viewCount` is null. Null whenever `viewCount` is a real measurement, including a real zero." }, "lastViewedAt": { "type": [ "string", "null" ] }, "lastViewedBy": { "anyOf": [ { "allOf": [ { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 }, "email": { "type": "string", "format": "email" }, "country": { "type": [ "string", "null" ] } }, "required": [ "id", "label", "email" ], "additionalProperties": false } ] }, { "type": "null" } ] }, "viewedFrom": { "type": "array", "items": { "type": "string", "enum": [ "task_route", "shared_link", "eval_analytics", "context_code", "eval_project" ] }, "description": "Distinct surfaces where a person opened this record's conversation, in the enum's order. Empty when `viewCount` is null or 0: a surface appears only when at least one view was recorded on it." } }, "required": [ "viewCount", "unmeasuredReason", "lastViewedAt", "lastViewedBy", "viewedFrom" ], "additionalProperties": false, "description": "View measurement for an analytics record (a task, or a standalone dispatch run matched by its chat)." }, "feedbackSummary": { "type": "object", "properties": { "count": { "type": "integer", "minimum": 0, "description": "Number of feedback submissions on this record" }, "avgRating": { "type": [ "number", "null" ], "minimum": 1, "maximum": 5, "description": "Mean numeric rating on the 1 to 5 scale, or null when there is no numeric rating" }, "maxRating": { "type": [ "number", "null" ], "minimum": 1, "maximum": 5, "description": "Highest numeric rating on the 1 to 5 scale, or null when there is no numeric rating" }, "withTextCount": { "type": "integer", "minimum": 0, "description": "Number of submissions with a non-empty comment" } }, "required": [ "count", "avgRating", "maxRating", "withTextCount" ], "additionalProperties": false, "description": "Feedback summary for one task, run, or trace session" }, "execution": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "traced" ] }, "sessionIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1, "description": "The chat's trace sessions, latest first." }, "model": { "type": [ "string", "null" ], "minLength": 1, "description": "The model that served the most calls in the chat; null when no call named a model" }, "models": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Every named model in the chat, most calls first; empty when no call named a model" }, "callCount": { "type": "integer", "minimum": 0 }, "tokens": { "type": "integer", "minimum": 0, "description": "Billed tokens (input + output + reasoning + cache write + cache read) across the chat's own calls" }, "durationMs": { "type": "integer", "minimum": 0, "description": "Total duration of the chat's own calls, in milliseconds" }, "errorCount": { "type": "integer", "minimum": 0, "description": "Number of calls that ended unsuccessfully" }, "estimatedCostUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "Estimated cost summed over the chat's calls. An estimate, not the bill: the billed amount is `billedCost`. Null when no call reported a cost." }, "errorMessageSample": { "type": [ "string", "null" ], "minLength": 1, "description": "The error message of one failed call (aborted calls excluded); null when no failed call had one" }, "lastActivityAt": { "type": "string", "description": "Time of the chat's latest call (ISO 8601)" }, "subagents": { "type": [ "object", "null" ], "properties": { "count": { "type": "integer", "exclusiveMinimum": 0, "description": "Number of sub-agent rows on this page grouped under the parent" }, "tokens": { "type": "integer", "minimum": 0, "description": "Billed tokens of the grouped sub-agent chats" }, "estimatedCostUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "Estimated cost of the grouped sub-agent chats; null when none of them reported one" }, "errorCount": { "type": "integer", "minimum": 0, "description": "Number of unsuccessful calls in the grouped sub-agent chats" } }, "required": [ "count", "tokens", "estimatedCostUsd", "errorCount" ], "additionalProperties": false, "description": "Totals for the grouped sub-agent rows; null when the row has no sub-agent rows on this page" } }, "required": [ "kind", "sessionIds", "model", "models", "callCount", "tokens", "durationMs", "errorCount", "estimatedCostUsd", "errorMessageSample", "lastActivityAt", "subagents" ], "additionalProperties": false, "description": "Execution data is available: the chat has trace data." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "none" ] }, "reason": { "type": "string", "enum": [ "no_chat_id", "no_trace_rows", "trace_store_unavailable" ] } }, "required": [ "kind", "reason" ], "additionalProperties": false, "description": "The row has no execution data; `reason` says why." } ], "description": "Execution data for a task row, taken from the chat's trace data" }, "billedCost": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "known" ] }, "costUsdMicros": { "type": "integer", "minimum": 0, "description": "Billed model spend in millionths of a US dollar, read from billing records (never recalculated)" } }, "required": [ "kind", "costUsdMicros" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "unknown" ] }, "reason": { "type": "string", "enum": [ "ledger_not_final", "device_calls_unbounded", "no_rows", "unrated_rows", "ledger_unavailable" ], "description": "Why the billed cost is unavailable. `ledger_not_final`: billing for the run is not final, or the run can still be billed for a call. `device_calls_unbounded`: a run ran on a paired machine, whose spend is never reported as final. `no_rows`: there is no billing record for it. `unrated_rows`: a billing record has no rated cost. `ledger_unavailable`: billing records could not be read." } }, "required": [ "kind", "reason" ], "additionalProperties": false } ], "description": "The billed cost of a task row, the same figure Settings › Usage shows" }, "workflows": { "type": [ "array", "null" ], "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "description": "Workflows the record's chat is classified into. `[]` when the chat is unclassified; null when classifications could not be read." }, "origin": { "type": "string", "enum": [ "evaluation-run", "judge", "skill-learning-replay", "production" ], "description": "The kind of work that started the record, taken from its chat's root task: whether an evaluation started it, then whether it runs as the organization's judge agent. A sub-agent takes its root's origin. A dispatch run whose chat has no task is judged on its own. This is the same rule the `origins` filter uses." } }, "required": [ "recordId", "recordKind", "recordType", "taskId", "parentTaskId", "runId", "taskOpencodeBranchId", "sourceType", "title", "createdAt", "taskStatus", "latestRunStatus", "displayStatus", "user", "agent", "workspace", "workspaceTag", "messageCount", "userMessageCount", "skillsUsed", "skillsUsedDetails", "connectorsAttached", "metrics", "taskViews", "feedbackSummary", "execution", "billedCost", "workflows", "origin" ], "additionalProperties": false } }, "pageInfo": { "type": "object", "properties": { "nextCursor": { "type": [ "string", "null" ], "description": "Cursor for fetching the next page; pass it as `cursor`" }, "hasNextPage": { "type": "boolean", "description": "Whether another page exists for the current filters" } }, "required": [ "nextCursor", "hasNextPage" ], "additionalProperties": false, "description": "Cursor pagination state" } }, "required": [ "data", "pageInfo" ], "additionalProperties": false }, "example": { "data": [ { "recordId": "76061c75-9c20-4694-8584-4f20270a243d", "recordKind": "task", "recordType": "rootTask", "taskId": "de707028-6ea2-4616-8864-c2ebd92b87f5", "parentTaskId": "2ad315e1-e215-4b88-8ba7-9869e414af3d", "runId": "599e5591-3334-4d0a-8bb9-069b278d9b20", "taskOpencodeBranchId": "80f2aa3d-c81e-4fb6-8912-3ed35b7daac9", "sourceType": "manual", "title": "text", "createdAt": "text", "taskStatus": "active", "latestRunStatus": "queued", "displayStatus": "text", "user": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text", "email": "ana@example.com", "country": "text" }, "agent": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" }, "workspace": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" }, "workspaceTag": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" }, "messageCount": 1, "userMessageCount": 1, "skillsUsed": [ "text" ], "skillsUsedDetails": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "connectorsAttached": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "metrics": { "runCount": 1, "inputFiles": 1, "outputFiles": 1, "attempts": 1, "totalTokens": 1 }, "taskViews": { "viewCount": 1, "unmeasuredReason": "no-view-key", "lastViewedAt": "text", "lastViewedBy": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text", "email": "ana@example.com", "country": "text" }, "viewedFrom": [ "task_route" ] }, "feedbackSummary": { "count": 1, "avgRating": 1.5, "maxRating": 1.5, "withTextCount": 1 }, "execution": { "kind": "traced", "sessionIds": [ "f6512788-45a7-480a-8953-a83849e71f91" ], "model": "anthropic/claude-sonnet-5", "models": [ "text" ], "callCount": 1, "tokens": 1, "durationMs": 1, "errorCount": 1, "estimatedCostUsd": 0.75, "errorMessageSample": "text", "lastActivityAt": "text", "subagents": { "count": 1, "tokens": 1, "estimatedCostUsd": 0.75, "errorCount": 1 } }, "billedCost": { "kind": "known", "costUsdMicros": 1 }, "workflows": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "origin": "evaluation-run" } ], "pageInfo": { "nextCursor": "text", "hasNextPage": true } } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "A workflow or error-message filter could not be applied because the trace store did not answer; retry, or drop that filter", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/analytics/records\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode 'limit=50'" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/analytics/records\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"limit\": 50},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/analytics/records`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"limit\", \"50\")\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/analytics/overview": { "get": { "tags": [ "Analytics" ], "summary": "Get the analytics overview", "description": "Summary cards plus chart series for the records matching the filters.", "operationId": "getAnalyticsOverview", "parameters": [ { "in": "query", "name": "agentIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`agentIds=[\"a\",\"b\"]`) or repeat the key (`agentIds=a&agentIds=b`). A single plain value (`agentIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "workspaceIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`workspaceIds=[\"a\",\"b\"]`) or repeat the key (`workspaceIds=a&workspaceIds=b`). A single plain value (`workspaceIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "workspaceTagIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`workspaceTagIds=[\"a\",\"b\"]`) or repeat the key (`workspaceTagIds=a&workspaceTagIds=b`). A single plain value (`workspaceTagIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "sourceType", "required": false, "schema": { "type": "string", "enum": [ "manual", "scheduled", "api", "internal-api", "prewarmed", "local-device", "agent-brain", "subagent" ], "description": "Started from: keep only records that started on this source (a task source type, or `subagent` for tasks started by another task)." } }, { "in": "query", "name": "usedFrom", "required": false, "schema": { "type": "array", "items": { "type": "string", "enum": [ "web", "mobile", "cli", "api" ], "description": "An interface a task was used from: `web` (the web app and the desktop app), `mobile` (the native mobile app), `cli` (the `context-code` terminal on a paired machine), or `api` (the Task API with your own key). The platform's own automation is not an interface." }, "minItems": 1, "description": "Used from: keep only tasks a person acted on through ANY of these interfaces (a turn sent from the app, a run driven by a paired machine, or where the task started). Send a JSON list in one value (`usedFrom=[\"a\",\"b\"]`) or repeat the key (`usedFrom=a&usedFrom=b`). A single plain value (`usedFrom=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "userIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`userIds=[\"a\",\"b\"]`) or repeat the key (`userIds=a&userIds=b`). A single plain value (`userIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "email", "required": false, "schema": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Case-insensitive substring matched against the owning user's email, to narrow results to one person." } }, { "in": "query", "name": "countries", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`countries=[\"a\",\"b\"]`) or repeat the key (`countries=a&countries=b`). A single plain value (`countries=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "taskOpencodeBranchId", "required": false, "schema": { "type": "string", "minLength": 1 } }, { "in": "query", "name": "skillIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`skillIds=[\"a\",\"b\"]`) or repeat the key (`skillIds=a&skillIds=b`). A single plain value (`skillIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "workflowIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1, "maxItems": 50, "description": "Workflow ids. A record matches when its chat is classified into any of them. Send a JSON list in one value (`workflowIds=[\"a\",\"b\"]`) or repeat the key (`workflowIds=a&workflowIds=b`). A single plain value (`workflowIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "errorMessage", "required": false, "schema": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Case-insensitive substring matched against the error messages of calls in the record's own chat (the same filter the app's Traces view offers). A record with no chat never matches." } }, { "in": "query", "name": "from", "required": false, "schema": { "type": "string", "format": "date-time" } }, { "in": "query", "name": "to", "required": false, "schema": { "type": "string", "format": "date-time" } }, { "in": "query", "name": "minRating", "required": false, "schema": { "type": "number", "minimum": 1, "maximum": 5 } }, { "in": "query", "name": "maxRating", "required": false, "schema": { "type": "number", "minimum": 1, "maximum": 5 } }, { "in": "query", "name": "hasFeedback", "required": false, "schema": { "type": "boolean" } }, { "in": "query", "name": "search", "required": false, "schema": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Case-insensitive substring matched against the task title (a dispatch run matches on its task's title, otherwise its prompt). A task with no title of its own matches on its latest dispatch run's title or prompt, the same fallback the list shows." } }, { "in": "query", "name": "statuses", "required": false, "schema": { "type": "array", "items": { "type": "string", "enum": [ "completed", "failed", "running", "pending", "cancelled", "archived" ], "description": "A status from the status breakdown: `completed` (Completed or Succeeded), `failed`, `running` (Queued or Running), `pending`, `cancelled`, or `archived`." }, "minItems": 1, "description": "Keep only records whose current status (as shown in the status breakdown) is ANY of these. Send a JSON list in one value (`statuses=[\"a\",\"b\"]`) or repeat the key (`statuses=a&statuses=b`). A single plain value (`statuses=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "origins", "required": false, "schema": { "type": "array", "items": { "type": "string", "enum": [ "evaluation-run", "judge", "skill-learning-replay", "production" ], "description": "What started a chat: a task in an evaluation run, a judge grading one, a skill-learning replay, or the organization's own production work." }, "minItems": 1, "description": "Keep only records started by ANY of these kinds of work: `production`, `evaluation-run`, `judge`, or `skill-learning-replay`. The origin comes from the record's root task: whether an evaluation started it, then whether it runs as the organization's judge agent. Omit to keep every origin. Send a JSON list in one value (`origins=[\"a\",\"b\"]`) or repeat the key (`origins=a&origins=b`). A single plain value (`origins=a`) is refused with 400." }, "style": "form", "explode": true } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "summary": { "type": "object", "properties": { "totalTasks": { "type": "integer", "minimum": 0, "description": "Number of top-level task records matching the filters; sub-agent tasks are excluded" }, "totalRecords": { "type": "integer", "minimum": 0 }, "totalUsers": { "type": "integer", "minimum": 0 }, "avgRating": { "type": [ "number", "null" ] }, "feedbackRate": { "type": "number", "minimum": 0 }, "totalFeedback": { "type": "integer", "minimum": 0 }, "skillsCreated": { "type": "integer", "minimum": 0, "description": "Skills created in the organization within the from/to date window (system skills excluded). Only the organization and date filters apply: a skill is not a task, so agent, user, and workspace filters do not narrow it." }, "billedCost": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "known" ] }, "costUsdMicros": { "type": "integer", "minimum": 0, "description": "Billed model spend in millionths of a US dollar, read from billing records (never recalculated)" } }, "required": [ "kind", "costUsdMicros" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "unknown" ] }, "reason": { "type": "string", "enum": [ "ledger_not_final", "device_calls_unbounded", "no_rows", "unrated_rows", "ledger_unavailable" ], "description": "Why the billed cost is unavailable. `ledger_not_final`: billing for the run is not final, or the run can still be billed for a call. `device_calls_unbounded`: a run ran on a paired machine, whose spend is never reported as final. `no_rows`: there is no billing record for it. `unrated_rows`: a billing record has no rated cost. `ledger_unavailable`: billing records could not be read." } }, "required": [ "kind", "reason" ], "additionalProperties": false } ], "description": "Billed model spend of every record matching the filters, from billing records" }, "execution": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "known" ] }, "tracedTasks": { "type": "integer", "minimum": 0, "description": "Records matching the filters whose chat has trace data" }, "tasksWithErrors": { "type": "integer", "minimum": 0, "description": "Of those, records with at least one unsuccessful call" } }, "required": [ "kind", "tracedTasks", "tasksWithErrors" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "unknown" ] }, "reason": { "type": "string", "enum": [ "too_many_tasks", "trace_store_unavailable" ], "description": "`too_many_tasks`: the filters match more chats than one trace data read covers (narrow the date window). `trace_store_unavailable`: trace data could not be read." } }, "required": [ "kind", "reason" ], "additionalProperties": false } ], "description": "Execution totals across all records matching the filters, for the Errors tile" } }, "required": [ "totalTasks", "totalRecords", "totalUsers", "avgRating", "feedbackRate", "totalFeedback", "skillsCreated", "billedCost", "execution" ], "additionalProperties": false }, "graphs": { "type": "object", "properties": { "ratingDistributionSeries": { "type": "array", "items": { "type": "object", "properties": { "rating": { "type": "integer", "minimum": 1, "maximum": 5 }, "recordCount": { "type": "integer", "minimum": 0 } }, "required": [ "rating", "recordCount" ], "additionalProperties": false, "description": "Rating bucket for the overview charts" } }, "runsOverTimeSeries": { "type": "array", "items": { "type": "object", "properties": { "date": { "type": "string", "minLength": 1, "description": "Date bucket in ISO `YYYY-MM-DD` format" }, "recordCount": { "type": "integer", "minimum": 0 } }, "required": [ "date", "recordCount" ], "additionalProperties": false, "description": "Daily record count for the overview charts" } }, "topAgentSeries": { "type": "array", "items": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1 }, "agentLabel": { "type": "string", "minLength": 1 }, "recordCount": { "type": "integer", "minimum": 0 } }, "required": [ "agentId", "agentLabel", "recordCount" ], "additionalProperties": false, "description": "Top-agent row for the overview charts" } }, "topUserSeries": { "type": "array", "items": { "type": "object", "properties": { "userId": { "type": "string", "minLength": 1 }, "userLabel": { "type": "string", "minLength": 1 }, "recordCount": { "type": "integer", "minimum": 0 } }, "required": [ "userId", "userLabel", "recordCount" ], "additionalProperties": false, "description": "Top-user row for the overview charts" } }, "statusBreakdownSeries": { "type": "array", "items": { "type": "object", "properties": { "status": { "type": "string", "minLength": 1 }, "recordCount": { "type": "integer", "minimum": 0 } }, "required": [ "status", "recordCount" ], "additionalProperties": false, "description": "Status row for the overview charts" } }, "countryBreakdownSeries": { "type": "array", "items": { "type": "object", "properties": { "countryCode": { "type": [ "string", "null" ], "minLength": 1 }, "recordCount": { "type": "integer", "minimum": 0 } }, "required": [ "countryCode", "recordCount" ], "additionalProperties": false, "description": "Country row for the overview charts" } }, "topSkillSeries": { "type": "array", "items": { "type": "object", "properties": { "skillName": { "type": "string", "minLength": 1 }, "recordCount": { "type": "integer", "minimum": 0 } }, "required": [ "skillName", "recordCount" ], "additionalProperties": false, "description": "Top-skill row for the overview charts" } }, "skillsCreatedSeries": { "type": "array", "items": { "type": "object", "properties": { "creatorId": { "type": "string", "description": "ID of the creator: a user ID, or the agent ID for skills an agent wrote; empty when no creator was recorded" }, "creatorType": { "type": "string", "enum": [ "user", "agent" ], "description": "Whether the creator ID is a user or an agent" }, "creatorLabel": { "type": "string", "minLength": 1, "description": "Creator display name, or the raw ID when the creator no longer exists" }, "skillCount": { "type": "integer", "minimum": 0 } }, "required": [ "creatorId", "creatorType", "creatorLabel", "skillCount" ], "additionalProperties": false, "description": "Skills-created-by-creator row for the overview" }, "description": "Top skill creators in the window, by skill count; same scope as `summary.skillsCreated`" }, "costByModelSeries": { "type": "array", "items": { "type": "object", "properties": { "modelGroup": { "type": "string", "minLength": 1, "description": "The model that served the calls (the model name as configured on the deployment)" }, "costUsdMicros": { "type": "integer", "minimum": 0, "description": "Billed spend for those calls, in millionths of a US dollar, from billing records" } }, "required": [ "modelGroup", "costUsdMicros" ], "additionalProperties": false, "description": "Billed-cost-by-model row for the task charts" }, "description": "Billed spend of the filtered records by model, from billing records; empty when billing is not final or cannot be read (see `summary.billedCost`)" } }, "required": [ "ratingDistributionSeries", "runsOverTimeSeries", "topAgentSeries", "topUserSeries", "statusBreakdownSeries", "countryBreakdownSeries", "topSkillSeries", "skillsCreatedSeries", "costByModelSeries" ], "additionalProperties": false, "description": "All overview chart series for the full set of records matching the filters" } }, "required": [ "summary", "graphs" ], "additionalProperties": false, "description": "Overview cards plus chart series for the full set of records matching the filters" }, "example": { "summary": { "totalTasks": 1, "totalRecords": 1, "totalUsers": 1, "avgRating": 0.75, "feedbackRate": 0.75, "totalFeedback": 1, "skillsCreated": 1, "billedCost": { "kind": "known", "costUsdMicros": 1 }, "execution": { "kind": "known", "tracedTasks": 1, "tasksWithErrors": 1 } }, "graphs": { "ratingDistributionSeries": [ { "rating": 1, "recordCount": 1 } ], "runsOverTimeSeries": [ { "date": "text", "recordCount": 1 } ], "topAgentSeries": [ { "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "agentLabel": "text", "recordCount": 1 } ], "topUserSeries": [ { "userId": "6ccb2121-4ffd-40b0-8c2c-1607cf6a05be", "userLabel": "text", "recordCount": 1 } ], "statusBreakdownSeries": [ { "status": "text", "recordCount": 1 } ], "countryBreakdownSeries": [ { "countryCode": "text", "recordCount": 1 } ], "topSkillSeries": [ { "skillName": "text", "recordCount": 1 } ], "skillsCreatedSeries": [ { "creatorId": "20c55b5b-17e8-46f6-8084-50803c010949", "creatorType": "user", "creatorLabel": "text", "skillCount": 1 } ], "costByModelSeries": [ { "modelGroup": "text", "costUsdMicros": 1 } ] } } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "A workflow or error-message filter could not be applied because the trace store did not answer; retry, or drop that filter", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/analytics/overview\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/analytics/overview\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/analytics/overview`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/analytics/options": { "get": { "tags": [ "Analytics" ], "summary": "List analytics filter values", "description": "Values for the analytics filters.", "operationId": "getAnalyticsOptions", "parameters": [], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "agents": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false } }, "sources": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "description": "Started from: the sources present in the organization's records, with labels" }, "interfaces": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "description": "Used from: every interface a task can be used from, with labels, in the same order as the export's `usedFrom`. This list is fixed: it includes every interface whether or not the organization has records from it." }, "users": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 }, "email": { "type": "string", "format": "email" }, "country": { "type": [ "string", "null" ] } }, "required": [ "id", "label", "email" ], "additionalProperties": false } }, "countries": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false } }, "workspaces": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "description": "First page of the organization's workspaces, ordered by name and capped (see `workspacesTruncated`). To find others, search for workspaces in the app." }, "workspacesTruncated": { "type": "boolean", "description": "True when the organization has more workspaces than this list holds, so the list is incomplete." }, "workspaceTags": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false } }, "skills": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false } } }, "required": [ "agents", "sources", "interfaces", "users", "countries", "workspaces", "workspacesTruncated", "workspaceTags", "skills" ], "additionalProperties": false }, "example": { "agents": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "sources": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "interfaces": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "users": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text", "email": "ana@example.com", "country": "text" } ], "countries": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "workspaces": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "workspacesTruncated": true, "workspaceTags": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "skills": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ] } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/analytics/options\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/analytics/options\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/analytics/options`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/analytics/records/export": { "get": { "tags": [ "Analytics" ], "summary": "Export analytics records", "description": "Streams every analytics record matching the filters as line-delimited JSON (`application/x-ndjson`). When `includeFeedback=true` (default), each row has a `feedback` array with one entry per scorecard, holding aggregated criterion responses and free-text comments. Numeric criteria become their average, boolean criteria combine to Yes/No, and multi-valued text criteria are comma-joined. Comments are newline-joined per scorecard in submission order. The app's CSV download uses the same data, with one column per scorecard and criterion pair plus 'Feedback Comments: ' columns. Use `limit` to fetch a sample; omit it to stream the full filtered set.", "operationId": "exportAnalyticsRecords", "parameters": [ { "in": "query", "name": "agentIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`agentIds=[\"a\",\"b\"]`) or repeat the key (`agentIds=a&agentIds=b`). A single plain value (`agentIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "workspaceIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`workspaceIds=[\"a\",\"b\"]`) or repeat the key (`workspaceIds=a&workspaceIds=b`). A single plain value (`workspaceIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "workspaceTagIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`workspaceTagIds=[\"a\",\"b\"]`) or repeat the key (`workspaceTagIds=a&workspaceTagIds=b`). A single plain value (`workspaceTagIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "sourceType", "required": false, "schema": { "type": "string", "enum": [ "manual", "scheduled", "api", "internal-api", "prewarmed", "local-device", "agent-brain", "subagent" ], "description": "Started from: keep only records that started on this source (a task source type, or `subagent` for tasks started by another task)." } }, { "in": "query", "name": "usedFrom", "required": false, "schema": { "type": "array", "items": { "type": "string", "enum": [ "web", "mobile", "cli", "api" ], "description": "An interface a task was used from: `web` (the web app and the desktop app), `mobile` (the native mobile app), `cli` (the `context-code` terminal on a paired machine), or `api` (the Task API with your own key). The platform's own automation is not an interface." }, "minItems": 1, "description": "Used from: keep only tasks a person acted on through ANY of these interfaces (a turn sent from the app, a run driven by a paired machine, or where the task started). Send a JSON list in one value (`usedFrom=[\"a\",\"b\"]`) or repeat the key (`usedFrom=a&usedFrom=b`). A single plain value (`usedFrom=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "userIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`userIds=[\"a\",\"b\"]`) or repeat the key (`userIds=a&userIds=b`). A single plain value (`userIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "email", "required": false, "schema": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Case-insensitive substring matched against the owning user's email, to narrow results to one person." } }, { "in": "query", "name": "countries", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`countries=[\"a\",\"b\"]`) or repeat the key (`countries=a&countries=b`). A single plain value (`countries=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "taskOpencodeBranchId", "required": false, "schema": { "type": "string", "minLength": 1 } }, { "in": "query", "name": "skillIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Send a JSON list in one value (`skillIds=[\"a\",\"b\"]`) or repeat the key (`skillIds=a&skillIds=b`). A single plain value (`skillIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "workflowIds", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1, "maxItems": 50, "description": "Workflow ids. A record matches when its chat is classified into any of them. Send a JSON list in one value (`workflowIds=[\"a\",\"b\"]`) or repeat the key (`workflowIds=a&workflowIds=b`). A single plain value (`workflowIds=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "errorMessage", "required": false, "schema": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Case-insensitive substring matched against the error messages of calls in the record's own chat (the same filter the app's Traces view offers). A record with no chat never matches." } }, { "in": "query", "name": "from", "required": false, "schema": { "type": "string", "format": "date-time" } }, { "in": "query", "name": "to", "required": false, "schema": { "type": "string", "format": "date-time" } }, { "in": "query", "name": "minRating", "required": false, "schema": { "type": "number", "minimum": 1, "maximum": 5 } }, { "in": "query", "name": "maxRating", "required": false, "schema": { "type": "number", "minimum": 1, "maximum": 5 } }, { "in": "query", "name": "hasFeedback", "required": false, "schema": { "type": "boolean" } }, { "in": "query", "name": "search", "required": false, "schema": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Case-insensitive substring matched against the task title (a dispatch run matches on its task's title, otherwise its prompt). A task with no title of its own matches on its latest dispatch run's title or prompt, the same fallback the list shows." } }, { "in": "query", "name": "statuses", "required": false, "schema": { "type": "array", "items": { "type": "string", "enum": [ "completed", "failed", "running", "pending", "cancelled", "archived" ], "description": "A status from the status breakdown: `completed` (Completed or Succeeded), `failed`, `running` (Queued or Running), `pending`, `cancelled`, or `archived`." }, "minItems": 1, "description": "Keep only records whose current status (as shown in the status breakdown) is ANY of these. Send a JSON list in one value (`statuses=[\"a\",\"b\"]`) or repeat the key (`statuses=a&statuses=b`). A single plain value (`statuses=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "origins", "required": false, "schema": { "type": "array", "items": { "type": "string", "enum": [ "evaluation-run", "judge", "skill-learning-replay", "production" ], "description": "What started a chat: a task in an evaluation run, a judge grading one, a skill-learning replay, or the organization's own production work." }, "minItems": 1, "description": "Keep only records started by ANY of these kinds of work: `production`, `evaluation-run`, `judge`, or `skill-learning-replay`. The origin comes from the record's root task: whether an evaluation started it, then whether it runs as the organization's judge agent. Omit to keep every origin. Send a JSON list in one value (`origins=[\"a\",\"b\"]`) or repeat the key (`origins=a&origins=b`). A single plain value (`origins=a`) is refused with 400." }, "style": "form", "explode": true }, { "in": "query", "name": "includeFeedback", "required": false, "schema": { "type": "boolean", "default": true, "description": "When true (default), each row includes aggregated feedback per scorecard in the `feedback` array. When false, the `feedback` array is always empty and the export skips feedback work." } }, { "in": "query", "name": "limit", "required": false, "schema": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Optional row limit, useful for fetching a small sample. When omitted, every matching record is streamed." } }, { "in": "query", "name": "columns", "required": false, "schema": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Optional list of CSV column headers to include. Applies only to the app's CSV download; the JSONL stream always includes every key. When omitted, all columns are included. Send a JSON list in one value (`columns=[\"a\",\"b\"]`) or repeat the key (`columns=a&columns=b`). A single plain value (`columns=a`) is refused with 400." }, "style": "form", "explode": true } ], "responses": { "200": { "description": "Line-delimited JSON: one `AnalyticsExportJsonRow` per line, each ending in a newline.", "content": { "application/x-ndjson": { "schema": { "type": "object", "properties": { "recordId": { "type": "string", "minLength": 1, "description": "Stable analytics record identifier; always present, so exported rows can be matched back to their records" }, "recordType": { "type": "string", "enum": [ "rootTask", "subAgent", "dispatchRun" ], "description": "Kind of record: `rootTask`, `subAgent`, or `dispatchRun`" }, "taskId": { "type": [ "string", "null" ], "description": "Task ID when the record is a task; null for standalone dispatch runs" }, "parentTaskId": { "type": [ "string", "null" ], "description": "Parent task ID when this row is a sub-agent task; null for root tasks and standalone dispatch runs" }, "runId": { "type": [ "string", "null" ], "description": "Run ID when the record is a dispatch run; null for tasks" }, "taskOpencodeBranchId": { "type": [ "string", "null" ], "description": "Chat ID when available; the preferred identifier for building project URLs" }, "sourceType": { "type": "string", "enum": [ "manual", "scheduled", "api", "internal-api", "prewarmed", "local-device", "agent-brain", "subagent" ], "description": "Started from: the one source the record started on. The CSV shows it as a label (`manual` → Web, `local-device` → CLI, `internal-api` → Automation)." }, "usedFrom": { "type": "array", "items": { "type": "string", "enum": [ "web", "mobile", "cli", "api" ], "description": "An interface a task was used from: `web` (the web app and the desktop app), `mobile` (the native mobile app), `cli` (the `context-code` terminal on a paired machine), or `api` (the Task API with your own key). The platform's own automation is not an interface." }, "description": "Used from: every interface a person acted on this record through, in a fixed order (`web`, `mobile`, `cli`, `api`), based on the record's turns, its runs on paired machines, and where it started. Empty for records no interface touched (scheduled, started ahead of use, or a sub-agent with no turns of its own). `api` means started through the Task API with your own key. The platform's own automation (the scheduler, applets, judges, replays) shows as Started from: Automation, with no interface. Known limits: a task started with `POST /api/v1/tasks/prompt` using a service-account key reads as automation, and a task later continued through the API is not detected. Replaces the removed `interfacesUsed` map." }, "taskApiKeyName": { "type": [ "string", "null" ], "description": "Name of the task API key that started this task, taken from the task's first dispatch (later continuations are ignored). Null for tasks and runs not started with a key (for example a task started in the web app, or an automated flow that uses no key, such as an agent judge). When the scheduler started the task, this is `internal-scheduler`." }, "taskApiKeyPrincipalType": { "type": [ "string", "null" ], "enum": [ "user", "agentServiceAccount", "applet", "service", null ], "description": "Type of account behind the originating key (same dispatch as `taskApiKeyName`): `user` for a person's own key, `agentServiceAccount` for a service key the platform created. Null when the record was not started with a key." }, "taskApiKeyNamespace": { "type": [ "string", "null" ], "description": "Namespace of the originating key (same dispatch as `taskApiKeyName`). A reserved namespace means an automated flow, not a person's key, dispatched the record. Null when the record was not started with a key." }, "invokedBy": { "type": "string", "minLength": 1, "description": "Who invoked the record: the dispatching key's name for records started by a service key or by platform automation (or the record's agent for automated flows that use no key, such as an agent judge); otherwise the owning user's display name." }, "title": { "type": "string", "minLength": 1, "description": "Readable title of the task or run" }, "createdAt": { "type": "string", "description": "Creation time in ISO 8601; the CSV formats it for display" }, "taskStatus": { "type": [ "string", "null" ], "enum": [ "active", "completed", "failed", "archived", null ] }, "latestRunStatus": { "type": [ "string", "null" ], "enum": [ "queued", "running", "succeeded", "failed", "cancelled", null ] }, "displayStatus": { "type": "string", "minLength": 1, "description": "Status label combining the task status and its latest run's status" }, "user": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 }, "email": { "type": "string", "format": "email" }, "country": { "type": [ "string", "null" ] } }, "required": [ "id", "label", "email" ], "additionalProperties": false }, "agent": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "workspace": { "type": [ "object", "null" ], "properties": { "id": { "type": "string", "minLength": 1 }, "name": { "type": "string", "minLength": 1 } }, "required": [ "id", "name" ], "additionalProperties": false }, "workspaceTag": { "anyOf": [ { "allOf": [ { "type": [ "object", "null" ], "properties": { "id": { "type": "string", "minLength": 1 }, "name": { "type": "string", "minLength": 1 } }, "required": [ "id", "name" ], "additionalProperties": false } ] }, { "type": "null" } ] }, "messageCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Number of archived messages; null until it is computed" }, "userMessageCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Number of archived user-written messages; null until it is computed. The CSV shows null as an empty cell, not 0." }, "skillsUsed": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "description": "Skills the record used. The CSV shows a comma-separated list of labels." }, "skillsCreated": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "description": "Skills this record's task created. Always empty for dispatch-run records (they have no task). The CSV shows a comma-separated list of labels." }, "metrics": { "type": "object", "properties": { "runCount": { "type": "integer", "minimum": 0 }, "inputFiles": { "type": "integer", "minimum": 0 }, "outputFiles": { "type": "integer", "minimum": 0 }, "attempts": { "type": "integer", "minimum": 0 }, "totalTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Billed tokens (input + output + reasoning + cache write + cache read) summed across the record's LLM calls in trace data. Null when the record has no chat, there is no trace data for it, or trace data cannot be read." } }, "required": [ "runCount", "inputFiles", "outputFiles", "attempts", "totalTokens" ], "additionalProperties": false }, "tokenUsage": { "type": [ "object", "null" ], "properties": { "input": { "type": "integer", "minimum": 0, "description": "Billed prompt tokens not served from cache, summed across the record's LLM calls" }, "output": { "type": "integer", "minimum": 0, "description": "Completion tokens, not counting reasoning, summed across the record's LLM calls" }, "reasoning": { "type": "integer", "minimum": 0, "description": "Reasoning (thinking) tokens, summed across the record's LLM calls" }, "cacheRead": { "type": "integer", "minimum": 0, "description": "Prompt tokens served from the provider cache, summed across the record's LLM calls" }, "cacheWrite": { "type": "integer", "minimum": 0, "description": "Prompt tokens written to the provider cache, summed across the record's LLM calls" } }, "required": [ "input", "output", "reasoning", "cacheRead", "cacheWrite" ], "additionalProperties": false, "description": "Billed tokens split by type; the parts add up to `metrics.totalTokens`. Null (not 0) when no usage was reported." }, "taskViews": { "type": "object", "properties": { "viewCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Recorded opens of this task's conversation across every surface that records views. Null (not 0) when views were never measured for this record (see `unmeasuredReason`)." }, "unmeasuredReason": { "type": [ "string", "null" ], "enum": [ "no-view-key", "predates-instrumentation", null ], "description": "Why `viewCount` is null. Null whenever `viewCount` is a real measurement, including a real zero." }, "lastViewedAt": { "type": [ "string", "null" ] }, "lastViewedBy": { "anyOf": [ { "allOf": [ { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 }, "email": { "type": "string", "format": "email" }, "country": { "type": [ "string", "null" ] } }, "required": [ "id", "label", "email" ], "additionalProperties": false } ] }, { "type": "null" } ] }, "viewedFrom": { "type": "array", "items": { "type": "string", "enum": [ "task_route", "shared_link", "eval_analytics", "context_code", "eval_project" ] }, "description": "Distinct surfaces where a person opened this record's conversation, in the enum's order. Empty when `viewCount` is null or 0: a surface appears only when at least one view was recorded on it." } }, "required": [ "viewCount", "unmeasuredReason", "lastViewedAt", "lastViewedBy", "viewedFrom" ], "additionalProperties": false, "description": "View summary. Tasks match views by task ID; standalone dispatch runs match by chat. `viewCount` is null (with `unmeasuredReason` set) when the record could not be measured: a run with no chat, or a record created before views were recorded. The CSV shows that as an empty cell, not 0." }, "feedbackSummary": { "type": "object", "properties": { "count": { "type": "integer", "minimum": 0, "description": "Number of feedback submissions on this record" }, "avgRating": { "type": [ "number", "null" ], "minimum": 1, "maximum": 5, "description": "Mean numeric rating on the 1 to 5 scale, or null when there is no numeric rating" }, "maxRating": { "type": [ "number", "null" ], "minimum": 1, "maximum": 5, "description": "Highest numeric rating on the 1 to 5 scale, or null when there is no numeric rating" }, "withTextCount": { "type": "integer", "minimum": 0, "description": "Number of submissions with a non-empty comment" } }, "required": [ "count", "avgRating", "maxRating", "withTextCount" ], "additionalProperties": false, "description": "Feedback counts and ratings across all feedback on this record. Always present." }, "projectUrl": { "type": [ "string", "null" ], "format": "uri", "description": "Absolute URL of the project or chat in the app; null when the record has no chat to link to or the deployment has no app address configured" }, "isEditorShared": { "type": [ "boolean", "null" ], "description": "Whether the project has an active editor-mode share. Null when the record has no shareable root task or the share state could not be read." }, "feedback": { "type": "array", "items": { "type": "object", "properties": { "scorecardId": { "type": "string", "minLength": 1, "description": "Stable ID of the scorecard the feedback came from" }, "scorecardName": { "type": "string", "minLength": 1, "description": "Scorecard display name; used as the column-name prefix in the CSV (for example 'Feedback: - ')" }, "criteria": { "type": "object", "additionalProperties": { "type": "string" }, "description": "Aggregated criterion responses by criterion name. Numeric criteria become their average, boolean criteria combine to Yes/No, and free-text responses are comma-joined." }, "comments": { "type": "string", "description": "Free-text comments under this scorecard, joined by newlines in submission order. Empty when there are no comments." } }, "required": [ "scorecardId", "scorecardName", "criteria", "comments" ], "additionalProperties": false, "description": "All feedback from one scorecard on one record, combined into one entry. A record with feedback from several scorecards has several entries." }, "description": "Aggregated feedback for this record, one entry per scorecard. Empty when the record has no feedback or the request set `includeFeedback=false`." } }, "required": [ "recordId", "recordType", "taskId", "parentTaskId", "runId", "taskOpencodeBranchId", "sourceType", "usedFrom", "taskApiKeyName", "taskApiKeyPrincipalType", "taskApiKeyNamespace", "invokedBy", "title", "createdAt", "taskStatus", "latestRunStatus", "displayStatus", "user", "agent", "workspace", "workspaceTag", "messageCount", "userMessageCount", "skillsUsed", "skillsCreated", "metrics", "tokenUsage", "taskViews", "feedbackSummary", "projectUrl", "isEditorShared", "feedback" ], "additionalProperties": false, "description": "One JSONL row from the export. Converts cleanly to CSV with flat 'Feedback: - ' and 'Feedback Comments: ' columns." }, "example": { "recordId": "76061c75-9c20-4694-8584-4f20270a243d", "recordType": "rootTask", "taskId": "de707028-6ea2-4616-8864-c2ebd92b87f5", "parentTaskId": "2ad315e1-e215-4b88-8ba7-9869e414af3d", "runId": "599e5591-3334-4d0a-8bb9-069b278d9b20", "taskOpencodeBranchId": "80f2aa3d-c81e-4fb6-8912-3ed35b7daac9", "sourceType": "manual", "usedFrom": [ "web" ], "taskApiKeyName": "text", "taskApiKeyPrincipalType": "user", "taskApiKeyNamespace": "text", "invokedBy": "text", "title": "text", "createdAt": "text", "taskStatus": "active", "latestRunStatus": "queued", "displayStatus": "text", "user": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text", "email": "ana@example.com", "country": "text" }, "agent": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" }, "workspace": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" }, "workspaceTag": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" }, "messageCount": 1, "userMessageCount": 1, "skillsUsed": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "skillsCreated": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "metrics": { "runCount": 1, "inputFiles": 1, "outputFiles": 1, "attempts": 1, "totalTokens": 1 }, "tokenUsage": { "input": 1, "output": 1, "reasoning": 1, "cacheRead": 1, "cacheWrite": 1 }, "taskViews": { "viewCount": 1, "unmeasuredReason": "no-view-key", "lastViewedAt": "text", "lastViewedBy": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text", "email": "ana@example.com", "country": "text" }, "viewedFrom": [ "task_route" ] }, "feedbackSummary": { "count": 1, "avgRating": 1.5, "maxRating": 1.5, "withTextCount": 1 }, "projectUrl": "https://example.com/file", "isEditorShared": true, "feedback": [ { "scorecardId": "50ada941-25ed-4b68-880b-6016b9e07ccd", "scorecardName": "text", "criteria": {}, "comments": "text" } ] } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "A workflow or error-message filter could not be applied because the trace store did not answer; retry, or drop that filter", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/analytics/records/export\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode 'limit=100'" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/analytics/records/export\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"limit\": 100},\n stream=True,\n timeout=60,\n)\nresp.raise_for_status()\nfor line in resp.iter_lines():\n if line:\n print(json.loads(line))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/analytics/records/export`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"limit\", \"100\")\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nfor (const line of (await res.text()).split(\"\\n\")) {\n if (line) console.log(JSON.parse(line))\n}" } ] } }, "/analytics/records/{recordId}": { "get": { "tags": [ "Analytics" ], "summary": "Get an analytics record", "description": "One analytics record plus its latest run and feedback entries.", "operationId": "getAnalyticsRecord", "parameters": [ { "in": "path", "name": "recordId", "required": true, "description": "Stable analytics record identifier", "schema": { "type": "string", "minLength": 1, "description": "Stable analytics record identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "record": { "type": [ "object", "null" ], "properties": { "recordId": { "type": "string", "minLength": 1 }, "recordKind": { "type": "string", "enum": [ "task", "dispatchRun" ] }, "recordType": { "type": "string", "enum": [ "rootTask", "subAgent", "dispatchRun" ], "description": "Kind of record: `rootTask`, `subAgent`, or `dispatchRun`." }, "taskId": { "type": [ "string", "null" ] }, "parentTaskId": { "type": [ "string", "null" ], "description": "Parent task ID when this row is a sub-agent task; null for root tasks and standalone dispatch runs." }, "runId": { "type": [ "string", "null" ] }, "taskOpencodeBranchId": { "type": [ "string", "null" ] }, "sourceType": { "type": "string", "enum": [ "manual", "scheduled", "api", "internal-api", "prewarmed", "local-device", "agent-brain", "subagent" ], "description": "Where the record started: one of the task source types, or `subagent` for child tasks started by another task." }, "title": { "type": "string", "minLength": 1 }, "createdAt": { "type": "string" }, "taskStatus": { "type": [ "string", "null" ], "enum": [ "active", "completed", "failed", "archived", null ] }, "latestRunStatus": { "type": [ "string", "null" ], "enum": [ "queued", "running", "succeeded", "failed", "cancelled", null ] }, "displayStatus": { "type": "string", "minLength": 1 }, "user": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 }, "email": { "type": "string", "format": "email" }, "country": { "type": [ "string", "null" ] } }, "required": [ "id", "label", "email" ], "additionalProperties": false }, "agent": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "workspace": { "type": [ "object", "null" ], "properties": { "id": { "type": "string", "minLength": 1 }, "name": { "type": "string", "minLength": 1 } }, "required": [ "id", "name" ], "additionalProperties": false }, "workspaceTag": { "anyOf": [ { "allOf": [ { "type": [ "object", "null" ], "properties": { "id": { "type": "string", "minLength": 1 }, "name": { "type": "string", "minLength": 1 } }, "required": [ "id", "name" ], "additionalProperties": false } ] }, { "type": "null" } ] }, "messageCount": { "type": [ "integer", "null" ], "minimum": 0 }, "userMessageCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Number of user-written messages in the task's latest archived conversation; null (not 0) when the task has no archived conversation." }, "skillsUsed": { "type": [ "array", "null" ], "items": { "type": "string" } }, "skillsUsedDetails": { "type": [ "array", "null" ], "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false } }, "connectorsAttached": { "type": [ "array", "null" ], "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "description": "MCP connectors attached to the record's agent (including the organization's custom connectors on the agent's list or covered by its All connectors setting), as connector names sorted by label; null when the agent has none. Attachments are not recorded per run, so this shows the agent's connectors at the time you read it, not when the record ran." }, "metrics": { "type": "object", "properties": { "runCount": { "type": "integer", "minimum": 0 }, "inputFiles": { "type": "integer", "minimum": 0 }, "outputFiles": { "type": "integer", "minimum": 0 }, "attempts": { "type": "integer", "minimum": 0 }, "totalTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Billed tokens (input + output + reasoning + cache write + cache read) summed across the record's LLM calls in trace data. Null when the record has no chat, there is no trace data for it, or trace data cannot be read." } }, "required": [ "runCount", "inputFiles", "outputFiles", "attempts", "totalTokens" ], "additionalProperties": false }, "taskViews": { "type": "object", "properties": { "viewCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Recorded opens of this task's conversation across every surface that records views. Null (not 0) when views were never measured for this record (see `unmeasuredReason`)." }, "unmeasuredReason": { "type": [ "string", "null" ], "enum": [ "no-view-key", "predates-instrumentation", null ], "description": "Why `viewCount` is null. Null whenever `viewCount` is a real measurement, including a real zero." }, "lastViewedAt": { "type": [ "string", "null" ] }, "lastViewedBy": { "anyOf": [ { "allOf": [ { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 }, "email": { "type": "string", "format": "email" }, "country": { "type": [ "string", "null" ] } }, "required": [ "id", "label", "email" ], "additionalProperties": false } ] }, { "type": "null" } ] }, "viewedFrom": { "type": "array", "items": { "type": "string", "enum": [ "task_route", "shared_link", "eval_analytics", "context_code", "eval_project" ] }, "description": "Distinct surfaces where a person opened this record's conversation, in the enum's order. Empty when `viewCount` is null or 0: a surface appears only when at least one view was recorded on it." } }, "required": [ "viewCount", "unmeasuredReason", "lastViewedAt", "lastViewedBy", "viewedFrom" ], "additionalProperties": false, "description": "View measurement for an analytics record (a task, or a standalone dispatch run matched by its chat)." }, "feedbackSummary": { "type": "object", "properties": { "count": { "type": "integer", "minimum": 0, "description": "Number of feedback submissions on this record" }, "avgRating": { "type": [ "number", "null" ], "minimum": 1, "maximum": 5, "description": "Mean numeric rating on the 1 to 5 scale, or null when there is no numeric rating" }, "maxRating": { "type": [ "number", "null" ], "minimum": 1, "maximum": 5, "description": "Highest numeric rating on the 1 to 5 scale, or null when there is no numeric rating" }, "withTextCount": { "type": "integer", "minimum": 0, "description": "Number of submissions with a non-empty comment" } }, "required": [ "count", "avgRating", "maxRating", "withTextCount" ], "additionalProperties": false, "description": "Feedback summary for one task, run, or trace session" }, "execution": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "traced" ] }, "sessionIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1, "description": "The chat's trace sessions, latest first." }, "model": { "type": [ "string", "null" ], "minLength": 1, "description": "The model that served the most calls in the chat; null when no call named a model" }, "models": { "type": "array", "items": { "type": "string", "minLength": 1 }, "description": "Every named model in the chat, most calls first; empty when no call named a model" }, "callCount": { "type": "integer", "minimum": 0 }, "tokens": { "type": "integer", "minimum": 0, "description": "Billed tokens (input + output + reasoning + cache write + cache read) across the chat's own calls" }, "durationMs": { "type": "integer", "minimum": 0, "description": "Total duration of the chat's own calls, in milliseconds" }, "errorCount": { "type": "integer", "minimum": 0, "description": "Number of calls that ended unsuccessfully" }, "estimatedCostUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "Estimated cost summed over the chat's calls. An estimate, not the bill: the billed amount is `billedCost`. Null when no call reported a cost." }, "errorMessageSample": { "type": [ "string", "null" ], "minLength": 1, "description": "The error message of one failed call (aborted calls excluded); null when no failed call had one" }, "lastActivityAt": { "type": "string", "description": "Time of the chat's latest call (ISO 8601)" }, "subagents": { "type": [ "object", "null" ], "properties": { "count": { "type": "integer", "exclusiveMinimum": 0, "description": "Number of sub-agent rows on this page grouped under the parent" }, "tokens": { "type": "integer", "minimum": 0, "description": "Billed tokens of the grouped sub-agent chats" }, "estimatedCostUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "Estimated cost of the grouped sub-agent chats; null when none of them reported one" }, "errorCount": { "type": "integer", "minimum": 0, "description": "Number of unsuccessful calls in the grouped sub-agent chats" } }, "required": [ "count", "tokens", "estimatedCostUsd", "errorCount" ], "additionalProperties": false, "description": "Totals for the grouped sub-agent rows; null when the row has no sub-agent rows on this page" } }, "required": [ "kind", "sessionIds", "model", "models", "callCount", "tokens", "durationMs", "errorCount", "estimatedCostUsd", "errorMessageSample", "lastActivityAt", "subagents" ], "additionalProperties": false, "description": "Execution data is available: the chat has trace data." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "none" ] }, "reason": { "type": "string", "enum": [ "no_chat_id", "no_trace_rows", "trace_store_unavailable" ] } }, "required": [ "kind", "reason" ], "additionalProperties": false, "description": "The row has no execution data; `reason` says why." } ], "description": "Execution data for a task row, taken from the chat's trace data" }, "billedCost": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "known" ] }, "costUsdMicros": { "type": "integer", "minimum": 0, "description": "Billed model spend in millionths of a US dollar, read from billing records (never recalculated)" } }, "required": [ "kind", "costUsdMicros" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "unknown" ] }, "reason": { "type": "string", "enum": [ "ledger_not_final", "device_calls_unbounded", "no_rows", "unrated_rows", "ledger_unavailable" ], "description": "Why the billed cost is unavailable. `ledger_not_final`: billing for the run is not final, or the run can still be billed for a call. `device_calls_unbounded`: a run ran on a paired machine, whose spend is never reported as final. `no_rows`: there is no billing record for it. `unrated_rows`: a billing record has no rated cost. `ledger_unavailable`: billing records could not be read." } }, "required": [ "kind", "reason" ], "additionalProperties": false } ], "description": "The billed cost of a task row, the same figure Settings › Usage shows" }, "workflows": { "type": [ "array", "null" ], "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "label": { "type": "string", "minLength": 1 } }, "required": [ "id", "label" ], "additionalProperties": false }, "description": "Workflows the record's chat is classified into. `[]` when the chat is unclassified; null when classifications could not be read." }, "origin": { "type": "string", "enum": [ "evaluation-run", "judge", "skill-learning-replay", "production" ], "description": "The kind of work that started the record, taken from its chat's root task: whether an evaluation started it, then whether it runs as the organization's judge agent. A sub-agent takes its root's origin. A dispatch run whose chat has no task is judged on its own. This is the same rule the `origins` filter uses." } }, "required": [ "recordId", "recordKind", "recordType", "taskId", "parentTaskId", "runId", "taskOpencodeBranchId", "sourceType", "title", "createdAt", "taskStatus", "latestRunStatus", "displayStatus", "user", "agent", "workspace", "workspaceTag", "messageCount", "userMessageCount", "skillsUsed", "skillsUsedDetails", "connectorsAttached", "metrics", "taskViews", "feedbackSummary", "execution", "billedCost", "workflows", "origin" ], "additionalProperties": false, "description": "The record; null (with a 200) when the ID is unknown or outside what the key can read." }, "latestRun": { "type": [ "object", "null" ], "properties": { "runId": { "type": "string", "minLength": 1 }, "sessionId": { "type": [ "string", "null" ] }, "taskOpencodeBranchId": { "type": [ "string", "null" ] }, "inputPrompt": { "type": "string", "minLength": 1 }, "outputText": { "type": [ "string", "null" ] }, "status": { "type": "string", "enum": [ "queued", "running", "succeeded", "failed", "cancelled" ] }, "createdAt": { "type": "string" }, "completedAt": { "type": [ "string", "null" ] }, "metrics": { "type": "object", "properties": { "inputFiles": { "type": "integer", "minimum": 0 }, "outputFiles": { "type": "integer", "minimum": 0 }, "attempts": { "type": "integer", "minimum": 0 } }, "required": [ "inputFiles", "outputFiles", "attempts" ], "additionalProperties": false } }, "required": [ "runId", "sessionId", "taskOpencodeBranchId", "inputPrompt", "outputText", "status", "createdAt", "completedAt", "metrics" ], "additionalProperties": false }, "feedback": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string" }, "organizationId": { "type": "string" }, "valinorProjectId": { "type": "string" }, "chatId": { "type": [ "string", "null" ] }, "valinorMessageId": { "type": [ "string", "null" ] }, "selectedText": { "type": [ "string", "null" ] }, "userId": { "type": [ "string", "null" ] }, "teamId": { "type": [ "string", "null" ] }, "feedbackType": { "type": "string" }, "responses": {}, "createdAt": { "type": "string" }, "rubric": { "type": [ "object", "null" ], "properties": { "id": { "type": "string" }, "name": { "type": "string" }, "description": { "type": [ "string", "null" ] } }, "required": [ "id", "name", "description" ], "additionalProperties": false }, "rubricVersion": { "type": [ "object", "null" ], "properties": { "id": { "type": "string" }, "version": { "type": "number" }, "snapshot": {} }, "required": [ "id", "version" ], "additionalProperties": false }, "submitter": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "user" ] }, "id": { "type": "string", "description": "The person's user ID" }, "name": { "type": "string", "description": "The person's display name" }, "email": { "type": "string", "description": "The person's email" } }, "required": [ "kind", "id", "name", "email" ], "additionalProperties": false, "description": "A person submitted the rating, in the app or with a task API key they own" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "serviceAccount" ] }, "id": { "type": "string", "description": "The agent service account's ID" }, "name": { "type": "string", "description": "The agent's name" } }, "required": [ "kind", "id", "name" ], "additionalProperties": false, "description": "An agent service account submitted the rating (through the agent's own tools, or a task API key the agent created)" }, { "type": "null" } ] }, "target": { "type": [ "object", "null" ], "properties": { "taskId": { "type": [ "string", "null" ] }, "taskTitle": { "type": [ "string", "null" ] }, "chatId": { "type": [ "string", "null" ] }, "messageText": { "type": [ "string", "null" ] } }, "required": [ "taskId", "taskTitle", "chatId", "messageText" ], "additionalProperties": false } }, "required": [ "id", "organizationId", "valinorProjectId", "valinorMessageId", "userId", "teamId", "feedbackType", "createdAt", "rubric", "rubricVersion" ], "additionalProperties": false } } }, "required": [ "record", "latestRun", "feedback" ], "additionalProperties": false }, "example": { "record": { "recordId": "76061c75-9c20-4694-8584-4f20270a243d", "recordKind": "task", "recordType": "rootTask", "taskId": "de707028-6ea2-4616-8864-c2ebd92b87f5", "parentTaskId": "2ad315e1-e215-4b88-8ba7-9869e414af3d", "runId": "599e5591-3334-4d0a-8bb9-069b278d9b20", "taskOpencodeBranchId": "80f2aa3d-c81e-4fb6-8912-3ed35b7daac9", "sourceType": "manual", "title": "text", "createdAt": "text", "taskStatus": "active", "latestRunStatus": "queued", "displayStatus": "text", "user": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text", "email": "ana@example.com", "country": "text" }, "agent": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" }, "workspace": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" }, "workspaceTag": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" }, "messageCount": 1, "userMessageCount": 1, "skillsUsed": [ "text" ], "skillsUsedDetails": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "connectorsAttached": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "metrics": { "runCount": 1, "inputFiles": 1, "outputFiles": 1, "attempts": 1, "totalTokens": 1 }, "taskViews": { "viewCount": 1, "unmeasuredReason": "no-view-key", "lastViewedAt": "text", "lastViewedBy": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text", "email": "ana@example.com", "country": "text" }, "viewedFrom": [ "task_route" ] }, "feedbackSummary": { "count": 1, "avgRating": 1.5, "maxRating": 1.5, "withTextCount": 1 }, "execution": { "kind": "traced", "sessionIds": [ "f6512788-45a7-480a-8953-a83849e71f91" ], "model": "anthropic/claude-sonnet-5", "models": [ "text" ], "callCount": 1, "tokens": 1, "durationMs": 1, "errorCount": 1, "estimatedCostUsd": 0.75, "errorMessageSample": "text", "lastActivityAt": "text", "subagents": { "count": 1, "tokens": 1, "estimatedCostUsd": 0.75, "errorCount": 1 } }, "billedCost": { "kind": "known", "costUsdMicros": 1 }, "workflows": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "label": "text" } ], "origin": "evaluation-run" }, "latestRun": { "runId": "599e5591-3334-4d0a-8bb9-069b278d9b20", "sessionId": "09724085-1664-4183-8554-bacab2816579", "taskOpencodeBranchId": "80f2aa3d-c81e-4fb6-8912-3ed35b7daac9", "inputPrompt": "text", "outputText": "text", "status": "queued", "createdAt": "text", "completedAt": "text", "metrics": { "inputFiles": 1, "outputFiles": 1, "attempts": 1 } }, "feedback": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "organizationId": "9a863583-7035-4b24-8dd5-efc00e4a1c8d", "valinorProjectId": "b7cfdf11-c17c-4106-8e7b-f1a0741f9b9e", "chatId": "18f0adc1-7acd-4c24-86e2-35315d7f8e76", "valinorMessageId": "0c0a7d07-8a75-44c2-866a-375c0f4f9140", "selectedText": "text", "userId": "6ccb2121-4ffd-40b0-8c2c-1607cf6a05be", "teamId": "73672067-345a-4307-8325-8154b2b650a6", "feedbackType": "text", "responses": {}, "createdAt": "text", "rubric": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions", "description": "text" }, "rubricVersion": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "version": 0.75, "snapshot": {} }, "submitter": { "kind": "user", "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions", "email": "ana@example.com" }, "target": { "taskId": "de707028-6ea2-4616-8864-c2ebd92b87f5", "taskTitle": "text", "chatId": "18f0adc1-7acd-4c24-86e2-35315d7f8e76", "messageText": "text" } } ] } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/analytics/records/$RECORD_ID\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nrecord_id = os.environ[\"RECORD_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/analytics/records/{record_id}\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/analytics/records/${encodeURIComponent(env(\"RECORD_ID\"))}`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/benchmarks": { "get": { "tags": [ "Evaluations" ], "summary": "List evaluations", "operationId": "listBenchmarks", "parameters": [ { "in": "query", "name": "workspaceScope", "required": false, "content": { "application/json": { "schema": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "all" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Every workspace in the organization plus the organization-wide evaluations; organization admins only" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "one" ] }, "workspaceId": { "type": "string", "minLength": 1, "description": "The workspace the request is about" } }, "required": [ "kind", "workspaceId" ], "additionalProperties": false, "description": "One workspace: its evaluations plus the organization-wide ones" } ], "description": "Which workspace to read. When omitted: every workspace for an organization admin; your one workspace if your Evals access covers exactly one; otherwise 403 asking you to name a workspace." } } }, "description": "Send it as JSON in one value, for example `workspaceScope={\"kind\":\"one\",\"workspaceId\":\"\"}`. Any other encoding is ignored." } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique evaluation identifier" }, "orgId": { "type": "string", "minLength": 1, "description": "Organization that owns this evaluation" }, "workspaceId": { "type": [ "string", "null" ], "minLength": 1, "description": "The workspace this evaluation belongs to; null means organization-wide, shown in every workspace" }, "createdById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who created this evaluation" }, "ownerUserId": { "type": [ "string", "null" ], "minLength": 1, "description": "The person who owns this evaluation; null for evaluations made before owners were recorded (organization-wide, and changeable by an organization admin)" }, "visibility": { "type": "string", "enum": [ "org", "workspace", "personal" ], "description": "Where this evaluation is placed: organization-wide, shared into its workspace, or private to its owner" }, "name": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Evaluation display name" }, "description": { "type": [ "string", "null" ], "maxLength": 2000, "description": "Optional evaluation description" }, "judgeModel": { "type": "string", "description": "The model that grades a dataset's or task's model criterion, unless the criterion names its own model" }, "threshold": { "type": [ "integer", "null" ], "minimum": 0, "maximum": 100, "description": "Pass threshold on the 0 to 100 result scale; null when the evaluation sets none" }, "axes": { "anyOf": [ { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "model" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "A model the held agent runs" }, "minItems": 1, "description": "The models to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." } }, "required": [ "skillId" ], "additionalProperties": false, "description": "A skill, at its newest version (no version given) or a pinned version number" }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A skill every cell runs with, and which version. A skill ID on its own means its newest version." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "agentId", "skills" ], "additionalProperties": false, "description": "The agent every model runs on, and the skills it runs with" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Model: one agent and its skills are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "skill" ] }, "values": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill" ] }, "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." } }, "required": [ "kind", "skillId" ], "additionalProperties": false, "description": "A skill the held agent runs with, at its newest version or a pinned one" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "no_skill" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The held agent with no skill attached: the baseline a skill is compared against" } ] }, "minItems": 1, "description": "The skills to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates (two versions of one skill count as two values)" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" } }, "required": [ "agentId", "model" ], "additionalProperties": false, "description": "The agent every skill runs on, and the model it runs" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Skill: one agent and its model are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "agent" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent" ] }, "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" } }, "required": [ "kind", "agentId" ], "additionalProperties": false, "description": "An agent" }, "minItems": 1, "description": "The agents to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." } }, "required": [ "skillId" ], "additionalProperties": false, "description": "A skill, at its newest version (no version given) or a pinned version number" }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A skill every cell runs with, and which version. A skill ID on its own means its newest version." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "model", "skills" ], "additionalProperties": false, "description": "The model every agent runs (one they can all run, or each agent's own default), and the skills they run with" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Agent: the model and the skills are held fixed" } ], "description": "The question this evaluation asks: the one axis it tests, the values it compares, and what it holds fixed. Editable at any time; each run keeps its own copy from when it started." }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" } }, "required": [ "id", "orgId", "workspaceId", "createdById", "ownerUserId", "visibility", "name", "description", "judgeModel", "threshold", "axes", "createdAt", "updatedAt" ], "additionalProperties": false, "description": "An evaluation: the datasets it measures, its axes (what it tests and what it holds fixed), the default judge model for the datasets' criteria, and its pass threshold. It has no grading criteria of its own." }, "description": "Evaluations in the organization" }, "example": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "workspaceId": "09707fa5-aaae-4493-8a23-827111000f89", "createdById": "f7661063-fadd-49f3-8a00-1467f872dba6", "ownerUserId": "ba132481-96c7-4211-8fb0-ca95d3072fd0", "visibility": "org", "name": "Refund questions", "description": "text", "judgeModel": "text", "threshold": 1, "axes": { "testing": "model", "values": [ { "kind": "model", "model": "anthropic/claude-sonnet-5" } ], "held": { "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "skills": [ { "skillId": "3bb7a9ef-8bba-4c48-8bc8-0cccec5c0a00", "version": 1 } ] } }, "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z" } ] } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/benchmarks\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/benchmarks\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/benchmarks`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Find benchmarks with in-flight runs\n# The CLI pages ({items, nextCursor}) and adds runCount and activeRunCount; GET /benchmarks returns a plain array without them.\ncontext evals benchmark list --limit 100 | jq '.items[] | select(.activeRunCount > 0) | .id'" } ] } }, "/benchmarks/{evaluationId}": { "get": { "tags": [ "Evaluations" ], "summary": "Get an evaluation", "operationId": "getBenchmark", "parameters": [ { "in": "path", "name": "evaluationId", "required": true, "description": "Evaluation identifier", "schema": { "type": "string", "minLength": 1, "description": "Evaluation identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique evaluation identifier" }, "orgId": { "type": "string", "minLength": 1, "description": "Organization that owns this evaluation" }, "workspaceId": { "type": [ "string", "null" ], "minLength": 1, "description": "The workspace this evaluation belongs to; null means organization-wide, shown in every workspace" }, "createdById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who created this evaluation" }, "ownerUserId": { "type": [ "string", "null" ], "minLength": 1, "description": "The person who owns this evaluation; null for evaluations made before owners were recorded (organization-wide, and changeable by an organization admin)" }, "visibility": { "type": "string", "enum": [ "org", "workspace", "personal" ], "description": "Where this evaluation is placed: organization-wide, shared into its workspace, or private to its owner" }, "name": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Evaluation display name" }, "description": { "type": [ "string", "null" ], "maxLength": 2000, "description": "Optional evaluation description" }, "judgeModel": { "type": "string", "description": "The model that grades a dataset's or task's model criterion, unless the criterion names its own model" }, "threshold": { "type": [ "integer", "null" ], "minimum": 0, "maximum": 100, "description": "Pass threshold on the 0 to 100 result scale; null when the evaluation sets none" }, "axes": { "anyOf": [ { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "model" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "A model the held agent runs" }, "minItems": 1, "description": "The models to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." } }, "required": [ "skillId" ], "additionalProperties": false, "description": "A skill, at its newest version (no version given) or a pinned version number" }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A skill every cell runs with, and which version. A skill ID on its own means its newest version." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "agentId", "skills" ], "additionalProperties": false, "description": "The agent every model runs on, and the skills it runs with" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Model: one agent and its skills are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "skill" ] }, "values": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill" ] }, "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." } }, "required": [ "kind", "skillId" ], "additionalProperties": false, "description": "A skill the held agent runs with, at its newest version or a pinned one" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "no_skill" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The held agent with no skill attached: the baseline a skill is compared against" } ] }, "minItems": 1, "description": "The skills to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates (two versions of one skill count as two values)" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" } }, "required": [ "agentId", "model" ], "additionalProperties": false, "description": "The agent every skill runs on, and the model it runs" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Skill: one agent and its model are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "agent" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent" ] }, "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" } }, "required": [ "kind", "agentId" ], "additionalProperties": false, "description": "An agent" }, "minItems": 1, "description": "The agents to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." } }, "required": [ "skillId" ], "additionalProperties": false, "description": "A skill, at its newest version (no version given) or a pinned version number" }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A skill every cell runs with, and which version. A skill ID on its own means its newest version." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "model", "skills" ], "additionalProperties": false, "description": "The model every agent runs (one they can all run, or each agent's own default), and the skills they run with" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Agent: the model and the skills are held fixed" } ], "description": "The question this evaluation asks: the one axis it tests, the values it compares, and what it holds fixed. Editable at any time; each run keeps its own copy from when it started." }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" } }, "required": [ "id", "orgId", "workspaceId", "createdById", "ownerUserId", "visibility", "name", "description", "judgeModel", "threshold", "axes", "createdAt", "updatedAt" ], "additionalProperties": false, "description": "An evaluation: the datasets it measures, its axes (what it tests and what it holds fixed), the default judge model for the datasets' criteria, and its pass threshold. It has no grading criteria of its own." }, "example": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "workspaceId": "09707fa5-aaae-4493-8a23-827111000f89", "createdById": "f7661063-fadd-49f3-8a00-1467f872dba6", "ownerUserId": "ba132481-96c7-4211-8fb0-ca95d3072fd0", "visibility": "org", "name": "Refund questions", "description": "text", "judgeModel": "text", "threshold": 1, "axes": { "testing": "model", "values": [ { "kind": "model", "model": "anthropic/claude-sonnet-5" } ], "held": { "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "skills": [ { "skillId": "3bb7a9ef-8bba-4c48-8bc8-0cccec5c0a00", "version": 1 } ] } }, "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z" } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/benchmarks/$EVALUATION_ID\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nevaluation_id = os.environ[\"EVALUATION_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/benchmarks/{evaluation_id}\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/benchmarks/${encodeURIComponent(env(\"EVALUATION_ID\"))}`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/benchmarks/{evaluationId}/datasets": { "get": { "tags": [ "Evaluations" ], "summary": "List an evaluation's dataset links", "operationId": "listBenchmarkDatasets", "parameters": [ { "in": "path", "name": "evaluationId", "required": true, "description": "Evaluation identifier", "schema": { "type": "string", "minLength": 1, "description": "Evaluation identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique link identifier" }, "evaluationId": { "type": "string", "minLength": 1, "description": "Parent evaluation" }, "datasetId": { "type": "string", "minLength": 1, "description": "Linked dataset" }, "order": { "type": "integer", "minimum": 0, "description": "Display and run order within the evaluation" } }, "required": [ "id", "evaluationId", "datasetId", "order" ], "additionalProperties": false, "description": "Ordered link between an evaluation and a dataset (many to many)" }, "description": "Dataset links for the evaluation" }, "example": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "evaluationId": "66248a43-3af8-4b5b-86b6-fcb59feb38b2", "datasetId": "921326a1-adf5-495c-89d8-b4b9afd6c36f", "order": 1 } ] } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "description": "One link row per dataset the evaluation measures, `{id, evaluationId, datasetId, order}`, sorted by `order`; archived datasets are left out. Fetch each dataset with `GET /datasets/{datasetId}`.", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/benchmarks/$EVALUATION_ID/datasets\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nevaluation_id = os.environ[\"EVALUATION_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/benchmarks/{evaluation_id}/datasets\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/benchmarks/${encodeURIComponent(env(\"EVALUATION_ID\"))}/datasets`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/benchmarks/{evaluationId}/runs": { "get": { "tags": [ "Runs" ], "summary": "List an evaluation's runs", "operationId": "listBenchmarkRuns", "parameters": [ { "in": "path", "name": "evaluationId", "required": true, "description": "Evaluation identifier", "schema": { "type": "string", "minLength": 1, "description": "Evaluation identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique run identifier" }, "orgId": { "type": "string", "minLength": 1, "description": "Organization that owns this run" }, "evaluationId": { "type": "string", "minLength": 1, "description": "Evaluation being run" }, "topicId": { "type": [ "string", "null" ], "minLength": 1, "description": "Workflow this run belongs to; null for a run started from the app's evaluations view" }, "evaluationVersionId": { "type": [ "string", "null" ], "minLength": 1, "description": "Evaluation version this run uses; null only for runs created before evaluation versions existed" }, "runById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who started this run" }, "axes": { "anyOf": [ { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "model" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "A model the held agent runs" }, "minItems": 1, "description": "The models to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "skillId" ], "additionalProperties": false }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A held skill as fixed by the run. A skill ID on its own means the run started before versions were fixed." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "agentId", "skills" ], "additionalProperties": false, "description": "The agent every model ran on, and the skills it ran with, each at its fixed version" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Model: one agent and its skills are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "skill" ] }, "values": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill" ] }, "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "kind", "skillId" ], "additionalProperties": false, "description": "A tested skill as fixed by the run: the version it runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "no_skill" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The held agent with no skill attached: the baseline a skill is compared against" } ] }, "minItems": 1, "description": "The skills to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates (two versions of one skill count as two values)" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" } }, "required": [ "agentId", "model" ], "additionalProperties": false, "description": "The agent every skill runs on, and the model it runs" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Skill: one agent and its model are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "agent" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent" ] }, "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" } }, "required": [ "kind", "agentId" ], "additionalProperties": false, "description": "An agent" }, "minItems": 1, "description": "The agents to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "skillId" ], "additionalProperties": false }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A held skill as fixed by the run. A skill ID on its own means the run started before versions were fixed." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "model", "skills" ], "additionalProperties": false, "description": "The model every agent ran, and the skills they ran with, each at its fixed version" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Agent: the model and the skills are held fixed" } ], "description": "The axes a run executed, fixed when it started, with every model resolved and every skill version fixed. Reports use these, not the evaluation's current settings." }, "cells": { "type": "array", "items": { "type": "object", "properties": { "cellId": { "type": "string", "format": "uuid", "description": "Identifies one cell within its run. A run's results, dispatches, and exports are keyed by it." }, "value": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "A model the held agent runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill" ] }, "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "kind", "skillId" ], "additionalProperties": false, "description": "A tested skill as fixed by the run: the version it runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "no_skill" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The held agent with no skill attached: the baseline a skill is compared against" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent" ] }, "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" } }, "required": [ "kind", "agentId" ], "additionalProperties": false, "description": "An agent" } ], "description": "The tested value this cell ran: one of the run's `axes.values`" }, "attempt": { "type": "integer", "minimum": 1, "maximum": 5, "description": "Which repeat of that value this is, starting at 1 (shown as \"Run n\"); a run with one run per task has only 1" } }, "required": [ "cellId", "value", "attempt" ], "additionalProperties": false, "description": "One (value x repeat) of a run. Every task in the run runs once per cell." }, "minItems": 1, "description": "The run's cells, ordered by value (in the axes' order), then by repeat within each value: number of values x runs per task" }, "status": { "type": "string", "enum": [ "PENDING", "RUNNING", "EVALUATING", "COMPLETED", "FAILED", "STOPPED", "UNKNOWN" ], "description": "Current run status, or `UNKNOWN` when the stored value is not one of the allowed values" }, "taskCount": { "type": "integer", "minimum": 0, "description": "Total number of (task x cell) pairs to run" }, "completedCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs completed so far" }, "concurrency": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Tasks run at once for this run (min 1, max 100 or the deployment's lower limit)." }, "judgeConcurrency": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Agent judges run at once for this run, across every task (min 1, max 100 or the deployment's lower limit). LLM judges and rules do not count." }, "succeededCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as succeeded" }, "failedCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as failed" }, "cancelledCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as canceled" }, "judgedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Result rows with a score: a judge score and no task error, or, on a criteria-graded run, a task where at least one criterion reached a verdict. Null until the run's totals are computed." }, "passedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Tasks the judge passed. Null for an evaluation scored on a 0 to 1 scale (no pass or fail), and null until the run's totals are computed." }, "maskedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Result rows left out of the score: the judge gave no score or the task errored, or, on a criteria-graded run, none of the task's criteria reached a verdict (they could not run). Null until the run's totals are computed." }, "avgScore": { "type": [ "number", "null" ], "description": "Mean judged score on the standard 0 to 1 scale. Null until the run's totals are computed, or when nothing was judged." }, "heldOutJudgedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "`judgedCount` for the run's held-out tasks only (tasks the system never learns from). Null until the run's totals are computed, or when the totals come from older stored results that do not record which tasks are held out." }, "heldOutAvgScore": { "type": [ "number", "null" ], "description": "`avgScore` for the run's held-out tasks only, on the same 0 to 1 scale. Null when `heldOutJudgedCount` is null or 0." }, "costUsd": { "type": [ "number", "null" ], "description": "Task cost plus judge cost for the whole run, in dollars (USD). Null when any part is unknown (never a partial sum)." }, "resolvedMismatch": { "type": "boolean", "description": "True when any dispatch was served by a different model than its cell asked for. This is a notice only: the run's result and every list, trend, and chart still show its score." }, "resolvedUnknownCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Dispatches whose serving model could not be identified from billing when the run finished. This is not a mismatch, only unknown, and it never holds back the run's result. Null until the run's totals are computed." }, "costUnknownReason": { "type": [ "string", "null" ], "enum": [ "ledger_not_final", "device_calls_unbounded", "no_rows", "unrated_rows", "warehouse_session_missing", "judge_cost_unknown", "ledger_reroll_failed", null ], "description": "Why `costUsd` is null on an ended run, by order of precedence. Null when the cost is known, while the run is still going, and on an ended run whose totals have not been computed." }, "startedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When the run started" }, "completedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When the run finished, whether it succeeded or failed" }, "error": { "type": [ "string", "null" ], "description": "Error message if the run failed" }, "stopRequestedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When someone asked to stop the run, or null if nobody did. It is set while the run is still going: the run counts as stopping until its status is `STOPPED`." }, "stopRequestedByUserId": { "type": [ "string", "null" ], "minLength": 1, "description": "Who asked to stop the run: the person, or the person an agent acts for when an agent asked. Null when nobody did." }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" } }, "required": [ "id", "orgId", "evaluationId", "topicId", "evaluationVersionId", "runById", "axes", "cells", "taskCount", "completedCount", "concurrency", "judgeConcurrency", "succeededCount", "failedCount", "cancelledCount", "judgedCount", "passedCount", "maskedCount", "avgScore", "heldOutJudgedCount", "heldOutAvgScore", "costUsd", "resolvedMismatch", "resolvedUnknownCount", "costUnknownReason", "startedAt", "completedAt", "error", "stopRequestedAt", "stopRequestedByUserId", "createdAt", "updatedAt" ], "additionalProperties": false, "description": "One execution of an evaluation: the axes it fixed when it started, its cells, and its progress" }, "description": "Runs belonging to the evaluation" }, "example": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "evaluationId": "66248a43-3af8-4b5b-86b6-fcb59feb38b2", "topicId": "c42387f0-94d3-427e-80d1-a63853a68624", "evaluationVersionId": "66fb5178-67cd-4706-8327-e5f646891209", "runById": "97bf5c87-0d64-4f5b-8600-9aec988e1e1c", "axes": { "testing": "model", "values": [ { "kind": "model", "model": "anthropic/claude-sonnet-5" } ], "held": { "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "skills": [ { "skillId": "3bb7a9ef-8bba-4c48-8bc8-0cccec5c0a00", "version": 1, "versionId": "772c657e-ac9c-4550-8ecf-9f77201cbb95", "followsLatest": true } ] } }, "cells": [ { "cellId": "5ec92200-48c8-4c37-8bf0-a28aa3d08fcd", "value": { "kind": "model", "model": "anthropic/claude-sonnet-5" }, "attempt": 1 } ], "status": "PENDING", "taskCount": 1, "completedCount": 1, "concurrency": 1, "judgeConcurrency": 1, "succeededCount": 1, "failedCount": 1, "cancelledCount": 1, "judgedCount": 1, "passedCount": 1, "maskedCount": 1, "avgScore": 0.75, "heldOutJudgedCount": 1, "heldOutAvgScore": 0.75, "costUsd": 0.75, "resolvedMismatch": true, "resolvedUnknownCount": 1, "costUnknownReason": "ledger_not_final", "startedAt": "2026-10-14T09:30:00.000Z", "completedAt": "2026-10-14T09:30:00.000Z", "error": "text", "stopRequestedAt": "2026-10-14T09:30:00.000Z", "stopRequestedByUserId": "c772e75e-d5d2-4cf6-8a64-846bcc9144d5", "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z" } ] } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "description": "Runs are in no set order: sort by `createdAt` to find the latest.", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/benchmarks/$EVALUATION_ID/runs\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nevaluation_id = os.environ[\"EVALUATION_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/benchmarks/{evaluation_id}/runs\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/benchmarks/${encodeURIComponent(env(\"EVALUATION_ID\"))}/runs`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/datasets": { "get": { "tags": [ "Datasets" ], "summary": "List datasets", "operationId": "listDatasets", "parameters": [ { "in": "query", "name": "workspaceScope", "required": false, "content": { "application/json": { "schema": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "all" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Every workspace in the organization plus the organization-wide datasets; organization admins only" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "one" ] }, "workspaceId": { "type": "string", "minLength": 1, "description": "The workspace the request is about" } }, "required": [ "kind", "workspaceId" ], "additionalProperties": false, "description": "One workspace: its datasets plus the organization-wide ones" } ], "description": "Which workspace to read. When omitted: every workspace for an organization admin; your one workspace if your Evals access covers exactly one; otherwise 403 asking you to name a workspace." } } }, "description": "Send it as JSON in one value, for example `workspaceScope={\"kind\":\"one\",\"workspaceId\":\"\"}`. Any other encoding is ignored." } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique dataset identifier" }, "orgId": { "type": [ "string", "null" ], "minLength": 1, "description": "Organization that owns this dataset" }, "workspaceId": { "type": [ "string", "null" ], "minLength": 1, "description": "The workspace this dataset belongs to; null means organization-wide, shown in every workspace" }, "createdById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who created this dataset" }, "ownerUserId": { "type": [ "string", "null" ], "minLength": 1, "description": "The person who owns this dataset; null for datasets made before owners were recorded (organization-wide, and changeable by an organization admin)" }, "visibility": { "type": "string", "enum": [ "org", "workspace", "personal" ], "description": "Where this dataset is placed: organization-wide, shared into its workspace, or private to its owner" }, "name": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Dataset display name" }, "description": { "type": [ "string", "null" ], "maxLength": 2000, "description": "Optional dataset description" }, "isPublic": { "type": "boolean", "description": "Whether this dataset is publicly shared" }, "source": { "type": [ "string", "null" ], "description": "Where the dataset came from (for example, `huggingface:openai/gsm8k`). Null for datasets created by hand." }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" } }, "required": [ "id", "orgId", "workspaceId", "createdById", "ownerUserId", "visibility", "name", "description", "isPublic", "source", "createdAt", "updatedAt" ], "additionalProperties": false, "description": "A collection of tasks (prompts) for evaluations" }, "description": "Datasets the organization can access" }, "example": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "workspaceId": "09707fa5-aaae-4493-8a23-827111000f89", "createdById": "f7661063-fadd-49f3-8a00-1467f872dba6", "ownerUserId": "ba132481-96c7-4211-8fb0-ca95d3072fd0", "visibility": "org", "name": "Refund questions", "description": "text", "isPublic": true, "source": "text", "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z" } ] } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/datasets\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode 'workspaceScope={\"kind\":\"one\",\"workspaceId\":\"'\"$WORKSPACE_ID\"'\"}'" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/datasets\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"workspaceScope\": json.dumps({\"kind\": \"one\", \"workspaceId\": os.environ[\"WORKSPACE_ID\"]})},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/datasets`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"workspaceScope\", JSON.stringify({ kind: \"one\", workspaceId: env(\"WORKSPACE_ID\") }))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/datasets/{datasetId}": { "get": { "tags": [ "Datasets" ], "summary": "Get a dataset", "operationId": "getDataset", "parameters": [ { "in": "path", "name": "datasetId", "required": true, "description": "Dataset identifier", "schema": { "type": "string", "minLength": 1, "description": "Dataset identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique dataset identifier" }, "orgId": { "type": [ "string", "null" ], "minLength": 1, "description": "Organization that owns this dataset" }, "workspaceId": { "type": [ "string", "null" ], "minLength": 1, "description": "The workspace this dataset belongs to; null means organization-wide, shown in every workspace" }, "createdById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who created this dataset" }, "ownerUserId": { "type": [ "string", "null" ], "minLength": 1, "description": "The person who owns this dataset; null for datasets made before owners were recorded (organization-wide, and changeable by an organization admin)" }, "visibility": { "type": "string", "enum": [ "org", "workspace", "personal" ], "description": "Where this dataset is placed: organization-wide, shared into its workspace, or private to its owner" }, "name": { "type": "string", "minLength": 1, "maxLength": 200, "description": "Dataset display name" }, "description": { "type": [ "string", "null" ], "maxLength": 2000, "description": "Optional dataset description" }, "isPublic": { "type": "boolean", "description": "Whether this dataset is publicly shared" }, "source": { "type": [ "string", "null" ], "description": "Where the dataset came from (for example, `huggingface:openai/gsm8k`). Null for datasets created by hand." }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" } }, "required": [ "id", "orgId", "workspaceId", "createdById", "ownerUserId", "visibility", "name", "description", "isPublic", "source", "createdAt", "updatedAt" ], "additionalProperties": false, "description": "A collection of tasks (prompts) for evaluations" }, "example": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "workspaceId": "09707fa5-aaae-4493-8a23-827111000f89", "createdById": "f7661063-fadd-49f3-8a00-1467f872dba6", "ownerUserId": "ba132481-96c7-4211-8fb0-ca95d3072fd0", "visibility": "org", "name": "Refund questions", "description": "text", "isPublic": true, "source": "text", "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z" } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/datasets/$DATASET_ID\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\ndataset_id = os.environ[\"DATASET_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/datasets/{dataset_id}\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/datasets/${encodeURIComponent(env(\"DATASET_ID\"))}`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/datasets/{datasetId}/items": { "get": { "tags": [ "Datasets" ], "summary": "List a dataset's tasks", "operationId": "listDatasetItems", "parameters": [ { "in": "path", "name": "datasetId", "required": true, "description": "Dataset identifier", "schema": { "type": "string", "minLength": 1, "description": "Dataset identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique task identifier" }, "datasetId": { "type": "string", "minLength": 1, "description": "Dataset this task belongs to" }, "name": { "type": [ "string", "null" ], "maxLength": 200, "description": "Optional task name or label for display" }, "prompt": { "type": "string", "minLength": 1, "maxLength": 100000, "description": "The prompt sent to the agent" }, "expectedOutput": { "type": [ "string", "null" ], "maxLength": 100000, "description": "Reference answer the judge compares against. Null if the task has no expected output." }, "fileIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "maxItems": 10, "description": "IDs of the Drive files attached to this task" }, "expectedFileIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "maxItems": 10, "description": "The task's expected files (answer keys): given to its criteria, never to the agent being evaluated" }, "heldOut": { "type": "boolean", "description": "A held-out task: graded like every other task, but the system never learns from it, and a run also scores its held-out tasks separately" }, "order": { "type": "integer", "minimum": 0, "description": "Display and execution order within the dataset" }, "status": { "type": "string", "enum": [ "ready", "draft", "needs_review" ], "description": "Authoring state: `ready` (ready to measure), `draft` (still being written), or `needs_review` (flagged for a second look). Informational only: a run measures every linked task whatever its status." }, "source": { "type": "string", "enum": [ "manual", "csv", "trace", "huggingface", "generated" ], "description": "Where the task came from: written by hand, imported from a CSV or other file, promoted from a trace, imported from Hugging Face, or generated (a variant written by a model)" }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" } }, "required": [ "id", "datasetId", "name", "prompt", "expectedOutput", "fileIds", "expectedFileIds", "heldOut", "order", "status", "source", "createdAt", "updatedAt" ], "additionalProperties": false, "description": "A single task within a dataset" }, "description": "Tasks in the dataset" }, "example": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "datasetId": "921326a1-adf5-495c-89d8-b4b9afd6c36f", "name": "Refund questions", "prompt": "text", "expectedOutput": "text", "fileIds": [ "71e134ff-914f-406c-876f-f39828b751a1" ], "expectedFileIds": [ "e74d97af-67bf-4815-8376-cb54c5a90204" ], "heldOut": true, "order": 1, "status": "ready", "source": "manual", "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z" } ] } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/datasets/$DATASET_ID/items\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\ndataset_id = os.environ[\"DATASET_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/datasets/{dataset_id}/items\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/datasets/${encodeURIComponent(env(\"DATASET_ID\"))}/items`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Iterate prompts in a dataset\n# The CLI pages ({items, nextCursor}); GET /datasets/{datasetId}/items returns a plain array: jq '.[].prompt'.\ncontext evals datasets items list $DATASET_ID --limit 200 | jq '.items[].prompt'" } ] } }, "/items/{taskId}": { "get": { "tags": [ "Datasets" ], "summary": "Get a task", "operationId": "getItem", "parameters": [ { "in": "path", "name": "taskId", "required": true, "description": "Task identifier", "schema": { "type": "string", "minLength": 1, "description": "Task identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique task identifier" }, "datasetId": { "type": "string", "minLength": 1, "description": "Dataset this task belongs to" }, "name": { "type": [ "string", "null" ], "maxLength": 200, "description": "Optional task name or label for display" }, "prompt": { "type": "string", "minLength": 1, "maxLength": 100000, "description": "The prompt sent to the agent" }, "expectedOutput": { "type": [ "string", "null" ], "maxLength": 100000, "description": "Reference answer the judge compares against. Null if the task has no expected output." }, "fileIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "maxItems": 10, "description": "IDs of the Drive files attached to this task" }, "expectedFileIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "maxItems": 10, "description": "The task's expected files (answer keys): given to its criteria, never to the agent being evaluated" }, "heldOut": { "type": "boolean", "description": "A held-out task: graded like every other task, but the system never learns from it, and a run also scores its held-out tasks separately" }, "order": { "type": "integer", "minimum": 0, "description": "Display and execution order within the dataset" }, "status": { "type": "string", "enum": [ "ready", "draft", "needs_review" ], "description": "Authoring state: `ready` (ready to measure), `draft` (still being written), or `needs_review` (flagged for a second look). Informational only: a run measures every linked task whatever its status." }, "source": { "type": "string", "enum": [ "manual", "csv", "trace", "huggingface", "generated" ], "description": "Where the task came from: written by hand, imported from a CSV or other file, promoted from a trace, imported from Hugging Face, or generated (a variant written by a model)" }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" } }, "required": [ "id", "datasetId", "name", "prompt", "expectedOutput", "fileIds", "expectedFileIds", "heldOut", "order", "status", "source", "createdAt", "updatedAt" ], "additionalProperties": false, "description": "A single task within a dataset" }, "example": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "datasetId": "921326a1-adf5-495c-89d8-b4b9afd6c36f", "name": "Refund questions", "prompt": "text", "expectedOutput": "text", "fileIds": [ "71e134ff-914f-406c-876f-f39828b751a1" ], "expectedFileIds": [ "e74d97af-67bf-4815-8376-cb54c5a90204" ], "heldOut": true, "order": 1, "status": "ready", "source": "manual", "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z" } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/items/$TASK_ID\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\ntask_id = os.environ[\"TASK_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/items/{task_id}\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/items/${encodeURIComponent(env(\"TASK_ID\"))}`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/runs/{runId}": { "get": { "tags": [ "Runs" ], "summary": "Get a run", "operationId": "getRun", "parameters": [ { "in": "path", "name": "runId", "required": true, "description": "Run identifier", "schema": { "type": "string", "minLength": 1, "description": "Run identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique run identifier" }, "orgId": { "type": "string", "minLength": 1, "description": "Organization that owns this run" }, "evaluationId": { "type": "string", "minLength": 1, "description": "Evaluation being run" }, "topicId": { "type": [ "string", "null" ], "minLength": 1, "description": "Workflow this run belongs to; null for a run started from the app's evaluations view" }, "evaluationVersionId": { "type": [ "string", "null" ], "minLength": 1, "description": "Evaluation version this run uses; null only for runs created before evaluation versions existed" }, "runById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who started this run" }, "axes": { "anyOf": [ { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "model" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "A model the held agent runs" }, "minItems": 1, "description": "The models to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "skillId" ], "additionalProperties": false }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A held skill as fixed by the run. A skill ID on its own means the run started before versions were fixed." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "agentId", "skills" ], "additionalProperties": false, "description": "The agent every model ran on, and the skills it ran with, each at its fixed version" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Model: one agent and its skills are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "skill" ] }, "values": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill" ] }, "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "kind", "skillId" ], "additionalProperties": false, "description": "A tested skill as fixed by the run: the version it runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "no_skill" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The held agent with no skill attached: the baseline a skill is compared against" } ] }, "minItems": 1, "description": "The skills to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates (two versions of one skill count as two values)" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" } }, "required": [ "agentId", "model" ], "additionalProperties": false, "description": "The agent every skill runs on, and the model it runs" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Skill: one agent and its model are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "agent" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent" ] }, "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" } }, "required": [ "kind", "agentId" ], "additionalProperties": false, "description": "An agent" }, "minItems": 1, "description": "The agents to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "skillId" ], "additionalProperties": false }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A held skill as fixed by the run. A skill ID on its own means the run started before versions were fixed." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "model", "skills" ], "additionalProperties": false, "description": "The model every agent ran, and the skills they ran with, each at its fixed version" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Agent: the model and the skills are held fixed" } ], "description": "The axes a run executed, fixed when it started, with every model resolved and every skill version fixed. Reports use these, not the evaluation's current settings." }, "cells": { "type": "array", "items": { "type": "object", "properties": { "cellId": { "type": "string", "format": "uuid", "description": "Identifies one cell within its run. A run's results, dispatches, and exports are keyed by it." }, "value": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "A model the held agent runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill" ] }, "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "kind", "skillId" ], "additionalProperties": false, "description": "A tested skill as fixed by the run: the version it runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "no_skill" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The held agent with no skill attached: the baseline a skill is compared against" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent" ] }, "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" } }, "required": [ "kind", "agentId" ], "additionalProperties": false, "description": "An agent" } ], "description": "The tested value this cell ran: one of the run's `axes.values`" }, "attempt": { "type": "integer", "minimum": 1, "maximum": 5, "description": "Which repeat of that value this is, starting at 1 (shown as \"Run n\"); a run with one run per task has only 1" } }, "required": [ "cellId", "value", "attempt" ], "additionalProperties": false, "description": "One (value x repeat) of a run. Every task in the run runs once per cell." }, "minItems": 1, "description": "The run's cells, ordered by value (in the axes' order), then by repeat within each value: number of values x runs per task" }, "status": { "type": "string", "enum": [ "PENDING", "RUNNING", "EVALUATING", "COMPLETED", "FAILED", "STOPPED", "UNKNOWN" ], "description": "Current run status, or `UNKNOWN` when the stored value is not one of the allowed values" }, "taskCount": { "type": "integer", "minimum": 0, "description": "Total number of (task x cell) pairs to run" }, "completedCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs completed so far" }, "concurrency": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Tasks run at once for this run (min 1, max 100 or the deployment's lower limit)." }, "judgeConcurrency": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Agent judges run at once for this run, across every task (min 1, max 100 or the deployment's lower limit). LLM judges and rules do not count." }, "succeededCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as succeeded" }, "failedCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as failed" }, "cancelledCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as canceled" }, "judgedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Result rows with a score: a judge score and no task error, or, on a criteria-graded run, a task where at least one criterion reached a verdict. Null until the run's totals are computed." }, "passedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Tasks the judge passed. Null for an evaluation scored on a 0 to 1 scale (no pass or fail), and null until the run's totals are computed." }, "maskedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Result rows left out of the score: the judge gave no score or the task errored, or, on a criteria-graded run, none of the task's criteria reached a verdict (they could not run). Null until the run's totals are computed." }, "avgScore": { "type": [ "number", "null" ], "description": "Mean judged score on the standard 0 to 1 scale. Null until the run's totals are computed, or when nothing was judged." }, "heldOutJudgedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "`judgedCount` for the run's held-out tasks only (tasks the system never learns from). Null until the run's totals are computed, or when the totals come from older stored results that do not record which tasks are held out." }, "heldOutAvgScore": { "type": [ "number", "null" ], "description": "`avgScore` for the run's held-out tasks only, on the same 0 to 1 scale. Null when `heldOutJudgedCount` is null or 0." }, "costUsd": { "type": [ "number", "null" ], "description": "Task cost plus judge cost for the whole run, in dollars (USD). Null when any part is unknown (never a partial sum)." }, "resolvedMismatch": { "type": "boolean", "description": "True when any dispatch was served by a different model than its cell asked for. This is a notice only: the run's result and every list, trend, and chart still show its score." }, "resolvedUnknownCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Dispatches whose serving model could not be identified from billing when the run finished. This is not a mismatch, only unknown, and it never holds back the run's result. Null until the run's totals are computed." }, "costUnknownReason": { "type": [ "string", "null" ], "enum": [ "ledger_not_final", "device_calls_unbounded", "no_rows", "unrated_rows", "warehouse_session_missing", "judge_cost_unknown", "ledger_reroll_failed", null ], "description": "Why `costUsd` is null on an ended run, by order of precedence. Null when the cost is known, while the run is still going, and on an ended run whose totals have not been computed." }, "startedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When the run started" }, "completedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When the run finished, whether it succeeded or failed" }, "error": { "type": [ "string", "null" ], "description": "Error message if the run failed" }, "stopRequestedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When someone asked to stop the run, or null if nobody did. It is set while the run is still going: the run counts as stopping until its status is `STOPPED`." }, "stopRequestedByUserId": { "type": [ "string", "null" ], "minLength": 1, "description": "Who asked to stop the run: the person, or the person an agent acts for when an agent asked. Null when nobody did." }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" } }, "required": [ "id", "orgId", "evaluationId", "topicId", "evaluationVersionId", "runById", "axes", "cells", "taskCount", "completedCount", "concurrency", "judgeConcurrency", "succeededCount", "failedCount", "cancelledCount", "judgedCount", "passedCount", "maskedCount", "avgScore", "heldOutJudgedCount", "heldOutAvgScore", "costUsd", "resolvedMismatch", "resolvedUnknownCount", "costUnknownReason", "startedAt", "completedAt", "error", "stopRequestedAt", "stopRequestedByUserId", "createdAt", "updatedAt" ], "additionalProperties": false, "description": "One execution of an evaluation: the axes it fixed when it started, its cells, and its progress" }, "example": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "evaluationId": "66248a43-3af8-4b5b-86b6-fcb59feb38b2", "topicId": "c42387f0-94d3-427e-80d1-a63853a68624", "evaluationVersionId": "66fb5178-67cd-4706-8327-e5f646891209", "runById": "97bf5c87-0d64-4f5b-8600-9aec988e1e1c", "axes": { "testing": "model", "values": [ { "kind": "model", "model": "anthropic/claude-sonnet-5" } ], "held": { "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "skills": [ { "skillId": "3bb7a9ef-8bba-4c48-8bc8-0cccec5c0a00", "version": 1, "versionId": "772c657e-ac9c-4550-8ecf-9f77201cbb95", "followsLatest": true } ] } }, "cells": [ { "cellId": "5ec92200-48c8-4c37-8bf0-a28aa3d08fcd", "value": { "kind": "model", "model": "anthropic/claude-sonnet-5" }, "attempt": 1 } ], "status": "PENDING", "taskCount": 1, "completedCount": 1, "concurrency": 1, "judgeConcurrency": 1, "succeededCount": 1, "failedCount": 1, "cancelledCount": 1, "judgedCount": 1, "passedCount": 1, "maskedCount": 1, "avgScore": 0.75, "heldOutJudgedCount": 1, "heldOutAvgScore": 0.75, "costUsd": 0.75, "resolvedMismatch": true, "resolvedUnknownCount": 1, "costUnknownReason": "ledger_not_final", "startedAt": "2026-10-14T09:30:00.000Z", "completedAt": "2026-10-14T09:30:00.000Z", "error": "text", "stopRequestedAt": "2026-10-14T09:30:00.000Z", "stopRequestedByUserId": "c772e75e-d5d2-4cf6-8a64-846bcc9144d5", "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z" } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/runs/$RUN_ID\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nrun_id = os.environ[\"RUN_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/runs/{run_id}\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/runs/${encodeURIComponent(env(\"RUN_ID\"))}`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Where a run is, and the cell ids `run tasks` and `run task` take\ncontext evals benchmark run get $RUN_ID | jq '{status, cells: [.cells[] | {cellId, value, attempt}]}'" } ] } }, "/runs/{runId}/status": { "get": { "tags": [ "Runs" ], "summary": "Get a run's status", "description": "The run as the evaluation's Runs table shows it (its result, each cell's result, the best of them, how many cells are at or above the threshold, cost per task, criteria that errored), plus who started it, the evaluation's current threshold, the per-cell stats (total, succeeded, failed, mean score), and the tasks waiting now, by reason. For per-task rows, use `GET /runs/{runId}/tasks`; for one task's full record, use `GET /runs/{runId}/tasks/{taskId}`.", "operationId": "getRunStatus", "parameters": [ { "in": "path", "name": "runId", "required": true, "description": "Run identifier", "schema": { "type": "string", "minLength": 1, "description": "Run identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "run": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique run identifier" }, "orgId": { "type": "string", "minLength": 1, "description": "Organization that owns this run" }, "evaluationId": { "type": "string", "minLength": 1, "description": "Evaluation being run" }, "topicId": { "type": [ "string", "null" ], "minLength": 1, "description": "Workflow this run belongs to; null for a run started from the app's evaluations view" }, "evaluationVersionId": { "type": [ "string", "null" ], "minLength": 1, "description": "Evaluation version this run uses; null only for runs created before evaluation versions existed" }, "runById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who started this run" }, "axes": { "anyOf": [ { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "model" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "A model the held agent runs" }, "minItems": 1, "description": "The models to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "skillId" ], "additionalProperties": false }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A held skill as fixed by the run. A skill ID on its own means the run started before versions were fixed." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "agentId", "skills" ], "additionalProperties": false, "description": "The agent every model ran on, and the skills it ran with, each at its fixed version" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Model: one agent and its skills are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "skill" ] }, "values": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill" ] }, "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "kind", "skillId" ], "additionalProperties": false, "description": "A tested skill as fixed by the run: the version it runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "no_skill" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The held agent with no skill attached: the baseline a skill is compared against" } ] }, "minItems": 1, "description": "The skills to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates (two versions of one skill count as two values)" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" } }, "required": [ "agentId", "model" ], "additionalProperties": false, "description": "The agent every skill runs on, and the model it runs" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Skill: one agent and its model are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "agent" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent" ] }, "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" } }, "required": [ "kind", "agentId" ], "additionalProperties": false, "description": "An agent" }, "minItems": 1, "description": "The agents to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "skillId" ], "additionalProperties": false }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A held skill as fixed by the run. A skill ID on its own means the run started before versions were fixed." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "model", "skills" ], "additionalProperties": false, "description": "The model every agent ran, and the skills they ran with, each at its fixed version" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Agent: the model and the skills are held fixed" } ], "description": "The axes a run executed, fixed when it started, with every model resolved and every skill version fixed. Reports use these, not the evaluation's current settings." }, "cells": { "type": "array", "items": { "type": "object", "properties": { "cellId": { "type": "string", "format": "uuid", "description": "Identifies one cell within its run. A run's results, dispatches, and exports are keyed by it." }, "value": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "A model the held agent runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill" ] }, "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "kind", "skillId" ], "additionalProperties": false, "description": "A tested skill as fixed by the run: the version it runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "no_skill" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The held agent with no skill attached: the baseline a skill is compared against" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent" ] }, "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" } }, "required": [ "kind", "agentId" ], "additionalProperties": false, "description": "An agent" } ], "description": "The tested value this cell ran: one of the run's `axes.values`" }, "attempt": { "type": "integer", "minimum": 1, "maximum": 5, "description": "Which repeat of that value this is, starting at 1 (shown as \"Run n\"); a run with one run per task has only 1" } }, "required": [ "cellId", "value", "attempt" ], "additionalProperties": false, "description": "One (value x repeat) of a run. Every task in the run runs once per cell." }, "minItems": 1, "description": "The run's cells, ordered by value (in the axes' order), then by repeat within each value: number of values x runs per task" }, "status": { "type": "string", "enum": [ "PENDING", "RUNNING", "EVALUATING", "COMPLETED", "FAILED", "STOPPED", "UNKNOWN" ], "description": "Current run status, or `UNKNOWN` when the stored value is not one of the allowed values" }, "taskCount": { "type": "integer", "minimum": 0, "description": "Total number of (task x cell) pairs to run" }, "completedCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs completed so far" }, "concurrency": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Tasks run at once for this run (min 1, max 100 or the deployment's lower limit)." }, "judgeConcurrency": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Agent judges run at once for this run, across every task (min 1, max 100 or the deployment's lower limit). LLM judges and rules do not count." }, "succeededCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as succeeded" }, "failedCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as failed" }, "cancelledCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as canceled" }, "judgedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Result rows with a score: a judge score and no task error, or, on a criteria-graded run, a task where at least one criterion reached a verdict. Null until the run's totals are computed." }, "passedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Tasks the judge passed. Null for an evaluation scored on a 0 to 1 scale (no pass or fail), and null until the run's totals are computed." }, "maskedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Result rows left out of the score: the judge gave no score or the task errored, or, on a criteria-graded run, none of the task's criteria reached a verdict (they could not run). Null until the run's totals are computed." }, "avgScore": { "type": [ "number", "null" ], "description": "Mean judged score on the standard 0 to 1 scale. Null until the run's totals are computed, or when nothing was judged." }, "heldOutJudgedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "`judgedCount` for the run's held-out tasks only (tasks the system never learns from). Null until the run's totals are computed, or when the totals come from older stored results that do not record which tasks are held out." }, "heldOutAvgScore": { "type": [ "number", "null" ], "description": "`avgScore` for the run's held-out tasks only, on the same 0 to 1 scale. Null when `heldOutJudgedCount` is null or 0." }, "costUsd": { "type": [ "number", "null" ], "description": "Task cost plus judge cost for the whole run, in dollars (USD). Null when any part is unknown (never a partial sum)." }, "resolvedMismatch": { "type": "boolean", "description": "True when any dispatch was served by a different model than its cell asked for. This is a notice only: the run's result and every list, trend, and chart still show its score." }, "resolvedUnknownCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Dispatches whose serving model could not be identified from billing when the run finished. This is not a mismatch, only unknown, and it never holds back the run's result. Null until the run's totals are computed." }, "costUnknownReason": { "type": [ "string", "null" ], "enum": [ "ledger_not_final", "device_calls_unbounded", "no_rows", "unrated_rows", "warehouse_session_missing", "judge_cost_unknown", "ledger_reroll_failed", null ], "description": "Why `costUsd` is null on an ended run, by order of precedence. Null when the cost is known, while the run is still going, and on an ended run whose totals have not been computed." }, "startedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When the run started" }, "completedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When the run finished, whether it succeeded or failed" }, "error": { "type": [ "string", "null" ], "description": "Error message if the run failed" }, "stopRequestedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When someone asked to stop the run, or null if nobody did. It is set while the run is still going: the run counts as stopping until its status is `STOPPED`." }, "stopRequestedByUserId": { "type": [ "string", "null" ], "minLength": 1, "description": "Who asked to stop the run: the person, or the person an agent acts for when an agent asked. Null when nobody did." }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" }, "runByName": { "type": [ "string", "null" ], "description": "Display name of who started the run; null for a run the system started or when that user no longer exists" }, "result": { "anyOf": [ { "type": "object", "properties": { "state": { "type": "string", "enum": [ "available" ] }, "mode": { "type": "string", "enum": [ "pass", "score" ], "description": "Whether the result is a boolean pass rate or a scaled score" }, "valuePct": { "type": "number", "minimum": 0, "maximum": 100, "description": "The result as a percentage from 0 to 100" } }, "required": [ "state", "mode", "valuePct" ], "additionalProperties": false }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "unavailable" ] }, "reason": { "type": "string", "enum": [ "not-scored", "invalid-rollup", "version-unavailable" ], "description": "Why a completed run cannot report a comparable result" } }, "required": [ "state", "reason" ], "additionalProperties": false }, { "type": "null" } ], "description": "The result of a completed run; null while the run has not completed or if it failed" }, "headlinePct": { "type": [ "number", "null" ], "minimum": 0, "maximum": 100, "description": "The run's headline figure, as the evaluations list shows it (its card's headline and trend point): the best cell for a run graded by criteria, otherwise `result.valuePct`. Null when `result` is not available." }, "heldOutResult": { "type": [ "object", "null" ], "properties": { "valuePct": { "type": "number", "minimum": 0, "maximum": 100, "description": "The result over the held-out tasks, on the same scale as `result.valuePct`" }, "rowCount": { "type": "integer", "minimum": 0, "description": "Number of held-out tasks the run graded" } }, "required": [ "valuePct", "rowCount" ], "additionalProperties": false, "description": "The run's result over its held-out tasks only (tasks the system never learns from), on the same scale as `result.valuePct`; null when the run has no available result or graded no held-out task" }, "cellResultsAvailable": { "type": [ "boolean", "null" ], "description": "Null when the run has no available result; false when its per-cell results cannot be read" }, "cellResults": { "type": "array", "items": { "type": "object", "properties": { "cellId": { "type": "string", "format": "uuid", "description": "One of the run's cells" }, "valuePct": { "type": [ "number", "null" ], "minimum": 0, "maximum": 100, "description": "This cell's result on the run's scale (a mean score for a run graded by criteria; a pass rate when the recorded results are pass/fail); null when nothing in the cell was graded" } }, "required": [ "cellId", "valuePct" ], "additionalProperties": false, "description": "One cell's result within a completed run" }, "description": "One result per cell, in the order of the run's cells; empty unless `cellResultsAvailable` is true" }, "bestScorePct": { "type": [ "number", "null" ], "minimum": 0, "maximum": 100, "description": "The best cell's result; null when no cell result is available" }, "aboveThresholdCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Number of cells whose result is at or above the evaluation's current threshold; null when the evaluation has no threshold or no cell result is available" }, "costPerTaskUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "The run's final cost divided by its (task x cell) pairs, given only when every pair ran (the run completed and the platform canceled none). Null for a failed or stopped run, a run with a canceled pair, a run still in progress, or an unknown cost. `costUsd` is still the run's total spend." }, "erroredCheckCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Number of checks that could not run across the run's settled tasks, in all cells: outcomes with no verdict (never started, files or sandbox unavailable, crashed, timed out). Each is shown as failed and left out of every score. Subtract `taskEnds.criteriaTaken` to get the judge's own failures, which the Runs table shows as \"couldn't be judged.\" 0 when there are none, including on a completed run that scored nothing. Null for a run that did not complete, a completed run whose cell details are unknown, or an older run that recorded no checks." }, "taskEnds": { "type": [ "object", "null" ], "properties": { "agentFailed": { "type": "integer", "minimum": 0, "description": "Tasks whose agent run failed (criteria recorded `subject_failed`)" }, "agentTimedOut": { "type": "integer", "minimum": 0, "description": "Tasks that hit the agent's working-time limit (criteria recorded `subject_timed_out`)" }, "notStarted": { "type": "object", "properties": { "sandbox": { "type": "integer", "minimum": 0 }, "ai_budget": { "type": "integer", "minimum": 0 }, "not_picked_up": { "type": "integer", "minimum": 0 }, "not_ready": { "type": "integer", "minimum": 0 }, "credits_exhausted": { "type": "integer", "minimum": 0, "default": 0 }, "user_budget_exceeded": { "type": "integer", "minimum": 0, "default": 0 } }, "required": [ "sandbox", "ai_budget", "not_picked_up", "not_ready" ], "additionalProperties": false, "description": "Tasks that never started, by reason: withdrawn at the run's start limit, or refused at dispatch for budget reasons (criteria recorded `not_started`)" }, "stopped": { "type": "integer", "minimum": 0, "description": "Tasks cut short because a person stopped the run (a `STOPPED` run; criteria recorded `cancelled`). These are not failures." }, "criteriaTaken": { "type": "integer", "minimum": 0, "description": "Number of criteria on the failed, timed-out, and never-started tasks, each of which recorded how the task ended instead of a verdict. Subtract this from a no-verdict check count (one that already excludes a person's stop) to get the judge's own failures." } }, "required": [ "agentFailed", "agentTimedOut", "notStarted", "stopped", "criteriaTaken" ], "additionalProperties": false, "description": "How the run's settled tasks that gave no answer ended, across all cells, counted once per task: the agent's task failed, the agent ran out of time, the task never started (by reason), or a person's stop cut it short. All zeros when every task answered. Present on a completed run that scored nothing, where it explains why. Null for a run that did not complete, a completed run whose cell details are unknown, or an older run that recorded no task endings." }, "stopRequestedByName": { "type": [ "string", "null" ], "description": "Display name of the person who asked to stop this run; null when nobody did or that user cannot be found" } }, "required": [ "id", "orgId", "evaluationId", "topicId", "evaluationVersionId", "runById", "axes", "cells", "taskCount", "completedCount", "concurrency", "judgeConcurrency", "succeededCount", "failedCount", "cancelledCount", "judgedCount", "passedCount", "maskedCount", "avgScore", "heldOutJudgedCount", "heldOutAvgScore", "costUsd", "resolvedMismatch", "resolvedUnknownCount", "costUnknownReason", "startedAt", "completedAt", "error", "stopRequestedAt", "stopRequestedByUserId", "createdAt", "updatedAt", "runByName", "result", "headlinePct", "heldOutResult", "cellResultsAvailable", "cellResults", "bestScorePct", "aboveThresholdCount", "costPerTaskUsd", "erroredCheckCount", "taskEnds", "stopRequestedByName" ], "additionalProperties": false, "description": "The run's row as the Runs table shows it, with the display names of who started it and who asked to stop it" }, "threshold": { "type": [ "integer", "null" ], "minimum": 0, "maximum": 100, "description": "The evaluation's current pass threshold, which `run.aboveThresholdCount` counts against. A cell passes when its `run.cellResults` `valuePct` is at or above it. Null when the evaluation has no threshold." }, "stats": { "type": "array", "items": { "type": "object", "properties": { "cellId": { "type": "string", "description": "The run cell these results belong to; the run's `cells` says which value and repeat it is" }, "total": { "type": "integer", "minimum": 0, "description": "Total tasks run" }, "succeeded": { "type": "integer", "minimum": 0, "description": "Tasks that completed successfully" }, "failed": { "type": "integer", "minimum": 0, "description": "Tasks that failed" }, "avgScore": { "type": [ "number", "null" ], "description": "Average judge score (null until there are scores)" } }, "required": [ "cellId", "total", "succeeded", "failed", "avgScore" ], "additionalProperties": false, "description": "Aggregated stats for one cell of a run" }, "description": "Per-cell aggregate stats for this run, one entry per cell, from its settled tasks (the same figures `GET /runs/{runId}/tasks` reports)" }, "waitingCounts": { "type": "object", "properties": { "sandbox": { "type": "integer", "minimum": 0, "description": "Cells waiting for a sandbox to run in" }, "aiBudget": { "type": "integer", "minimum": 0, "description": "Cells on hold because the AI budget is used up" }, "judgeSlot": { "type": "integer", "minimum": 0, "description": "Cells whose agent judges are waiting for a free agent judge slot in the run" } }, "required": [ "sandbox", "aiBudget", "judgeSlot" ], "additionalProperties": false, "description": "Cells of the run waiting now, by reason (the same figures `GET /runs/{runId}/tasks` reports)" } }, "required": [ "run", "threshold", "stats", "waitingCounts" ], "additionalProperties": false, "description": "Run status: the run's row as the Runs table shows it, the threshold it is measured against, its aggregate stats, and its waiting counts" }, "example": { "run": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "evaluationId": "66248a43-3af8-4b5b-86b6-fcb59feb38b2", "topicId": "c42387f0-94d3-427e-80d1-a63853a68624", "evaluationVersionId": "66fb5178-67cd-4706-8327-e5f646891209", "runById": "97bf5c87-0d64-4f5b-8600-9aec988e1e1c", "axes": { "testing": "model", "values": [ { "kind": "model", "model": "anthropic/claude-sonnet-5" } ], "held": { "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "skills": [ { "skillId": "3bb7a9ef-8bba-4c48-8bc8-0cccec5c0a00", "version": 1, "versionId": "772c657e-ac9c-4550-8ecf-9f77201cbb95", "followsLatest": true } ] } }, "cells": [ { "cellId": "5ec92200-48c8-4c37-8bf0-a28aa3d08fcd", "value": { "kind": "model", "model": "anthropic/claude-sonnet-5" }, "attempt": 1 } ], "status": "PENDING", "taskCount": 1, "completedCount": 1, "concurrency": 1, "judgeConcurrency": 1, "succeededCount": 1, "failedCount": 1, "cancelledCount": 1, "judgedCount": 1, "passedCount": 1, "maskedCount": 1, "avgScore": 0.75, "heldOutJudgedCount": 1, "heldOutAvgScore": 0.75, "costUsd": 0.75, "resolvedMismatch": true, "resolvedUnknownCount": 1, "costUnknownReason": "ledger_not_final", "startedAt": "2026-10-14T09:30:00.000Z", "completedAt": "2026-10-14T09:30:00.000Z", "error": "text", "stopRequestedAt": "2026-10-14T09:30:00.000Z", "stopRequestedByUserId": "c772e75e-d5d2-4cf6-8a64-846bcc9144d5", "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z", "runByName": "text", "result": { "state": "available", "mode": "pass", "valuePct": 1.5 }, "headlinePct": 1.5, "heldOutResult": { "valuePct": 1.5, "rowCount": 1 }, "cellResultsAvailable": true, "cellResults": [ { "cellId": "5ec92200-48c8-4c37-8bf0-a28aa3d08fcd", "valuePct": 1.5 } ], "bestScorePct": 1.5, "aboveThresholdCount": 1, "costPerTaskUsd": 0.75, "erroredCheckCount": 1, "taskEnds": { "agentFailed": 1, "agentTimedOut": 1, "notStarted": { "sandbox": 1, "ai_budget": 1, "not_picked_up": 1, "not_ready": 1, "credits_exhausted": 1, "user_budget_exceeded": 1 }, "stopped": 1, "criteriaTaken": 1 }, "stopRequestedByName": "text" }, "threshold": 1, "stats": [ { "cellId": "5ec92200-48c8-4c37-8bf0-a28aa3d08fcd", "total": 1, "succeeded": 1, "failed": 1, "avgScore": 0.75 } ], "waitingCounts": { "sandbox": 1, "aiBudget": 1, "judgeSlot": 1 } } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/runs/$RUN_ID/status\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nrun_id = os.environ[\"RUN_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/runs/{run_id}/status\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/runs/${encodeURIComponent(env(\"RUN_ID\"))}/status`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/runs/{runId}/tasks": { "get": { "tags": [ "Runs" ], "summary": "List a run's task results", "description": "The tasks of one cell in the run's order, each with its score, its criteria outcomes (pending, completed with a 0 to 1 score, error, or didn't run), a preview of its recorded answer, its cost and latency, and why it is not running when it is not. Also includes the run's whole-run figures (stats per cell, waiting counts). To page, pass the previous page's `nextCursor` as `cursor`; a cursor outside the run's pinned task set is rejected.", "operationId": "listRunTasks", "parameters": [ { "in": "path", "name": "runId", "required": true, "description": "Run identifier", "schema": { "type": "string", "minLength": 1, "description": "Run identifier" } }, { "in": "query", "name": "cellId", "required": true, "schema": { "type": "string", "minLength": 1, "description": "One of this run's cells" } }, { "in": "query", "name": "cursor", "required": false, "schema": { "type": "string", "minLength": 1, "description": "Last task identifier from the previous page of this run (its `nextCursor`)" } }, { "in": "query", "name": "limit", "required": false, "schema": { "type": "integer", "minimum": 1, "maximum": 50, "default": 50, "description": "Tasks per page" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "run": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique run identifier" }, "orgId": { "type": "string", "minLength": 1, "description": "Organization that owns this run" }, "evaluationId": { "type": "string", "minLength": 1, "description": "Evaluation being run" }, "topicId": { "type": [ "string", "null" ], "minLength": 1, "description": "Workflow this run belongs to; null for a run started from the app's evaluations view" }, "evaluationVersionId": { "type": [ "string", "null" ], "minLength": 1, "description": "Evaluation version this run uses; null only for runs created before evaluation versions existed" }, "runById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who started this run" }, "axes": { "anyOf": [ { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "model" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "A model the held agent runs" }, "minItems": 1, "description": "The models to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "skillId" ], "additionalProperties": false }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A held skill as fixed by the run. A skill ID on its own means the run started before versions were fixed." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "agentId", "skills" ], "additionalProperties": false, "description": "The agent every model ran on, and the skills it ran with, each at its fixed version" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Model: one agent and its skills are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "skill" ] }, "values": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill" ] }, "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "kind", "skillId" ], "additionalProperties": false, "description": "A tested skill as fixed by the run: the version it runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "no_skill" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The held agent with no skill attached: the baseline a skill is compared against" } ] }, "minItems": 1, "description": "The skills to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates (two versions of one skill count as two values)" }, "held": { "type": "object", "properties": { "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" }, "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" } }, "required": [ "agentId", "model" ], "additionalProperties": false, "description": "The agent every skill runs on, and the model it runs" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Skill: one agent and its model are held fixed" }, { "type": "object", "properties": { "testing": { "type": "string", "enum": [ "agent" ] }, "values": { "type": "array", "items": { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent" ] }, "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" } }, "required": [ "kind", "agentId" ], "additionalProperties": false, "description": "An agent" }, "minItems": 1, "description": "The agents to run, in display order: at least 1 (one measures, two or more compare), no maximum, no duplicates" }, "held": { "type": "object", "properties": { "model": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent_default" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "Each agent runs its own default model. The model that served a task is recorded on its dispatch." }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "One model that every agent in the evaluation runs" } ], "description": "The model held fixed when Model is not the tested axis" }, "skills": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "skillId" ], "additionalProperties": false }, { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" } ], "description": "A held skill as fixed by the run. A skill ID on its own means the run started before versions were fixed." }, "maxItems": 20, "description": "Skills every cell runs with when Skill is not the tested axis (at most 20, no duplicates, each at one version); empty for none" } }, "required": [ "model", "skills" ], "additionalProperties": false, "description": "The model every agent ran, and the skills they ran with, each at its fixed version" } }, "required": [ "testing", "values", "held" ], "additionalProperties": false, "description": "Testing Agent: the model and the skills are held fixed" } ], "description": "The axes a run executed, fixed when it started, with every model resolved and every skill version fixed. Reports use these, not the evaluation's current settings." }, "cells": { "type": "array", "items": { "type": "object", "properties": { "cellId": { "type": "string", "format": "uuid", "description": "Identifies one cell within its run. A run's results, dispatches, and exports are keyed by it." }, "value": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "model" ] }, "model": { "type": "string", "minLength": 1, "maxLength": 200, "description": "A model as `/` (with a gateway prefix or direct), at most 200 characters. Checked against the running agent's models when a run starts; a model the agent cannot run is refused then." } }, "required": [ "kind", "model" ], "additionalProperties": false, "description": "A model the held agent runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill" ] }, "skillId": { "type": "string", "minLength": 1, "description": "A skill in the evaluation's organization" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "The skill version to run, by number. When absent, the skill's newest version runs, fixed when a run starts (shown as 'Latest')." }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the skill version this run uses. Present exactly when `version` is present." }, "followsLatest": { "type": "boolean", "enum": [ true ], "description": "Present when the evaluation asked for the newest version and this run fixed it when the run started; absent when the evaluation pinned the version" } }, "required": [ "kind", "skillId" ], "additionalProperties": false, "description": "A tested skill as fixed by the run: the version it runs" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "no_skill" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The held agent with no skill attached: the baseline a skill is compared against" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "agent" ] }, "agentId": { "type": "string", "minLength": 1, "description": "An agent in the evaluation's organization" } }, "required": [ "kind", "agentId" ], "additionalProperties": false, "description": "An agent" } ], "description": "The tested value this cell ran: one of the run's `axes.values`" }, "attempt": { "type": "integer", "minimum": 1, "maximum": 5, "description": "Which repeat of that value this is, starting at 1 (shown as \"Run n\"); a run with one run per task has only 1" } }, "required": [ "cellId", "value", "attempt" ], "additionalProperties": false, "description": "One (value x repeat) of a run. Every task in the run runs once per cell." }, "minItems": 1, "description": "The run's cells, ordered by value (in the axes' order), then by repeat within each value: number of values x runs per task" }, "status": { "type": "string", "enum": [ "PENDING", "RUNNING", "EVALUATING", "COMPLETED", "FAILED", "STOPPED", "UNKNOWN" ], "description": "Current run status, or `UNKNOWN` when the stored value is not one of the allowed values" }, "taskCount": { "type": "integer", "minimum": 0, "description": "Total number of (task x cell) pairs to run" }, "completedCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs completed so far" }, "concurrency": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Tasks run at once for this run (min 1, max 100 or the deployment's lower limit)." }, "judgeConcurrency": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Agent judges run at once for this run, across every task (min 1, max 100 or the deployment's lower limit). LLM judges and rules do not count." }, "succeededCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as succeeded" }, "failedCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as failed" }, "cancelledCount": { "type": "integer", "minimum": 0, "description": "Number of (task x cell) pairs whose task ended as canceled" }, "judgedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Result rows with a score: a judge score and no task error, or, on a criteria-graded run, a task where at least one criterion reached a verdict. Null until the run's totals are computed." }, "passedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Tasks the judge passed. Null for an evaluation scored on a 0 to 1 scale (no pass or fail), and null until the run's totals are computed." }, "maskedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Result rows left out of the score: the judge gave no score or the task errored, or, on a criteria-graded run, none of the task's criteria reached a verdict (they could not run). Null until the run's totals are computed." }, "avgScore": { "type": [ "number", "null" ], "description": "Mean judged score on the standard 0 to 1 scale. Null until the run's totals are computed, or when nothing was judged." }, "heldOutJudgedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "`judgedCount` for the run's held-out tasks only (tasks the system never learns from). Null until the run's totals are computed, or when the totals come from older stored results that do not record which tasks are held out." }, "heldOutAvgScore": { "type": [ "number", "null" ], "description": "`avgScore` for the run's held-out tasks only, on the same 0 to 1 scale. Null when `heldOutJudgedCount` is null or 0." }, "costUsd": { "type": [ "number", "null" ], "description": "Task cost plus judge cost for the whole run, in dollars (USD). Null when any part is unknown (never a partial sum)." }, "resolvedMismatch": { "type": "boolean", "description": "True when any dispatch was served by a different model than its cell asked for. This is a notice only: the run's result and every list, trend, and chart still show its score." }, "resolvedUnknownCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Dispatches whose serving model could not be identified from billing when the run finished. This is not a mismatch, only unknown, and it never holds back the run's result. Null until the run's totals are computed." }, "costUnknownReason": { "type": [ "string", "null" ], "enum": [ "ledger_not_final", "device_calls_unbounded", "no_rows", "unrated_rows", "warehouse_session_missing", "judge_cost_unknown", "ledger_reroll_failed", null ], "description": "Why `costUsd` is null on an ended run, by order of precedence. Null when the cost is known, while the run is still going, and on an ended run whose totals have not been computed." }, "startedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When the run started" }, "completedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When the run finished, whether it succeeded or failed" }, "error": { "type": [ "string", "null" ], "description": "Error message if the run failed" }, "stopRequestedAt": { "type": [ "string", "null" ], "format": "date-time", "description": "When someone asked to stop the run, or null if nobody did. It is set while the run is still going: the run counts as stopping until its status is `STOPPED`." }, "stopRequestedByUserId": { "type": [ "string", "null" ], "minLength": 1, "description": "Who asked to stop the run: the person, or the person an agent acts for when an agent asked. Null when nobody did." }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" }, "runByName": { "type": [ "string", "null" ], "description": "Display name of the user who started this run, or null if `runById` is null or the user cannot be found" }, "stopRequestedByName": { "type": [ "string", "null" ], "description": "Display name of the person who asked to stop this run; null when nobody did or that user cannot be found" } }, "required": [ "id", "orgId", "evaluationId", "topicId", "evaluationVersionId", "runById", "axes", "cells", "taskCount", "completedCount", "concurrency", "judgeConcurrency", "succeededCount", "failedCount", "cancelledCount", "judgedCount", "passedCount", "maskedCount", "avgScore", "heldOutJudgedCount", "heldOutAvgScore", "costUsd", "resolvedMismatch", "resolvedUnknownCount", "costUnknownReason", "startedAt", "completedAt", "error", "stopRequestedAt", "stopRequestedByUserId", "createdAt", "updatedAt", "runByName", "stopRequestedByName" ], "additionalProperties": false, "description": "The run, with the display names of who started it and who asked to stop it" }, "version": { "anyOf": [ { "type": "object", "properties": { "state": { "type": "string", "enum": [ "legacy" ] } }, "required": [ "state" ], "additionalProperties": false, "description": "A run created before evaluation versions were pinned" }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "available" ] }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the immutable version the run is pinned to" }, "version": { "type": "integer", "exclusiveMinimum": 0, "description": "Evaluation version number; it only increases" }, "taskCount": { "type": "integer", "minimum": 0, "description": "Number of tasks fixed in this version's task set" }, "judgeOutputType": { "type": "string", "enum": [ "float", "boolean" ], "description": "Format this version's results were recorded in" }, "sample": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "all" ] } }, "required": [ "kind" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "sample" ] }, "size": { "type": "integer", "exclusiveMinimum": 0, "description": "Number of tasks drawn (equals `caseSet.itemCount`)" }, "seed": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Unsigned 32-bit seed; the same seed over the same tasks draws the same sample" }, "populationCount": { "type": "integer", "exclusiveMinimum": 0, "description": "Number of tasks the sample was drawn from" } }, "required": [ "kind", "size", "seed", "populationCount" ], "additionalProperties": false } ], "description": "Whether the run measured every task or a seeded sample (`taskCount` is the sample size)" }, "datasets": { "anyOf": [ { "type": "object", "properties": { "state": { "type": "string", "enum": [ "recorded" ] }, "items": { "type": "array", "items": { "type": "object", "properties": { "datasetId": { "type": "string", "minLength": 1 }, "datasetName": { "type": [ "string", "null" ], "description": "Null when the dataset has since been deleted (its versions remain)" }, "version": { "type": "integer", "exclusiveMinimum": 0 }, "taskCount": { "type": "integer", "minimum": 0, "description": "Tasks in that version" } }, "required": [ "datasetId", "datasetName", "version", "taskCount" ], "additionalProperties": false }, "description": "The version of each linked dataset that the run graded, in link order" } }, "required": [ "state", "items" ], "additionalProperties": false }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "working_copy_preview" ] }, "itemId": { "type": "string", "minLength": 1, "description": "The working copy of the task that the preview graded" } }, "required": [ "state", "itemId" ], "additionalProperties": false }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "unrecorded" ] } }, "required": [ "state" ], "additionalProperties": false, "description": "The run was created before dataset versions existed" } ], "description": "Which dataset versions a run graded" } }, "required": [ "state", "versionId", "version", "taskCount", "judgeOutputType", "sample", "datasets" ], "additionalProperties": false, "description": "The immutable version details the run used" }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "unavailable" ] }, "versionId": { "type": "string", "minLength": 1, "description": "ID of the pinned version whose record is missing" } }, "required": [ "state", "versionId" ], "additionalProperties": false, "description": "The run is pinned to a version whose record is missing" } ], "description": "State of the evaluation version pinned for this run" }, "resultContract": { "type": "string", "enum": [ "native", "legacy", "unavailable" ], "description": "Which recorded-result export this run supports" }, "pinnedCriteriaCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Number of criteria the run's version pins across its tasks; 0 for a run nothing could grade; null for a run not graded by criteria" }, "stats": { "type": "array", "items": { "type": "object", "properties": { "cellId": { "type": "string", "description": "The run cell these results belong to; the run's `cells` says which value and repeat it is" }, "total": { "type": "integer", "minimum": 0, "description": "Total tasks run" }, "succeeded": { "type": "integer", "minimum": 0, "description": "Tasks that completed successfully" }, "failed": { "type": "integer", "minimum": 0, "description": "Tasks that failed" }, "avgScore": { "type": [ "number", "null" ], "description": "Average judge score (null until there are scores)" }, "costPerTaskUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "Mean cost per settled task, including the agent and its checks; null until the run completes, when any charge is unknown, and for older runs" }, "p95LatencyMs": { "type": [ "number", "null" ], "minimum": 0, "description": "Agent latency that 95 percent of tasks beat across the whole cell; null when no task has a duration" }, "passedCount": { "type": [ "integer", "null" ], "minimum": 0, "description": "Number of tasks whose score is at or above the evaluation's current threshold; null when there is no threshold" }, "criterionCount": { "type": "integer", "minimum": 0, "description": "Number of distinct checks with a settled outcome on this cell (checks may be capped)" }, "criteria": { "type": "array", "items": { "type": "object", "properties": { "criterionKey": { "type": "string", "minLength": 1, "description": "`evaluation:`, `dataset:`, or `task:`, as on task rows" }, "name": { "type": "string", "minLength": 1, "description": "Check display name as fixed in the run's version" }, "scored": { "type": "integer", "minimum": 0, "description": "Settled tasks where this check completed: `passed` + `failed` + `graded`" }, "errored": { "type": "integer", "minimum": 0, "description": "Settled tasks where this check reached no verdict (it could not run) for a reason other than a person's stop. Shown as failed with its reason, and left out of `meanScore` and of the task's score." }, "stopped": { "type": "integer", "minimum": 0, "description": "Settled tasks where a person's stop cut this check short (the same reading as a task's `stoppedBeforeFinish`). Left out of `meanScore` like `errored`, and counted separately because a stop is not a failure. 0 on a run that was not stopped." }, "passed": { "type": "integer", "minimum": 0, "description": "Completed pass/fail verdicts that passed" }, "failed": { "type": "integer", "minimum": 0, "description": "Completed pass/fail verdicts that failed. A graded (0 to 1) verdict is never counted here, whatever its score." }, "graded": { "type": "integer", "minimum": 0, "description": "Completed graded verdicts (a 0 to 1 score); their credit is in `meanScore`, not in `passed` or `failed`" }, "meanScore": { "type": [ "number", "null" ], "minimum": 0, "maximum": 1, "description": "Mean 0 to 1 score over this check's verdicts: a passed verdict counts as 1, a failed one as 0, a graded one as its score. Outcomes with no verdict (errored) are left out. Null when no outcome reached a verdict." } }, "required": [ "criterionKey", "name", "scored", "errored", "stopped", "passed", "failed", "graded", "meanScore" ], "additionalProperties": false, "description": "One check's whole-run outcome for one cell" }, "maxItems": 50, "description": "The evaluation's, datasets', and tasks' criteria, by name; empty for older runs, which recorded none" }, "taskEnds": { "type": [ "object", "null" ], "properties": { "agentFailed": { "type": "integer", "minimum": 0, "description": "Tasks whose agent run failed (criteria recorded `subject_failed`)" }, "agentTimedOut": { "type": "integer", "minimum": 0, "description": "Tasks that hit the agent's working-time limit (criteria recorded `subject_timed_out`)" }, "notStarted": { "type": "object", "properties": { "sandbox": { "type": "integer", "minimum": 0 }, "ai_budget": { "type": "integer", "minimum": 0 }, "not_picked_up": { "type": "integer", "minimum": 0 }, "not_ready": { "type": "integer", "minimum": 0 }, "credits_exhausted": { "type": "integer", "minimum": 0, "default": 0 }, "user_budget_exceeded": { "type": "integer", "minimum": 0, "default": 0 } }, "required": [ "sandbox", "ai_budget", "not_picked_up", "not_ready" ], "additionalProperties": false, "description": "Tasks that never started, by reason: withdrawn at the run's start limit, or refused at dispatch for budget reasons (criteria recorded `not_started`)" }, "stopped": { "type": "integer", "minimum": 0, "description": "Tasks cut short because a person stopped the run (a `STOPPED` run; criteria recorded `cancelled`). These are not failures." }, "criteriaTaken": { "type": "integer", "minimum": 0, "description": "Number of criteria on the failed, timed-out, and never-started tasks, each of which recorded how the task ended instead of a verdict. Subtract this from a no-verdict check count (one that already excludes a person's stop) to get the judge's own failures." } }, "required": [ "agentFailed", "agentTimedOut", "notStarted", "stopped", "criteriaTaken" ], "additionalProperties": false, "description": "How this cell's settled tasks that gave no answer ended, counted once per task (the agent's task failed, the agent ran out of time, the task never started (by reason), or a person's stop cut it short). These are the same figures the Runs table row adds up. All zeros when every settled task answered; null for an older run, which recorded no task endings." } }, "required": [ "cellId", "total", "succeeded", "failed", "avgScore", "costPerTaskUsd", "p95LatencyMs", "passedCount", "criterionCount", "criteria", "taskEnds" ], "additionalProperties": false, "description": "Whole-run totals for one cell, including the per-criterion figures" }, "description": "Whole-run totals, one per cell of the run in cell order; never an average over the current page" }, "statsAvailable": { "type": "boolean", "description": "False when results for this run cannot be read" }, "rows": { "type": "array", "items": { "type": "object", "properties": { "taskId": { "type": "string", "minLength": 1 }, "position": { "type": [ "integer", "null" ], "exclusiveMinimum": 0, "description": "Position in the run's fixed task order; null for runs without an ordered version" }, "text": { "anyOf": [ { "type": "object", "properties": { "state": { "type": "string", "enum": [ "captured" ] }, "prompt": { "type": "object", "properties": { "text": { "type": "string", "maxLength": 512 }, "truncated": { "type": "boolean", "description": "The value is cut short; read the full value from `GET /runs/{runId}/tasks/{taskId}`" } }, "required": [ "text", "truncated" ], "additionalProperties": false }, "expected": { "anyOf": [ { "allOf": [ { "type": "object", "properties": { "text": { "type": "string", "maxLength": 512 }, "truncated": { "type": "boolean", "description": "The value is cut short; read the full value from `GET /runs/{runId}/tasks/{taskId}`" } }, "required": [ "text", "truncated" ], "additionalProperties": false } ] }, { "type": "null" } ] }, "expectedFileCount": { "type": "integer", "minimum": 0, "description": "Number of expected files (answer keys) the run pinned for this task, alongside its expected text. A task may expect files and no text." } }, "required": [ "state", "prompt", "expected", "expectedFileCount" ], "additionalProperties": false }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "unavailable" ] }, "reason": { "type": "string", "enum": [ "not_captured", "version_unavailable", "snapshot_missing" ] } }, "required": [ "state", "reason" ], "additionalProperties": false } ] }, "output": { "description": "Preview of the agent's recorded output, or null when not available", "anyOf": [ { "allOf": [ { "type": "object", "properties": { "text": { "type": "string", "maxLength": 512 }, "truncated": { "type": "boolean", "description": "The value is cut short; read the full value from `GET /runs/{runId}/tasks/{taskId}`" } }, "required": [ "text", "truncated" ], "additionalProperties": false } ] }, { "type": "null" } ] }, "outputStatus": { "type": "string", "enum": [ "answered", "pending", "unavailable", "didnt_run" ], "description": "Exact state of the agent command's answer; an empty answer is not a missing one. `didnt_run`: no answer came before the run ended, and none will (`notRunning` says why)." }, "criteriaAvailable": { "type": "boolean", "description": "False when this run did not record task checks or the pinned task record is missing" }, "criteria": { "type": "object", "properties": { "totalCount": { "type": "integer", "minimum": 0 }, "entries": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string", "minLength": 1 }, "owner": { "type": "string", "enum": [ "evaluation", "dataset", "task" ], "description": "An evaluation's verifiers grade every task it runs; a dataset's criteria grade every task in that dataset; a task's criteria grade only that task" }, "name": { "type": "string", "minLength": 1, "maxLength": 200 }, "kind": { "type": "string", "enum": [ "llm_judge", "agent_judge", "exact_match", "code_verifier", "rule" ] }, "weight": { "type": "number", "exclusiveMinimum": 0 }, "outcome": { "anyOf": [ { "type": "object", "properties": { "status": { "type": "string", "enum": [ "pending" ] } }, "required": [ "status" ], "additionalProperties": false, "description": "The task has not settled and the run is still going" }, { "type": "object", "properties": { "status": { "type": "string", "enum": [ "didnt_run" ] } }, "required": [ "status" ], "additionalProperties": false, "description": "The run ended before the task settled; the row's `notRunning` says why" }, { "type": "object", "properties": { "status": { "type": "string", "enum": [ "completed" ] }, "score": { "type": "number", "minimum": 0, "maximum": 1 } }, "required": [ "status", "score" ], "additionalProperties": false }, { "type": "object", "properties": { "status": { "type": "string", "enum": [ "error" ] }, "code": { "type": "string", "enum": [ "invalid_code", "invalid_verdict", "execution_failed", "timed_out", "cancelled", "input_unavailable", "output_limit", "runtime_unavailable", "batch_deadline", "inference_unavailable", "not_started", "malformed_output", "subject_failed", "subject_timed_out" ] }, "message": { "type": "string", "maxLength": 512, "description": "The error's recorded message, cut to the preview length (for example, the shared message for `runtime_unavailable`)" } }, "required": [ "status", "code", "message" ], "additionalProperties": false } ] } }, "required": [ "key", "owner", "name", "kind", "weight", "outcome" ], "additionalProperties": false, "description": "One of a task's checks with its outcome on this run" }, "maxItems": 8 } }, "required": [ "totalCount", "entries" ], "additionalProperties": false }, "recordedCriteria": { "type": [ "object", "null" ], "properties": { "total": { "type": "integer", "minimum": 0 }, "errored": { "type": "integer", "minimum": 0 } }, "required": [ "total", "errored" ], "additionalProperties": false, "description": "The outcomes this task's settled execution recorded, which make up its score; null until it settles. A run recorded while an evaluation could own checks holds more outcomes than the task has pinned checks." }, "dispatch": { "type": [ "object", "null" ], "properties": { "dispatchId": { "anyOf": [ { "allOf": [ { "type": "string", "minLength": 1 } ] }, { "type": "null" } ] }, "taskId": { "anyOf": [ { "allOf": [ { "type": "string", "minLength": 1 } ] }, { "type": "null" } ] }, "taskPath": { "type": [ "string", "null" ] }, "agentId": { "type": "string", "minLength": 1 }, "status": { "type": [ "string", "null" ] }, "error": { "anyOf": [ { "allOf": [ { "type": "object", "properties": { "text": { "type": "string", "maxLength": 512 }, "truncated": { "type": "boolean", "description": "The message is cut to 512 characters; this API has no longer copy." } }, "required": [ "text", "truncated" ], "additionalProperties": false } ] }, { "type": "null" } ] } }, "required": [ "dispatchId", "taskId", "taskPath", "agentId", "status", "error" ], "additionalProperties": false }, "score": { "type": [ "number", "null" ], "minimum": 0, "maximum": 1 }, "costUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "Set only when the full cost of the agent and its checks is known; otherwise null" }, "latencyMs": { "type": [ "number", "null" ], "minimum": 0, "description": "How long the agent took to complete, from its dispatch timestamps" }, "notRunning": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "waiting" ] }, "wait": { "anyOf": [ { "type": "object", "properties": { "since": { "type": "string", "format": "date-time", "description": "When the longest current wait began (ISO 8601)" }, "waiting": { "type": "array", "items": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "subject" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The cell's own task (the agent being evaluated) is still waiting for a sandbox to run in" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "check" ] }, "key": { "type": "string", "minLength": 1, "description": "The waiting judge check's verifier key, as on task rows" } }, "required": [ "kind", "key" ], "additionalProperties": false, "description": "An agent-judge check whose own task is still waiting for a sandbox to run in" } ], "description": "One part of a cell that is waiting for a sandbox to run in" }, "minItems": 1, "description": "Every part of the cell waiting now: the agent being evaluated first, then checks in the order they began waiting" }, "reason": { "type": "string", "enum": [ "sandbox" ] } }, "required": [ "since", "waiting", "reason" ], "additionalProperties": false, "description": "Waiting for a sandbox to run in: none was free when the agent or a judge check tried to start" }, { "type": "object", "properties": { "reason": { "type": "string", "enum": [ "ai_budget" ] }, "since": { "type": "string", "format": "date-time", "description": "When the budget refusal holding the cell's model was observed (ISO 8601)" } }, "required": [ "reason", "since" ], "additionalProperties": false, "description": "On hold: the AI budget is used up, so the agent's task is not started" }, { "type": "object", "properties": { "reason": { "type": "string", "enum": [ "judge_slot" ] }, "since": { "type": "string", "format": "date-time", "description": "When the cell's longest-waiting agent judge asked for its turn (ISO 8601)" } }, "required": [ "reason", "since" ], "additionalProperties": false, "description": "Judging, waiting for a free agent judge slot: the run already has as many agent judges running as it allows" } ], "description": "Why the cell is waiting now" } }, "required": [ "kind", "wait" ], "additionalProperties": false, "description": "The run is live and the task is waiting (for a sandbox to run in, or on hold because of the AI budget)" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "didnt_run" ] }, "reason": { "type": "object", "properties": { "text": { "type": "string", "maxLength": 512, "minLength": 1 }, "truncated": { "type": "boolean", "description": "The reason is cut to 512 characters; the run's whole error is `error` on `GET /runs/{runId}`." } }, "required": [ "text", "truncated" ], "additionalProperties": false, "description": "Why, as the run recorded it: the run's own error (cut to the preview length; the full run record has all of it), or a fixed sentence for a run that ended without one. Display it as is; do not parse it." } }, "required": [ "kind", "reason" ], "additionalProperties": false, "description": "Didn't run: the run ended before this task settled" }, { "type": "null" } ], "description": "Why the unsettled task is not running now: it is waiting (live run), or it didn't run (the run ended before it settled). Null while it runs, once it settles, and for runs not graded by criteria." }, "stoppedBeforeFinish": { "type": "boolean", "description": "A person stopped the run before this task finished: it didn't run, or it settled with what the stop canceled. Not a failure. False for a task that settled on its own (even with an error) and on a run that completed or failed." } }, "required": [ "taskId", "position", "text", "output", "outputStatus", "criteriaAvailable", "criteria", "recordedCriteria", "dispatch", "score", "costUsd", "latencyMs", "notRunning", "stoppedBeforeFinish" ], "additionalProperties": false }, "maxItems": 50 }, "totalCount": { "type": "integer", "minimum": 0, "description": "Number of tasks known to be in this run" }, "nextCursor": { "anyOf": [ { "allOf": [ { "type": "string", "minLength": 1 } ] }, { "type": "null" } ] }, "waitingCounts": { "type": "object", "properties": { "sandbox": { "type": "integer", "minimum": 0, "description": "Cells waiting for a sandbox to run in" }, "aiBudget": { "type": "integer", "minimum": 0, "description": "Cells on hold because the AI budget is used up" }, "judgeSlot": { "type": "integer", "minimum": 0, "description": "Cells whose agent judges are waiting for a free agent judge slot in the run" } }, "required": [ "sandbox", "aiBudget", "judgeSlot" ], "additionalProperties": false, "description": "Tasks across every cell of the run waiting now, by reason (alongside the run's settled and total figures)" } }, "required": [ "run", "version", "resultContract", "pinnedCriteriaCount", "stats", "statsAvailable", "rows", "totalCount", "nextCursor", "waitingCounts" ], "additionalProperties": false }, "example": { "run": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "evaluationId": "66248a43-3af8-4b5b-86b6-fcb59feb38b2", "topicId": "c42387f0-94d3-427e-80d1-a63853a68624", "evaluationVersionId": "66fb5178-67cd-4706-8327-e5f646891209", "runById": "97bf5c87-0d64-4f5b-8600-9aec988e1e1c", "axes": { "testing": "model", "values": [ { "kind": "model", "model": "anthropic/claude-sonnet-5" } ], "held": { "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "skills": [ { "skillId": "3bb7a9ef-8bba-4c48-8bc8-0cccec5c0a00", "version": 1, "versionId": "772c657e-ac9c-4550-8ecf-9f77201cbb95", "followsLatest": true } ] } }, "cells": [ { "cellId": "5ec92200-48c8-4c37-8bf0-a28aa3d08fcd", "value": { "kind": "model", "model": "anthropic/claude-sonnet-5" }, "attempt": 1 } ], "status": "PENDING", "taskCount": 1, "completedCount": 1, "concurrency": 1, "judgeConcurrency": 1, "succeededCount": 1, "failedCount": 1, "cancelledCount": 1, "judgedCount": 1, "passedCount": 1, "maskedCount": 1, "avgScore": 0.75, "heldOutJudgedCount": 1, "heldOutAvgScore": 0.75, "costUsd": 0.75, "resolvedMismatch": true, "resolvedUnknownCount": 1, "costUnknownReason": "ledger_not_final", "startedAt": "2026-10-14T09:30:00.000Z", "completedAt": "2026-10-14T09:30:00.000Z", "error": "text", "stopRequestedAt": "2026-10-14T09:30:00.000Z", "stopRequestedByUserId": "c772e75e-d5d2-4cf6-8a64-846bcc9144d5", "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z", "runByName": "text", "stopRequestedByName": "text" }, "version": { "state": "legacy" }, "resultContract": "native", "pinnedCriteriaCount": 1, "stats": [ { "cellId": "5ec92200-48c8-4c37-8bf0-a28aa3d08fcd", "total": 1, "succeeded": 1, "failed": 1, "avgScore": 0.75, "costPerTaskUsd": 0.75, "p95LatencyMs": 0.75, "passedCount": 1, "criterionCount": 1, "criteria": [ { "criterionKey": "text", "name": "Refund questions", "scored": 1, "errored": 1, "stopped": 1, "passed": 1, "failed": 1, "graded": 1, "meanScore": 0.75 } ], "taskEnds": { "agentFailed": 1, "agentTimedOut": 1, "notStarted": { "sandbox": 1, "ai_budget": 1, "not_picked_up": 1, "not_ready": 1, "credits_exhausted": 1, "user_budget_exceeded": 1 }, "stopped": 1, "criteriaTaken": 1 } } ], "statsAvailable": true, "rows": [ { "taskId": "de707028-6ea2-4616-8864-c2ebd92b87f5", "position": 1, "text": { "state": "captured", "prompt": { "text": "text", "truncated": true }, "expected": { "text": "text", "truncated": true }, "expectedFileCount": 1 }, "output": { "text": "text", "truncated": true }, "outputStatus": "answered", "criteriaAvailable": true, "criteria": { "totalCount": 1, "entries": [ { "key": "text", "owner": "evaluation", "name": "Refund questions", "kind": "llm_judge", "weight": 0.5, "outcome": { "status": "pending" } } ] }, "recordedCriteria": { "total": 1, "errored": 1 }, "dispatch": { "dispatchId": "6a7d95aa-ff7a-4b7e-85ef-c4dbe5728571", "taskId": "de707028-6ea2-4616-8864-c2ebd92b87f5", "taskPath": "text", "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "status": "text", "error": { "text": "text", "truncated": true } }, "score": 0.75, "costUsd": 0.75, "latencyMs": 0.75, "notRunning": { "kind": "waiting", "wait": { "since": "2026-10-14T09:30:00.000Z", "waiting": [ { "kind": "subject" } ], "reason": "sandbox" } }, "stoppedBeforeFinish": true } ], "totalCount": 1, "nextCursor": "text", "waitingCounts": { "sandbox": 1, "aiBudget": 1, "judgeSlot": 1 } } } } }, "400": { "description": "A parameter is missing or invalid (`error` is `Invalid request`, with `details`); or `cellId`, `cursor` or `taskId` is not part of this run (`error` says which; no `details`, no `code`).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/runs/$RUN_ID/tasks\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode \"cellId=$CELL_ID\" \\\n --data-urlencode 'limit=50'" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nrun_id = os.environ[\"RUN_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/runs/{run_id}/tasks\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"cellId\": os.environ[\"CELL_ID\"], \"limit\": 50},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/runs/${encodeURIComponent(env(\"RUN_ID\"))}/tasks`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"cellId\", env(\"CELL_ID\"))\nurl.searchParams.set(\"limit\", \"50\")\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Each task's score and the criteria it did not fully pass, on one cell (page with --cursor )\ncontext evals benchmark run tasks $RUN_ID --cell $CELL_ID | jq '.rows[] | {taskId, score, failed: [.criteria.entries[] | select(.outcome.status == \"completed\" and .outcome.score < 1) | .name]}'" } ] } }, "/runs/{runId}/tasks/{taskId}": { "get": { "tags": [ "Runs" ], "summary": "Get one task's result", "description": "The task's full record, as the full export writes it: the pinned task text, the complete recorded answer (or why there is none), every criterion's settled outcome with its reasoning, the task's cost, its criteria cost, and its latency. A task outside the run's pinned task set is rejected.", "operationId": "getRunTask", "parameters": [ { "in": "path", "name": "runId", "required": true, "description": "Run identifier", "schema": { "type": "string", "minLength": 1, "description": "Run identifier" } }, { "in": "path", "name": "taskId", "required": true, "description": "Task identifier", "schema": { "type": "string", "minLength": 1, "description": "A task in this run's pinned task set" } }, { "in": "query", "name": "cellId", "required": true, "schema": { "type": "string", "minLength": 1, "description": "One of this run's cells" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "taskId": { "type": "string", "minLength": 1 }, "cellId": { "type": "string", "minLength": 1 }, "position": { "type": "integer", "exclusiveMinimum": 0 }, "task": { "anyOf": [ { "type": "object", "properties": { "state": { "type": "string", "enum": [ "captured" ] }, "prompt": { "type": "string" }, "expected": { "type": [ "string", "null" ] }, "expectedFileCount": { "type": "integer", "minimum": 0, "description": "Number of expected files (answer keys) the run pinned for this task, alongside its expected text. A task may expect files and no text." } }, "required": [ "state", "prompt", "expected", "expectedFileCount" ], "additionalProperties": false }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "unavailable" ] }, "reason": { "type": "string", "enum": [ "not_captured", "version_unavailable", "snapshot_missing" ] } }, "required": [ "state", "reason" ], "additionalProperties": false } ], "description": "Full pinned prompt and expected text, or an explicit state saying the snapshot is missing" }, "output": { "anyOf": [ { "type": "object", "properties": { "state": { "type": "string", "enum": [ "answered" ] }, "text": { "type": "string" } }, "required": [ "state", "text" ], "additionalProperties": false }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "pending" ] }, "phase": { "type": "string", "enum": [ "not_dispatched", "queued", "running" ] } }, "required": [ "state", "phase" ], "additionalProperties": false }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "unavailable" ] }, "reason": { "type": "string", "enum": [ "runtime_missing", "turn_missing", "turn_failed", "turn_cancelled", "reader_failed" ] }, "errorCode": { "type": [ "string", "null" ] }, "message": { "type": [ "string", "null" ] } }, "required": [ "state", "reason", "errorCode", "message" ], "additionalProperties": false }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "didnt_run" ] } }, "required": [ "state" ], "additionalProperties": false, "description": "No answer came before the run ended, and none will; the row's `notRunning` says why" } ], "description": "The answer from the agent's registered command itself, never a later turn or text taken from elsewhere in the task" }, "result": { "type": [ "object", "null" ], "properties": { "revision": { "type": "string", "pattern": "^[a-f0-9]{64}$" }, "results": { "type": "array", "items": { "type": "object", "properties": { "key": { "type": "string", "minLength": 1 }, "outcome": { "anyOf": [ { "type": "object", "properties": { "status": { "type": "string", "enum": [ "completed" ] }, "verdict": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "boolean" ] }, "passed": { "type": "boolean" }, "reasoning": { "type": "string" } }, "required": [ "kind", "passed", "reasoning" ], "additionalProperties": false }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "float" ] }, "score": { "type": "number", "minimum": 0, "maximum": 1 }, "reasoning": { "type": "string" } }, "required": [ "kind", "score", "reasoning" ], "additionalProperties": false } ], "description": "Validated check output; a failing judgment is a valid result, not an execution error" } }, "required": [ "status", "verdict" ], "additionalProperties": false }, { "type": "object", "properties": { "status": { "type": "string", "enum": [ "error" ] }, "code": { "type": "string", "enum": [ "invalid_code", "invalid_verdict", "execution_failed", "timed_out", "cancelled", "input_unavailable", "output_limit", "runtime_unavailable", "batch_deadline", "inference_unavailable", "not_started" ] }, "message": { "type": "string", "minLength": 1, "maxLength": 2048, "description": "Why, in words. For a crash, the error itself (a check's exception and where it was raised, or a judge's reason), with untrusted text redacted." } }, "required": [ "status", "code", "message" ], "additionalProperties": false }, { "type": "object", "properties": { "status": { "type": "string", "enum": [ "error" ] }, "code": { "type": "string", "enum": [ "subject_failed", "subject_timed_out" ] }, "message": { "type": "string", "minLength": 1, "maxLength": 2048, "description": "Why, in words. For a crash, the error itself (a check's exception and where it was raised, or a judge's reason), with untrusted text redacted." } }, "required": [ "status", "code", "message" ], "additionalProperties": false, "description": "The agent's task failed or ran out of time, so this criterion could not be judged; the message gives the task's own reason" }, { "type": "object", "properties": { "status": { "type": "string", "enum": [ "error" ] }, "code": { "type": "string", "enum": [ "malformed_output" ] }, "message": { "type": "string", "minLength": 1, "maxLength": 2048, "description": "Why, in words. For a crash, the error itself (a check's exception and where it was raised, or a judge's reason), with untrusted text redacted." }, "judgeAnswer": { "type": "string", "minLength": 1, "maxLength": 4000, "description": "The opening characters of what the judge returned on its last attempt, with untrusted text redacted" } }, "required": [ "status", "code", "message", "judgeAnswer" ], "additionalProperties": false, "description": "An LLM judge answered, but its answer could not be read as a verdict" } ], "description": "Execution outcome. An error is not a verdict: it is shown as a failure with its reason and left out of the score." } }, "required": [ "key", "outcome" ], "additionalProperties": false } }, "score": { "type": [ "number", "null" ], "minimum": 0, "maximum": 1, "description": "Weighted mean of the checks that reached a verdict. Null when this task has no configured checks or none of them reached a verdict." }, "errorCount": { "type": "integer", "minimum": 0, "description": "Number of checks that reached no verdict (could not run): shown as failed and left out of `score`" }, "judgeCostUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "The checks' cost known when they returned: the LLM judges' charge (null if any is unavailable), and zero for deterministic checks and for agent judges, whose cost is tracked through `judgeTaskRunIds`" }, "judgeTaskRunIds": { "type": "array", "items": { "type": "string", "minLength": 1 }, "default": [], "description": "The judge task runs that agent judges started for this task; they are billed with the agent's run. Absent on results settled before agent-judge runs were recorded (those have a null `judgeCostUsd`, so their cost stays unknown)." } }, "required": [ "revision", "results", "score", "errorCount", "judgeCostUsd" ], "additionalProperties": false, "description": "All settled check outcomes with full reasoning; null until the task settles" }, "costUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "Cost of the task run plus its checks, set only when both are final and known" }, "criteriaCostUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "The checks' cost: LLM judges' charges plus the billed cost of agent judges' task runs; null while any part is unknown or the task has not settled" }, "latencyMs": { "type": [ "number", "null" ], "minimum": 0, "description": "Duration of this registered command's turn" }, "taskPath": { "type": [ "string", "null" ] }, "notRunning": { "type": [ "object", "null" ], "properties": { "kind": { "type": "string", "enum": [ "didnt_run" ] }, "reason": { "type": "object", "properties": { "text": { "type": "string", "maxLength": 512, "minLength": 1 }, "truncated": { "type": "boolean", "description": "The reason is cut to 512 characters; the run's whole error is `error` on `GET /runs/{runId}`." } }, "required": [ "text", "truncated" ], "additionalProperties": false, "description": "Why, as the run recorded it: the run's own error (cut to the preview length; the full run record has all of it), or a fixed sentence for a run that ended without one. Display it as is; do not parse it." } }, "required": [ "kind", "reason" ], "additionalProperties": false, "description": "Set when the run ended before this task settled; an export never shows a live wait" }, "stoppedBeforeFinish": { "type": "boolean", "description": "A person stopped the run before this task finished: it didn't run, or it settled with what the stop canceled. Not a failure. False for a task that settled on its own (even with an error) and on a run that completed or failed." } }, "required": [ "taskId", "cellId", "position", "task", "output", "result", "costUsd", "criteriaCostUsd", "latencyMs", "taskPath", "notRunning", "stoppedBeforeFinish" ], "additionalProperties": false, "description": "One task's full record on one cell, as the full export writes it" }, "example": { "taskId": "de707028-6ea2-4616-8864-c2ebd92b87f5", "cellId": "5ec92200-48c8-4c37-8bf0-a28aa3d08fcd", "position": 1, "task": { "state": "captured", "prompt": "text", "expected": "text", "expectedFileCount": 1 }, "output": { "state": "answered", "text": "text" }, "result": { "revision": "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c", "results": [ { "key": "text", "outcome": { "status": "completed", "verdict": { "kind": "boolean", "passed": true, "reasoning": "text" } } } ], "score": 0.75, "errorCount": 1, "judgeCostUsd": 0.75, "judgeTaskRunIds": [ "50ebcb0e-6b10-4712-81e7-d3f71142a582" ] }, "costUsd": 0.75, "criteriaCostUsd": 0.75, "latencyMs": 0.75, "taskPath": "text", "notRunning": { "kind": "didnt_run", "reason": { "text": "text", "truncated": true } }, "stoppedBeforeFinish": true } } } }, "400": { "description": "A parameter is missing or invalid (`error` is `Invalid request`, with `details`); or `cellId`, `cursor` or `taskId` is not part of this run (`error` says which; no `details`, no `code`). Or the run is from before per-task records were kept (`This historical run recorded no per-task record`).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/runs/$RUN_ID/tasks/$TASK_ID\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode \"cellId=$CELL_ID\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nrun_id = os.environ[\"RUN_ID\"]\ntask_id = os.environ[\"TASK_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/runs/{run_id}/tasks/{task_id}\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"cellId\": os.environ[\"CELL_ID\"]},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/runs/${encodeURIComponent(env(\"RUN_ID\"))}/tasks/${encodeURIComponent(env(\"TASK_ID\"))}`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"cellId\", env(\"CELL_ID\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Every criterion's verdict and reasoning for one task\ncontext evals benchmark run task \"$RUN_ID\" \"$TASK_ID\" --cell \"$CELL_ID\" | jq '.result.results[] | {key, outcome}'" } ] } }, "/feedback/enabled": { "get": { "tags": [ "Feedback" ], "summary": "Check whether feedback is on", "operationId": "getFeedbackEnabled", "parameters": [], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "enabled": { "type": "boolean" } }, "required": [ "enabled" ], "additionalProperties": false }, "example": { "enabled": true } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`), or its owner is not a member of the organization. A key limited to specific agents is not refused here.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "orgRead", "description": "Deployment-wide: whether Evals is turned on for this deployment; the same answer for every organization.", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/feedback/enabled\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/feedback/enabled\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/feedback/enabled`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/feedback/schemas": { "get": { "tags": [ "Feedback" ], "summary": "List feedback schemas", "operationId": "listFeedbackSchemas", "parameters": [], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "schemas": { "type": "array", "items": {} } }, "required": [ "schemas" ], "additionalProperties": false }, "example": { "schemas": [ {} ] } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`), or its owner is not a member of the organization. A key limited to specific agents is not refused here.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "orgRead", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/feedback/schemas\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/feedback/schemas\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/feedback/schemas`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/feedback": { "get": { "tags": [ "Feedback" ], "summary": "List feedback", "operationId": "listFeedback", "parameters": [ { "in": "query", "name": "sessionId", "required": false, "schema": { "type": "string", "description": "Filter by session identifier" } }, { "in": "query", "name": "page", "required": false, "schema": { "type": "integer", "minimum": 1, "default": 1 } }, { "in": "query", "name": "limit", "required": false, "schema": { "type": "integer", "minimum": 1, "maximum": 100, "default": 20 } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "entries": { "type": "array", "items": {} }, "pagination": { "type": "object", "properties": { "page": { "type": "integer" }, "limit": { "type": "integer" }, "total": { "type": "integer" }, "totalPages": { "type": "integer" } }, "required": [ "page", "limit", "total", "totalPages" ], "additionalProperties": false } }, "required": [ "entries", "pagination" ], "additionalProperties": false }, "example": { "entries": [ {} ], "pagination": { "page": 1, "limit": 1, "total": 1, "totalPages": 1 } } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/feedback\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/feedback\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/feedback`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Feedback on one session, 50 a page (GET /feedback takes only sessionId, page and limit)\ncontext evals feedback list --session \"$SESSION_ID\" --page-size 50" } ] } }, "/judges": { "get": { "tags": [ "Judges" ], "summary": "List judges", "operationId": "listJudges", "parameters": [], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique judge identifier" }, "orgId": { "type": [ "string", "null" ], "minLength": 1, "description": "Organization that owns this judge (null for `SYSTEM` judges)" }, "createdById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who created this judge" }, "name": { "type": "string", "minLength": 1, "maxLength": 100, "description": "Judge display name" }, "description": { "type": [ "string", "null" ], "maxLength": 1000, "description": "Optional judge description" }, "instructions": { "type": "string", "minLength": 1, "maxLength": 10000, "description": "Evaluation criteria for the judge: what to assess. The system builds the full prompt, with the trace input and output, when the judge runs." }, "outputType": { "type": "string", "enum": [ "FLOAT", "BOOLEAN", "TEXT" ], "description": "How the judge outputs its evaluation" }, "model": { "type": "string", "minLength": 1, "description": "Model the judge uses for evaluation" }, "type": { "type": "string", "enum": [ "SYSTEM", "USER" ], "description": "Whether this is a system-provided or user-created judge" }, "scorecardCount": { "type": "integer", "minimum": 0, "description": "Number of scorecards that use this judge" }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" } }, "required": [ "id", "orgId", "createdById", "name", "description", "instructions", "outputType", "model", "type", "createdAt", "updatedAt" ], "additionalProperties": false, "description": "An LLM-as-judge evaluator configuration" }, "description": "Judges visible to the organization" }, "example": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "createdById": "f7661063-fadd-49f3-8a00-1467f872dba6", "name": "Refund questions", "description": "text", "instructions": "text", "outputType": "FLOAT", "model": "anthropic/claude-sonnet-5", "type": "SYSTEM", "scorecardCount": 1, "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z" } ] } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/judges\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/judges\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/judges`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/judges/{judgeId}": { "get": { "tags": [ "Judges" ], "summary": "Get a judge", "operationId": "getJudge", "parameters": [ { "in": "path", "name": "judgeId", "required": true, "description": "Judge identifier", "schema": { "type": "string", "minLength": 1, "description": "Judge identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique judge identifier" }, "orgId": { "type": [ "string", "null" ], "minLength": 1, "description": "Organization that owns this judge (null for `SYSTEM` judges)" }, "createdById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who created this judge" }, "name": { "type": "string", "minLength": 1, "maxLength": 100, "description": "Judge display name" }, "description": { "type": [ "string", "null" ], "maxLength": 1000, "description": "Optional judge description" }, "instructions": { "type": "string", "minLength": 1, "maxLength": 10000, "description": "Evaluation criteria for the judge: what to assess. The system builds the full prompt, with the trace input and output, when the judge runs." }, "outputType": { "type": "string", "enum": [ "FLOAT", "BOOLEAN", "TEXT" ], "description": "How the judge outputs its evaluation" }, "model": { "type": "string", "minLength": 1, "description": "Model the judge uses for evaluation" }, "type": { "type": "string", "enum": [ "SYSTEM", "USER" ], "description": "Whether this is a system-provided or user-created judge" }, "scorecardCount": { "type": "integer", "minimum": 0, "description": "Number of scorecards that use this judge" }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" } }, "required": [ "id", "orgId", "createdById", "name", "description", "instructions", "outputType", "model", "type", "createdAt", "updatedAt" ], "additionalProperties": false, "description": "An LLM-as-judge evaluator configuration" }, "example": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "createdById": "f7661063-fadd-49f3-8a00-1467f872dba6", "name": "Refund questions", "description": "text", "instructions": "text", "outputType": "FLOAT", "model": "anthropic/claude-sonnet-5", "type": "SYSTEM", "scorecardCount": 1, "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z" } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/judges/$JUDGE_ID\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\njudge_id = os.environ[\"JUDGE_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/judges/{judge_id}\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/judges/${encodeURIComponent(env(\"JUDGE_ID\"))}`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/scorecards": { "get": { "tags": [ "Scorecards" ], "summary": "List scorecards", "operationId": "listScorecards", "parameters": [], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique scorecard identifier" }, "orgId": { "type": "string", "minLength": 1, "description": "Organization that owns this scorecard" }, "createdById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who created this scorecard" }, "name": { "type": "string", "minLength": 1, "maxLength": 100, "description": "Scorecard display name" }, "description": { "type": [ "string", "null" ], "maxLength": 1000, "description": "Optional scorecard description" }, "isActive": { "type": "boolean", "description": "Whether the scorecard is active" }, "isAdditionalCommentsRequired": { "type": "boolean", "description": "Whether the additional comments field is required when submitting feedback" }, "displayOrder": { "type": "integer", "minimum": 0, "description": "Position in the scorecard list" }, "kind": { "type": "string", "enum": [ "auto", "skill-rubric" ], "description": "Always `auto` on this API (automatic feedback scorecards). Skill rubrics, managed with `context evals scorecard`, are never served here; a skill rubric's ID answers 404." }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" }, "judges": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique link identifier" }, "scorecardId": { "type": "string", "minLength": 1, "description": "Scorecard this link belongs to" }, "judgeId": { "type": "string", "minLength": 1, "description": "Linked judge" }, "order": { "type": "integer", "minimum": 0, "description": "Execution and display order within the scorecard" }, "isRequired": { "type": "boolean", "description": "Whether this judge must pass for the scorecard to pass" }, "createdAt": { "type": "string", "format": "date-time", "description": "When the judge was added to the scorecard" }, "name": { "type": "string", "minLength": 1, "description": "Linked judge's display name" }, "instructions": { "type": "string", "description": "Linked judge's evaluation instructions" }, "outputType": { "type": "string", "enum": [ "FLOAT", "BOOLEAN", "TEXT" ], "description": "Linked judge's output type" } }, "required": [ "id", "scorecardId", "judgeId", "order", "isRequired", "createdAt", "name", "instructions", "outputType" ], "additionalProperties": false, "description": "Links a judge to a scorecard, with its order, required flag, and the judge's content" }, "description": "Attached judges, in order" }, "scopes": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique scope identifier" }, "scorecardId": { "type": "string", "minLength": 1, "description": "Scorecard this link belongs to" }, "orgId": { "type": [ "string", "null" ], "minLength": 1, "description": "Organization filter (null matches any organization)" }, "workspaceId": { "type": [ "string", "null" ], "minLength": 1, "description": "Workspace filter (null matches any workspace)" }, "userId": { "type": [ "string", "null" ], "minLength": 1, "description": "User filter (null matches any user)" }, "createdAt": { "type": "string", "format": "date-time", "description": "When this scope was added" } }, "required": [ "id", "scorecardId", "orgId", "workspaceId", "userId", "createdAt" ], "additionalProperties": false, "description": "Scope filter for a scorecard: null fields match any value" }, "description": "Scope filters" }, "judgeCount": { "type": "integer", "minimum": 0, "description": "Number of attached judges" }, "evalRunCount": { "type": "integer", "minimum": 0, "description": "Number of evaluation runs that use this scorecard" } }, "required": [ "id", "orgId", "createdById", "name", "description", "isActive", "isAdditionalCommentsRequired", "displayOrder", "kind", "createdAt", "updatedAt", "judges", "scopes", "judgeCount" ], "additionalProperties": false, "description": "A scorecard with its judges and scopes" }, "description": "The organization's automatic feedback scorecards, with their judges and scopes. Skill rubrics are not included." }, "example": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "createdById": "f7661063-fadd-49f3-8a00-1467f872dba6", "name": "Refund questions", "description": "text", "isActive": true, "isAdditionalCommentsRequired": true, "displayOrder": 1, "kind": "auto", "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z", "judges": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "scorecardId": "50ada941-25ed-4b68-880b-6016b9e07ccd", "judgeId": "bc182770-0382-40fb-8da5-ebf661383595", "order": 1, "isRequired": true, "createdAt": "2026-10-14T09:30:00.000Z", "name": "Refund questions", "instructions": "text", "outputType": "FLOAT" } ], "scopes": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "scorecardId": "50ada941-25ed-4b68-880b-6016b9e07ccd", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "workspaceId": "09707fa5-aaae-4493-8a23-827111000f89", "userId": "6ccb2121-4ffd-40b0-8c2c-1607cf6a05be", "createdAt": "2026-10-14T09:30:00.000Z" } ], "judgeCount": 1, "evalRunCount": 1 } ] } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/scorecards\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/scorecards\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/scorecards`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/scorecards/{scorecardId}": { "get": { "tags": [ "Scorecards" ], "summary": "Get a scorecard", "operationId": "getScorecard", "parameters": [ { "in": "path", "name": "scorecardId", "required": true, "description": "Scorecard identifier", "schema": { "type": "string", "minLength": 1, "description": "Scorecard identifier" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique scorecard identifier" }, "orgId": { "type": "string", "minLength": 1, "description": "Organization that owns this scorecard" }, "createdById": { "type": [ "string", "null" ], "minLength": 1, "description": "User who created this scorecard" }, "name": { "type": "string", "minLength": 1, "maxLength": 100, "description": "Scorecard display name" }, "description": { "type": [ "string", "null" ], "maxLength": 1000, "description": "Optional scorecard description" }, "isActive": { "type": "boolean", "description": "Whether the scorecard is active" }, "isAdditionalCommentsRequired": { "type": "boolean", "description": "Whether the additional comments field is required when submitting feedback" }, "displayOrder": { "type": "integer", "minimum": 0, "description": "Position in the scorecard list" }, "kind": { "type": "string", "enum": [ "auto", "skill-rubric" ], "description": "Always `auto` on this API (automatic feedback scorecards). Skill rubrics, managed with `context evals scorecard`, are never served here; a skill rubric's ID answers 404." }, "createdAt": { "type": "string", "format": "date-time", "description": "Creation timestamp" }, "updatedAt": { "type": "string", "format": "date-time", "description": "Last update timestamp" }, "judges": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique link identifier" }, "scorecardId": { "type": "string", "minLength": 1, "description": "Scorecard this link belongs to" }, "judgeId": { "type": "string", "minLength": 1, "description": "Linked judge" }, "order": { "type": "integer", "minimum": 0, "description": "Execution and display order within the scorecard" }, "isRequired": { "type": "boolean", "description": "Whether this judge must pass for the scorecard to pass" }, "createdAt": { "type": "string", "format": "date-time", "description": "When the judge was added to the scorecard" }, "name": { "type": "string", "minLength": 1, "description": "Linked judge's display name" }, "instructions": { "type": "string", "description": "Linked judge's evaluation instructions" }, "outputType": { "type": "string", "enum": [ "FLOAT", "BOOLEAN", "TEXT" ], "description": "Linked judge's output type" } }, "required": [ "id", "scorecardId", "judgeId", "order", "isRequired", "createdAt", "name", "instructions", "outputType" ], "additionalProperties": false, "description": "Links a judge to a scorecard, with its order, required flag, and the judge's content" }, "description": "Attached judges, in order" }, "scopes": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1, "description": "Unique scope identifier" }, "scorecardId": { "type": "string", "minLength": 1, "description": "Scorecard this link belongs to" }, "orgId": { "type": [ "string", "null" ], "minLength": 1, "description": "Organization filter (null matches any organization)" }, "workspaceId": { "type": [ "string", "null" ], "minLength": 1, "description": "Workspace filter (null matches any workspace)" }, "userId": { "type": [ "string", "null" ], "minLength": 1, "description": "User filter (null matches any user)" }, "createdAt": { "type": "string", "format": "date-time", "description": "When this scope was added" } }, "required": [ "id", "scorecardId", "orgId", "workspaceId", "userId", "createdAt" ], "additionalProperties": false, "description": "Scope filter for a scorecard: null fields match any value" }, "description": "Scope filters" }, "judgeCount": { "type": "integer", "minimum": 0, "description": "Number of attached judges" }, "evalRunCount": { "type": "integer", "minimum": 0, "description": "Number of evaluation runs that use this scorecard" } }, "required": [ "id", "orgId", "createdById", "name", "description", "isActive", "isAdditionalCommentsRequired", "displayOrder", "kind", "createdAt", "updatedAt", "judges", "scopes", "judgeCount" ], "additionalProperties": false, "description": "A scorecard with its judges and scopes" }, "example": { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "createdById": "f7661063-fadd-49f3-8a00-1467f872dba6", "name": "Refund questions", "description": "text", "isActive": true, "isAdditionalCommentsRequired": true, "displayOrder": 1, "kind": "auto", "createdAt": "2026-10-14T09:30:00.000Z", "updatedAt": "2026-10-14T09:30:00.000Z", "judges": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "scorecardId": "50ada941-25ed-4b68-880b-6016b9e07ccd", "judgeId": "bc182770-0382-40fb-8da5-ebf661383595", "order": 1, "isRequired": true, "createdAt": "2026-10-14T09:30:00.000Z", "name": "Refund questions", "instructions": "text", "outputType": "FLOAT" } ], "scopes": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "scorecardId": "50ada941-25ed-4b68-880b-6016b9e07ccd", "orgId": "64f91306-4d3f-4344-8778-62a4a4b328be", "workspaceId": "09707fa5-aaae-4493-8a23-827111000f89", "userId": "6ccb2121-4ffd-40b0-8c2c-1607cf6a05be", "createdAt": "2026-10-14T09:30:00.000Z" } ], "judgeCount": 1, "evalRunCount": 1 } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/scorecards/$SCORECARD_ID\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nscorecard_id = os.environ[\"SCORECARD_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/scorecards/{scorecard_id}\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/scorecards/${encodeURIComponent(env(\"SCORECARD_ID\"))}`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/scorecards/scope-options": { "get": { "tags": [ "Scorecards" ], "summary": "List scorecard scope options", "operationId": "getScorecardScopeOptions", "parameters": [], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "workspaces": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "description": "Workspace identifier" }, "name": { "type": "string", "description": "Display name" } }, "required": [ "id", "name" ], "additionalProperties": false, "description": "An option you can pick when setting a scorecard's scope" }, "description": "Up to 50 workspaces in the organization, by name" }, "users": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "description": "User identifier" }, "name": { "type": "string", "description": "Display name" } }, "required": [ "id", "name" ], "additionalProperties": false, "description": "An option you can pick when setting a scorecard's scope" }, "description": "Up to 50 users in the organization, by name" } }, "required": [ "workspaces", "users" ], "additionalProperties": false, "description": "Workspaces and users you can choose from when setting a scorecard's scope" }, "example": { "workspaces": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" } ], "users": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" } ] } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/scorecards/scope-options\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/scorecards/scope-options\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/scorecards/scope-options`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/traces/sessions": { "get": { "tags": [ "Traces" ], "summary": "List trace sessions", "description": "Returns trace sessions one window at a time, latest activity first (ordered by each session's latest call in the range). Follow `nextCursor` until it is null. The first page fixes three things that the cursor carries through the rest of the walk: the calls stored by the time of that read (`window.writtenBy`), the range end it used (`window.to`), and the data source it read (`readSource`). As a result, every session with such a call is returned once, in last-activity order, with no duplicates; calls stored later appear in the next walk. `startedAt` is the session's first call, even if it falls before the range; calls, tokens, cost, and duration cover only the range. `filters` is a JSON-encoded array of filter conditions. `totalCount` stops at 10,000; `totalCountIsCapped: true` marks a capped count.", "operationId": "listTraceSessions", "parameters": [ { "in": "query", "name": "from", "required": true, "schema": { "type": "string", "format": "date-time", "description": "Start of the date range (ISO 8601)" } }, { "in": "query", "name": "to", "required": true, "schema": { "type": "string", "format": "date-time", "description": "End of the date range (ISO 8601)" } }, { "in": "query", "name": "page", "required": false, "schema": { "type": "integer", "minimum": 1, "description": "Page number, starting at 1, for table-style paging. Ignored when `cursor` is set.", "default": 1 } }, { "in": "query", "name": "cursor", "required": false, "schema": { "type": "string", "description": "Token from a previous response's `nextCursor`. Gives stable iteration and is the preferred way to page through results in code. A walk keeps the range end it started with (returned as `window.to`). A token from the older start-time order, or a malformed token, returns 400 `INVALID_CURSOR`: start again from the first page." } }, { "in": "query", "name": "pageSize", "required": false, "schema": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Sessions per page", "default": 15 } }, { "in": "query", "name": "search", "required": false, "schema": { "type": "string", "maxLength": 500, "description": "Token search over trace text. Terms match as whole tokens (runs of letters, digits, or non-ASCII characters; punctuation and underscores split terms). Matching ignores ASCII case, and every term must match. A `*` widens a term: `zip*` matches words starting with zip (zip, zipped, zipping), `*ip` matches words ending in `ip`, and `*ognit*` matches any text containing `ognit`. A match covers the whole conversation: every turn of a conversation whose text contains all the terms is returned, not only the turn that introduced them, and the terms may appear in different turns. Terms must occur inside the queried time range. Input with no searchable tokens (punctuation only) returns 400. Some time ranges are served from a data source that matches terms by substring instead; `readSource` in the response says which source served the range." } }, { "in": "query", "name": "filters", "required": false, "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "field": { "type": "string", "enum": [ "success", "cost_usd", "duration_ms", "model", "model_provider", "input_tokens", "output_tokens", "error_message", "agent_id", "user_id", "workspace_id", "chat_id", "skill_id", "origin" ], "description": "Field to filter on" }, "operator": { "type": "string", "enum": [ "equals", "not_equals", "greater_than", "less_than", "gte", "lte", "contains", "in", "not_in" ], "description": "Comparison operator for a filter condition" }, "value": { "type": "string", "minLength": 1, "description": "Filter value as a string, interpreted according to the field's type" } }, "required": [ "field", "operator", "value" ], "additionalProperties": false, "description": "One filter condition: a field, an operator, and a value" }, "maxItems": 20, "description": "Filter conditions" } } }, "description": "Send it as JSON in one value, for example `filters=[{\"field\":\"success\",\"operator\":\"equals\",\"value\":\"false\"}]`. Any other encoding is ignored." }, { "in": "query", "name": "matchAll", "required": false, "schema": { "type": "boolean", "description": "Combine filter conditions with AND (true) or with OR (false)", "default": true } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "data": { "type": "array", "items": { "type": "object", "properties": { "sessionId": { "type": "string", "description": "Session ID" }, "taskId": { "type": [ "string", "null" ], "description": "ID of the linked task; null when no task is linked" }, "taskTitle": { "type": [ "string", "null" ], "description": "Title of the linked task; null when no task is linked" }, "startedAt": { "type": "string", "description": "The session's first call (ISO 8601), even if it falls outside the requested range" }, "totalDurationMs": { "type": "number", "minimum": 0, "description": "Sum of call durations within the requested range, in milliseconds" }, "turnCount": { "type": "integer", "minimum": 0, "description": "Number of assistant turns" }, "totalTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Billed tokens: the sum of input, output, reasoning, cache write, and cache read tokens across all calls. Null when no call in the session reported token usage (different from a real 0)." }, "totalCost": { "type": [ "number", "null" ], "minimum": 0, "description": "Session cost in USD, computed the same way for the session list and session detail. When the session's calls in scope (inside the requested range, if one is given) belong to task runs, this is the billed amount for those runs, summed. A run's bill is its full total, including its sub-agents' and auxiliary calls, so on a range-bounded read it can include calls of that run outside the range; a run with no calls in the range is not counted. Null for a sub-agent session, for a session whose calls in scope mix calls with and without a run, for a run that another session of the same chat also belongs to, and when any run's billed amount is unknown (not final, missing, or not rated) or billing data could not be read. When no call in scope belongs to a run, this is the sum of those calls' costs, or null when no call reported a cost (unknown, different from a real 0). `totalCostSource` says which method was used." }, "models": { "type": "array", "items": { "type": "string" }, "description": "Distinct models used in this session" }, "allSuccess": { "type": "boolean", "description": "`false` if any turn failed" }, "errorCount": { "type": "integer", "minimum": 0, "description": "Number of failed turns in this session" }, "errorMessageSample": { "type": [ "string", "null" ], "description": "One example error message from a failed turn; null if all turns succeeded or none had a message" }, "agentId": { "type": "string", "description": "Agent ID (the same for the whole session)" }, "userId": { "type": [ "string", "null" ], "description": "User ID of the agent owner" }, "agentName": { "type": [ "string", "null" ], "description": "Agent display name" }, "userName": { "type": [ "string", "null" ], "description": "User display name" }, "workspaceId": { "type": "string", "description": "Workspace ID for the chat; empty when the task had no workspace when its traces were recorded" }, "workspaceName": { "type": [ "string", "null" ], "description": "Workspace display name; null when `workspaceId` is empty or the workspace is not found" }, "skills": { "type": "array", "items": { "type": "string" }, "description": "Distinct skill IDs invoked across all turns in this session" }, "sessionRole": { "type": [ "string", "null" ], "enum": [ "root", "subtask", "continuation", null ], "description": "`root`, `subtask` (a sub-agent session), or `continuation`. Null when the session has no role on record (sessions recorded before roles were tracked)." }, "parentSessionId": { "type": [ "string", "null" ], "description": "Session ID of the session that started this one; set for `subtask` sessions, null otherwise" }, "subagents": { "type": "object", "properties": { "count": { "type": "integer", "minimum": 0, "description": "Number of sub-agent (`subtask`) sessions started under this session, including nested ones" }, "costUsd": { "type": "number", "minimum": 0, "description": "Estimated provider cost of those sub-agent sessions' model calls in the query range (sum of per-call costs), not billed credits. Billing data is not tracked per session." } }, "required": [ "count", "costUsd" ], "additionalProperties": false, "description": "Sub-agent summary for this session; count 0 and cost 0 when it started none" }, "origin": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "evaluation-run" ] }, "run": { "type": [ "object", "null" ], "properties": { "runId": { "type": "string", "minLength": 1, "description": "The evaluation run that started this task (or the task its sub-agent belongs to)" }, "evaluationId": { "type": "string", "minLength": 1, "description": "The run's evaluation, so you can link to the run page" }, "evaluationName": { "type": "string", "description": "The run's evaluation name, so you can label the task with it instead of its generated title (run and task IDs)" } }, "required": [ "runId", "evaluationId", "evaluationName" ], "additionalProperties": false, "description": "The run to link to; null when no run records it anymore (its evaluation was deleted)" } }, "required": [ "kind", "run" ], "additionalProperties": false, "description": "A task started by an evaluation run, or a sub-agent of one" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "judge" ], "description": "A judge's session" }, "monitor": { "type": [ "object", "null" ], "properties": { "monitorId": { "type": "string", "minLength": 1, "description": "The monitor whose agent judge ran in this session, so you can link to it" }, "monitorName": { "type": "string", "description": "The monitor's name, so you can group the session under it instead of among the person's chats" } }, "required": [ "monitorId", "monitorName" ], "additionalProperties": false, "description": "The monitor whose verdict names this session's task as its judge. Null for the organization's judge agent grading an evaluation, a sub-agent of a judge, or a monitor judge that has not posted a verdict." } }, "required": [ "kind", "monitor" ], "additionalProperties": false, "description": "A task of the organization's judge agent grading another task's output, or a monitor's agent judge grading a person's turn" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill-learning-replay" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "A skill-learning replay scoring a prompt candidate" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "production" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The organization's own work: classified into workflows and counted as production" } ], "description": "What started this session's chat: an evaluation run (with the run to link to), a judge, a skill-learning replay, or production. Only production sessions are included in workflows." }, "lastActivityAt": { "type": "string", "description": "Latest call of this session inside the requested range (ISO 8601). The list is ordered by this, newest first." }, "feedbackSummary": { "anyOf": [ { "type": "object", "properties": { "state": { "type": "string", "enum": [ "summary" ] }, "summary": { "type": "object", "properties": { "count": { "type": "integer", "minimum": 0, "description": "Number of feedback submissions on this record" }, "avgRating": { "type": [ "number", "null" ], "minimum": 1, "maximum": 5, "description": "Mean numeric rating on the 1 to 5 scale, or null when there is no numeric rating" }, "maxRating": { "type": [ "number", "null" ], "minimum": 1, "maximum": 5, "description": "Highest numeric rating on the 1 to 5 scale, or null when there is no numeric rating" }, "withTextCount": { "type": "integer", "minimum": 0, "description": "Number of submissions with a non-empty comment" } }, "required": [ "count", "avgRating", "maxRating", "withTextCount" ], "additionalProperties": false, "description": "Feedback summary for one task, run, or trace session" } }, "required": [ "state", "summary" ], "additionalProperties": false, "description": "Feedback summary loaded; a count of zero means no feedback" }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "unavailable" ] } }, "required": [ "state" ], "additionalProperties": false, "description": "Feedback summary could not be loaded; trace data is still available" } ], "description": "Feedback state for list rows, telling no feedback apart from feedback that could not be loaded" }, "chatUrl": { "type": [ "string", "null" ], "description": "Absolute link to the chat in the app (`/client/tasks/`); null when no task is linked" }, "originKind": { "type": "string", "enum": [ "evaluation-run", "judge", "skill-learning-replay", "production" ], "description": "`origin.kind` as a flat field, for tabular use" }, "preview": { "anyOf": [ { "type": "object", "properties": { "state": { "type": "string", "enum": [ "loaded" ] }, "firstUserMessage": { "type": [ "string", "null" ], "description": "The first message the person typed in this chat; null when the range contains none" }, "matchSnippet": { "type": [ "string", "null" ], "description": "The first text in the chat that matches a search term; null when no search is given" } }, "required": [ "state", "firstUserMessage", "matchSnippet" ], "additionalProperties": false, "description": "Preview read from the chat's message parts" }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "not-loaded" ] }, "reason": { "type": "string", "enum": [ "not-chats-scope", "legacy-source" ], "description": "`not-chats-scope`: previews load only in the chats scope. `legacy-source`: this range is served from a data source that has no per-part text." } }, "required": [ "state", "reason" ], "additionalProperties": false, "description": "No preview for this row, with the reason" } ], "description": "First user message and matching text for a listed chat" }, "totalCostSource": { "type": "string", "enum": [ "billing_ledger", "warehouse" ], "description": "Where the session's `totalCost` comes from. `billing_ledger`: the session's calls in scope (inside the requested range, if one is given) belong to task runs, and `totalCost` is the billed amount for those runs, including sub-agents' and auxiliary calls (null while the billed amount cannot be determined). `warehouse`: no call in scope belongs to a run, and `totalCost` is the sum of those calls' costs, which excludes sub-agents." }, "workflows": { "anyOf": [ { "type": "object", "properties": { "state": { "type": "string", "enum": [ "loaded" ] }, "workflows": { "type": "array", "items": { "type": "object", "properties": { "topicId": { "type": "string", "description": "Workflow ID; the workflow page is `/client/evals/workflows/t/`" }, "name": { "type": "string", "description": "Workflow display name" } }, "required": [ "topicId", "name" ], "additionalProperties": false, "description": "An active workflow the session's chat belongs to" }, "description": "The chat's current workflows, most confident first; empty when the chat belongs to none (unassigned)" } }, "required": [ "state", "workflows" ], "additionalProperties": false, "description": "Workflow membership read from the chat's current workflows" }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "unavailable" ] } }, "required": [ "state" ], "additionalProperties": false, "description": "Workflow membership could not be read; trace data is still available" }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "withheld" ] } }, "required": [ "state" ], "additionalProperties": false, "description": "Workflow names are hidden from this caller (workflows are turned off for the organization, or the caller lacks organization-wide evaluations access); membership was not read" } ], "description": "Workflow membership, telling unassigned apart from unreadable and from hidden" } }, "required": [ "sessionId", "taskId", "taskTitle", "startedAt", "totalDurationMs", "turnCount", "totalTokens", "totalCost", "models", "allSuccess", "errorCount", "errorMessageSample", "agentId", "userId", "agentName", "userName", "workspaceId", "workspaceName", "skills", "sessionRole", "parentSessionId", "subagents", "origin", "lastActivityAt", "feedbackSummary", "chatUrl", "originKind", "preview", "totalCostSource", "workflows" ], "additionalProperties": false, "description": "Trace session list row with a feedback summary, link, and preview. Counts, tokens, and duration cover calls inside the requested range. When `totalCostSource` is `billing_ledger`, `totalCost` is the bill for the runs those calls belong to, computed the same way as in session detail." } }, "totalCount": { "type": "integer", "minimum": 0, "description": "Number of matching sessions for pagination, counted up to a cap of 10,000" }, "totalCountIsCapped": { "type": "boolean", "description": "`true` when more sessions matched than the cap; `totalCount` is then the cap, not the exact total" }, "nextCursor": { "type": [ "string", "null" ], "description": "Token for the next window; null when every matching session has been returned" }, "window": { "type": "object", "properties": { "from": { "type": "string", "description": "Start of the range searched (ISO 8601), after applying the server's maximum range" }, "to": { "type": "string", "description": "End of the range searched (ISO 8601). On a cursor page, this is the end the cursor carries, not the request's `to`." }, "writtenBy": { "type": "string", "description": "Cutoff time (ISO 8601, server clock): only calls stored by this time are listed and counted. A first page sets it when it reads; a cursor page uses the one its token carries, so a whole walk reads one fixed set of calls." }, "clamped": { "type": "boolean", "description": "`true` when the requested start was earlier than the server's maximum range allows and was moved forward" }, "maxDays": { "type": "integer", "exclusiveMinimum": 0, "description": "Maximum range width the server allows, in days" } }, "required": [ "from", "to", "writtenBy", "clamped", "maxDays" ], "additionalProperties": false, "description": "The time range the server searched, so you can display it" }, "readSource": { "type": "string", "enum": [ "v2", "legacy" ], "description": "Which data source served this range; it decides how search terms match" }, "searchMode": { "type": [ "string", "null" ], "description": "How the search was matched, in words (for example, 'zip* (starts with), earlier (whole word)'); null when no search is given" }, "viewerTimeZone": { "type": [ "string", "null" ], "description": "IANA time zone from the requesting person's profile, for showing dates in their zone; null when no person is behind the request or no zone is set" } }, "required": [ "data", "totalCount", "totalCountIsCapped", "nextCursor", "window", "readSource", "searchMode", "viewerTimeZone" ], "additionalProperties": false, "description": "One window of the session list; follow `nextCursor` for the rest" }, "example": { "data": [ { "sessionId": "09724085-1664-4183-8554-bacab2816579", "taskId": "de707028-6ea2-4616-8864-c2ebd92b87f5", "taskTitle": "text", "startedAt": "text", "totalDurationMs": 0.75, "turnCount": 1, "totalTokens": 1, "totalCost": 0.75, "models": [ "text" ], "allSuccess": true, "errorCount": 1, "errorMessageSample": "text", "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "userId": "6ccb2121-4ffd-40b0-8c2c-1607cf6a05be", "agentName": "text", "userName": "text", "workspaceId": "09707fa5-aaae-4493-8a23-827111000f89", "workspaceName": "text", "skills": [ "text" ], "sessionRole": "root", "parentSessionId": "35de55cd-12cf-465c-8d0a-7317274054a7", "subagents": { "count": 1, "costUsd": 0.75 }, "origin": { "kind": "evaluation-run", "run": { "runId": "599e5591-3334-4d0a-8bb9-069b278d9b20", "evaluationId": "66248a43-3af8-4b5b-86b6-fcb59feb38b2", "evaluationName": "text" } }, "lastActivityAt": "text", "feedbackSummary": { "state": "summary", "summary": { "count": 1, "avgRating": 1.5, "maxRating": 1.5, "withTextCount": 1 } }, "chatUrl": "text", "originKind": "evaluation-run", "preview": { "state": "loaded", "firstUserMessage": "text", "matchSnippet": "text" }, "totalCostSource": "billing_ledger", "workflows": { "state": "loaded", "workflows": [ { "topicId": "c42387f0-94d3-427e-80d1-a63853a68624", "name": "Refund questions" } ] } } ], "totalCount": 1, "totalCountIsCapped": true, "nextCursor": "text", "window": { "from": "text", "to": "text", "writtenBy": "text", "clamped": true, "maxDays": 1 }, "readSource": "v2", "searchMode": "text", "viewerTimeZone": "text" } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`), or its owner is not a member of the organization. A key limited to specific agents is not refused here: it reads those agents' traces only.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Trace data can't be read: this deployment has no trace store configured (code `CLICKHOUSE_UNAVAILABLE`), or the server could not yet tell which store holds this range (`TRACE_READ_SOURCE_UNRESOLVED`, retry). A trace store that fails during the query answers 500 with code `QUERY_FAILED`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "traces", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/traces/sessions\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode 'from=2026-10-01T00:00:00Z' \\\n --data-urlencode 'to=2026-10-31T00:00:00Z' \\\n --data-urlencode 'pageSize=50' \\\n --data-urlencode 'filters=[{\"field\":\"success\",\"operator\":\"equals\",\"value\":\"false\"}]'" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/traces/sessions\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"from\": \"2026-10-01T00:00:00Z\", \"to\": \"2026-10-31T00:00:00Z\", \"pageSize\": 50, \"filters\": json.dumps([{\"field\": \"success\", \"operator\": \"equals\", \"value\": \"false\"}])},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/traces/sessions`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"from\", \"2026-10-01T00:00:00Z\")\nurl.searchParams.set(\"to\", \"2026-10-31T00:00:00Z\")\nurl.searchParams.set(\"pageSize\", \"50\")\nurl.searchParams.set(\"filters\", JSON.stringify([{\"field\": \"success\", \"operator\": \"equals\", \"value\": \"false\"}]))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Production sessions only: no judge, Evaluation-run or replay work (rows stored before origin was recorded are not listed)\ncontext evals traces list --from 7d --where \"origin=production\"\n\n# FTS over the last week\ncontext evals traces list --from 7d --search \"rate limit\"\n\n# Failed turns over 50 cents in the last month\ncontext evals traces list --from 30d --where \"success=false\" --where \"cost_usd>0.50\"\n\n# Triage failure patterns from row data alone, with no per-session fetches\ncontext evals traces list --from 7d --where \"success=false\" | jq -r '.data[] | select(.errorCount > 0) | \"\\(.sessionId)\\t\\(.errorCount)\\t\\(.errorMessageSample)\"'\n\n# Workspace-scoped sessions that used either PDF or DOCX skill\ncontext evals traces list --from 7d --where \"workspace_id=$WORKSPACE_ID\" --where \"skill_id in $PDF_SKILL_ID,$DOCX_SKILL_ID\"\n\n# Sessions that used BOTH skills (--match all is the default, so repeated --where flags must all match)\ncontext evals traces list --from 7d --where \"skill_id=$PDF_SKILL_ID\" --where \"skill_id=$DOCX_SKILL_ID\"\n\n# Drill in to every session in the last 24h (each get writes its own file)\ncontext evals traces list --from 24h --page-size 50 | jq -r '.data[].sessionId' | xargs -I {} context evals traces get {}" } ] } }, "/traces/sessions/{sessionId}": { "get": { "tags": [ "Traces" ], "summary": "Get a trace session", "operationId": "getTraceSessionDetail", "parameters": [ { "in": "path", "name": "sessionId", "required": true, "description": "Trace session ID", "schema": { "type": "string", "minLength": 1, "description": "Trace session ID" } }, { "in": "query", "name": "from", "required": false, "schema": { "type": "string", "format": "date-time", "description": "Optional range start (ISO 8601). When both bounds are given, the server skips data outside the range." } }, { "in": "query", "name": "to", "required": false, "schema": { "type": "string", "format": "date-time", "description": "Optional range end (ISO 8601)" } }, { "in": "query", "name": "cursor", "required": false, "schema": { "type": "string", "description": "Token from a previous response's `nextCursor`; omit it for the first window" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "session": { "type": "object", "properties": { "sessionId": { "type": "string", "description": "Session ID" }, "taskId": { "type": [ "string", "null" ], "description": "ID of the linked task; null when no task is linked" }, "taskTitle": { "type": [ "string", "null" ], "description": "Title of the linked task; null when no task is linked" }, "startedAt": { "type": "string", "description": "The session's first call (ISO 8601), even if it falls outside the requested range" }, "totalDurationMs": { "type": "number", "minimum": 0, "description": "Sum of call durations within the requested range, in milliseconds" }, "turnCount": { "type": "integer", "minimum": 0, "description": "Number of assistant turns" }, "totalTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Billed tokens: the sum of input, output, reasoning, cache write, and cache read tokens across all calls. Null when no call in the session reported token usage (different from a real 0)." }, "totalCost": { "type": [ "number", "null" ], "minimum": 0, "description": "Session cost in USD, computed the same way for the session list and session detail. When the session's calls in scope (inside the requested range, if one is given) belong to task runs, this is the billed amount for those runs, summed. A run's bill is its full total, including its sub-agents' and auxiliary calls, so on a range-bounded read it can include calls of that run outside the range; a run with no calls in the range is not counted. Null for a sub-agent session, for a session whose calls in scope mix calls with and without a run, for a run that another session of the same chat also belongs to, and when any run's billed amount is unknown (not final, missing, or not rated) or billing data could not be read. When no call in scope belongs to a run, this is the sum of those calls' costs, or null when no call reported a cost (unknown, different from a real 0). `totalCostSource` says which method was used." }, "models": { "type": "array", "items": { "type": "string" }, "description": "Distinct models used in this session" }, "allSuccess": { "type": "boolean", "description": "`false` if any turn failed" }, "errorCount": { "type": "integer", "minimum": 0, "description": "Number of failed turns in this session" }, "errorMessageSample": { "type": [ "string", "null" ], "description": "One example error message from a failed turn; null if all turns succeeded or none had a message" }, "agentId": { "type": "string", "description": "Agent ID (the same for the whole session)" }, "userId": { "type": [ "string", "null" ], "description": "User ID of the agent owner" }, "agentName": { "type": [ "string", "null" ], "description": "Agent display name" }, "userName": { "type": [ "string", "null" ], "description": "User display name" }, "workspaceId": { "type": "string", "description": "Workspace ID for the chat; empty when the task had no workspace when its traces were recorded" }, "workspaceName": { "type": [ "string", "null" ], "description": "Workspace display name; null when `workspaceId` is empty or the workspace is not found" }, "skills": { "type": "array", "items": { "type": "string" }, "description": "Distinct skill IDs invoked across all turns in this session" }, "sessionRole": { "type": [ "string", "null" ], "enum": [ "root", "subtask", "continuation", null ], "description": "`root`, `subtask` (a sub-agent session), or `continuation`. Null when the session has no role on record (sessions recorded before roles were tracked)." }, "parentSessionId": { "type": [ "string", "null" ], "description": "Session ID of the session that started this one; set for `subtask` sessions, null otherwise" }, "subagents": { "type": "object", "properties": { "count": { "type": "integer", "minimum": 0, "description": "Number of sub-agent (`subtask`) sessions started under this session, including nested ones" }, "costUsd": { "type": "number", "minimum": 0, "description": "Estimated provider cost of those sub-agent sessions' model calls in the query range (sum of per-call costs), not billed credits. Billing data is not tracked per session." } }, "required": [ "count", "costUsd" ], "additionalProperties": false, "description": "Sub-agent summary for this session; count 0 and cost 0 when it started none" }, "origin": { "anyOf": [ { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "evaluation-run" ] }, "run": { "type": [ "object", "null" ], "properties": { "runId": { "type": "string", "minLength": 1, "description": "The evaluation run that started this task (or the task its sub-agent belongs to)" }, "evaluationId": { "type": "string", "minLength": 1, "description": "The run's evaluation, so you can link to the run page" }, "evaluationName": { "type": "string", "description": "The run's evaluation name, so you can label the task with it instead of its generated title (run and task IDs)" } }, "required": [ "runId", "evaluationId", "evaluationName" ], "additionalProperties": false, "description": "The run to link to; null when no run records it anymore (its evaluation was deleted)" } }, "required": [ "kind", "run" ], "additionalProperties": false, "description": "A task started by an evaluation run, or a sub-agent of one" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "judge" ], "description": "A judge's session" }, "monitor": { "type": [ "object", "null" ], "properties": { "monitorId": { "type": "string", "minLength": 1, "description": "The monitor whose agent judge ran in this session, so you can link to it" }, "monitorName": { "type": "string", "description": "The monitor's name, so you can group the session under it instead of among the person's chats" } }, "required": [ "monitorId", "monitorName" ], "additionalProperties": false, "description": "The monitor whose verdict names this session's task as its judge. Null for the organization's judge agent grading an evaluation, a sub-agent of a judge, or a monitor judge that has not posted a verdict." } }, "required": [ "kind", "monitor" ], "additionalProperties": false, "description": "A task of the organization's judge agent grading another task's output, or a monitor's agent judge grading a person's turn" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "skill-learning-replay" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "A skill-learning replay scoring a prompt candidate" }, { "type": "object", "properties": { "kind": { "type": "string", "enum": [ "production" ] } }, "required": [ "kind" ], "additionalProperties": false, "description": "The organization's own work: classified into workflows and counted as production" } ], "description": "What started this session's chat: an evaluation run (with the run to link to), a judge, a skill-learning replay, or production. Only production sessions are included in workflows." } }, "required": [ "sessionId", "taskId", "taskTitle", "startedAt", "totalDurationMs", "turnCount", "totalTokens", "totalCost", "models", "allSuccess", "errorCount", "errorMessageSample", "agentId", "userId", "agentName", "userName", "workspaceId", "workspaceName", "skills", "sessionRole", "parentSessionId", "subagents", "origin" ], "additionalProperties": false, "description": "Session summary computed over the whole session, not only the returned window. Totals are exact no matter how many windows you have fetched." }, "calls": { "type": "array", "items": { "type": "object", "properties": { "traceId": { "type": "string", "description": "Composite trace ID: `sessionId:messageId`" }, "timestamp": { "type": "string", "description": "Turn timestamp (ISO 8601)" }, "durationMs": { "type": "number", "minimum": 0, "description": "Turn duration in milliseconds" }, "model": { "type": "string", "description": "Model used" }, "modelProvider": { "type": "string", "description": "Model provider" }, "input": { "type": [ "string", "null" ], "description": "Input messages as a JSON string. When `parts` is present, this is a 2048-character preview; the full content is in `parts` and `GET /traces/payloads`." }, "output": { "type": [ "string", "null" ], "description": "Assistant output parts as JSON. When `parts` is present, this is a 2048-character preview; render from `parts` instead." }, "inputTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Input token count" }, "outputTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Output token count" }, "reasoningTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Reasoning token count" }, "cacheReadTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Cache read token count" }, "cacheWriteTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Cache write token count" }, "costUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "Cost in USD" }, "success": { "type": "boolean", "description": "Whether this turn succeeded" }, "errorMessage": { "type": [ "string", "null" ], "description": "Error message if the turn failed" }, "workspaceId": { "type": "string", "description": "Workspace that owns the chat for this call; empty when the task had no workspace" }, "skillsUsed": { "type": "array", "items": { "type": "string" }, "description": "Skill IDs invoked in this turn" }, "parts": { "type": "array", "items": { "type": "object", "properties": { "messageId": { "type": "string", "description": "ID of the message the part belongs to" }, "partIndex": { "type": "integer", "minimum": 0, "description": "Position of the part within its source array" }, "role": { "type": "string", "enum": [ "user", "assistant" ], "description": "Message role the part was stored under" }, "kind": { "type": "string", "enum": [ "user_text", "assistant_text", "tool_call", "tool_result", "compaction", "legacy_snapshot" ], "description": "Stored content kind" }, "toolName": { "type": "string", "description": "Tool name for `tool_call` and `tool_result` parts; empty string otherwise" }, "text": { "type": "string", "description": "Searchable text of the part, capped at 64 KB (see `truncated`)" }, "truncated": { "type": "boolean", "description": "`true` when the text was capped; fetch the payload for the full content" }, "payloadKey": { "type": "object", "properties": { "chatId": { "type": "string", "description": "Stable conversation ID" }, "messageId": { "type": "string", "description": "ID of the message that owns the part" }, "partIndex": { "type": "integer", "minimum": 0, "description": "Position of the part (an identifier, not a dense sequence)" }, "day": { "type": "string", "description": "Date of the trace row that introduced the part (YYYY-MM-DD)" } }, "required": [ "chatId", "messageId", "partIndex", "day" ], "additionalProperties": false, "description": "Pass these fields to `GET /traces/payloads` to fetch the part's full raw JSON" } }, "required": [ "messageId", "partIndex", "role", "kind", "toolName", "text", "truncated", "payloadKey" ], "additionalProperties": false, "description": "One stored message part: `text` is the capped searchable text, and `payloadKey` fetches the full content" }, "description": "This call's own message parts; present only when the session is stored in the current trace format" }, "turnIndex": { "type": "integer", "minimum": 0, "description": "Index into the response's `turns` array; present only when the session is stored in the current trace format" } }, "required": [ "traceId", "timestamp", "durationMs", "model", "modelProvider", "input", "output", "inputTokens", "outputTokens", "reasoningTokens", "cacheReadTokens", "cacheWriteTokens", "costUsd", "success", "errorMessage", "workspaceId", "skillsUsed" ], "additionalProperties": false, "description": "One model turn within a trace session" }, "description": "This window's turns, ordered by timestamp, then trace ID, ascending" }, "turns": { "type": "array", "items": { "type": "object", "properties": { "turnIndex": { "type": "integer", "minimum": 0, "description": "Turn position within the session, starting at 0" }, "userText": { "type": [ "string", "null" ], "description": "Text of the user message that started the turn; null for continuation segments or segments after compaction" }, "traceIds": { "type": "array", "items": { "type": "string" }, "description": "Trace IDs of the calls that make up this turn, in chronological order" } }, "required": [ "turnIndex", "userText", "traceIds" ], "additionalProperties": false, "description": "Turn boundary computed by the server (current trace format only)" }, "description": "Turn boundaries for this window, computed by the server (`turnIndex` counts from the start of this window). Present only when the session is stored in the current trace format. When joining windows, offset the indexes by the number of turns collected so far, and merge the first turn into the previous window's last turn when `windowStartsMidTurn` is true." }, "nextCursor": { "type": [ "string", "null" ], "description": "Token for the next window; null when the whole session has been returned. You can fetch any amount; keep following windows." }, "totalCostSource": { "type": "string", "enum": [ "billing_ledger", "warehouse" ], "description": "Where the session's `totalCost` comes from. `billing_ledger`: the session's calls in scope (inside the requested range, if one is given) belong to task runs, and `totalCost` is the billed amount for those runs, including sub-agents' and auxiliary calls (null while the billed amount cannot be determined). `warehouse`: no call in scope belongs to a run, and `totalCost` is the sum of those calls' costs, which excludes sub-agents." }, "workflows": { "anyOf": [ { "type": "object", "properties": { "state": { "type": "string", "enum": [ "loaded" ] }, "workflows": { "type": "array", "items": { "type": "object", "properties": { "topicId": { "type": "string", "description": "Workflow ID; the workflow page is `/client/evals/workflows/t/`" }, "name": { "type": "string", "description": "Workflow display name" } }, "required": [ "topicId", "name" ], "additionalProperties": false, "description": "An active workflow the session's chat belongs to" }, "description": "The chat's current workflows, most confident first; empty when the chat belongs to none (unassigned)" } }, "required": [ "state", "workflows" ], "additionalProperties": false, "description": "Workflow membership read from the chat's current workflows" }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "unavailable" ] } }, "required": [ "state" ], "additionalProperties": false, "description": "Workflow membership could not be read; trace data is still available" }, { "type": "object", "properties": { "state": { "type": "string", "enum": [ "withheld" ] } }, "required": [ "state" ], "additionalProperties": false, "description": "Workflow names are hidden from this caller (workflows are turned off for the organization, or the caller lacks organization-wide evaluations access); membership was not read" } ], "description": "The workflows of the session's chat, read the same way as in the list row" }, "windowStartsMidTurn": { "type": "boolean", "description": "`true` when this window's first calls continue a turn that started in a previous window (no boundary before them); use it to join windows. `false` or absent when the first turn starts in this window (with user text or compaction) or this is the first window of a continuation session. Current trace format only." } }, "required": [ "session", "calls", "nextCursor", "totalCostSource", "workflows" ], "additionalProperties": false, "description": "One window of session detail, with a summary of the whole session; follow `nextCursor` for the rest" }, "example": { "session": { "sessionId": "09724085-1664-4183-8554-bacab2816579", "taskId": "de707028-6ea2-4616-8864-c2ebd92b87f5", "taskTitle": "text", "startedAt": "text", "totalDurationMs": 0.75, "turnCount": 1, "totalTokens": 1, "totalCost": 0.75, "models": [ "text" ], "allSuccess": true, "errorCount": 1, "errorMessageSample": "text", "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "userId": "6ccb2121-4ffd-40b0-8c2c-1607cf6a05be", "agentName": "text", "userName": "text", "workspaceId": "09707fa5-aaae-4493-8a23-827111000f89", "workspaceName": "text", "skills": [ "text" ], "sessionRole": "root", "parentSessionId": "35de55cd-12cf-465c-8d0a-7317274054a7", "subagents": { "count": 1, "costUsd": 0.75 }, "origin": { "kind": "evaluation-run", "run": { "runId": "599e5591-3334-4d0a-8bb9-069b278d9b20", "evaluationId": "66248a43-3af8-4b5b-86b6-fcb59feb38b2", "evaluationName": "text" } } }, "calls": [ { "traceId": "3cd81d34-6c57-46ba-844a-64899bb8721d", "timestamp": "text", "durationMs": 0.75, "model": "anthropic/claude-sonnet-5", "modelProvider": "text", "input": "text", "output": "text", "inputTokens": 1, "outputTokens": 1, "reasoningTokens": 1, "cacheReadTokens": 1, "cacheWriteTokens": 1, "costUsd": 0.75, "success": true, "errorMessage": "text", "workspaceId": "09707fa5-aaae-4493-8a23-827111000f89", "skillsUsed": [ "text" ], "parts": [ { "messageId": "ae2bea39-b23d-4c35-81aa-f3f821c61382", "partIndex": 1, "role": "user", "kind": "user_text", "toolName": "text", "text": "text", "truncated": true, "payloadKey": { "chatId": "18f0adc1-7acd-4c24-86e2-35315d7f8e76", "messageId": "ae2bea39-b23d-4c35-81aa-f3f821c61382", "partIndex": 1, "day": "text" } } ], "turnIndex": 1 } ], "turns": [ { "turnIndex": 1, "userText": "text", "traceIds": [ "7285e3a6-1049-4749-8a7a-ec68011b96ac" ] } ], "nextCursor": "text", "totalCostSource": "billing_ledger", "workflows": { "state": "loaded", "workflows": [ { "topicId": "c42387f0-94d3-427e-80d1-a63853a68624", "name": "Refund questions" } ] }, "windowStartsMidTurn": true } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`), or its owner is not a member of the organization. A key limited to specific agents is not refused here: it reads those agents' traces only.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Trace data can't be read: this deployment has no trace store configured (code `CLICKHOUSE_UNAVAILABLE`), or the server could not yet tell which store holds this range (`TRACE_READ_SOURCE_UNRESOLVED`, retry). A trace store that fails during the query answers 500 with code `QUERY_FAILED`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "traces", "description": "An unknown session ID answers 200 with an empty session (`turnCount` 0, `taskId` null) and no calls, not 404.", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/traces/sessions/$SESSION_ID\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nsession_id = os.environ[\"SESSION_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/traces/sessions/{session_id}\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/traces/sessions/${encodeURIComponent(env(\"SESSION_ID\"))}`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# One session: the file lands under ~/.context-cli/runs/, summary on stdout\ncontext evals traces get $SESSION_ID\n\n# Explicit output path\ncontext evals traces get $SESSION_ID --out /tmp/session.json" } ] } }, "/traces/calls": { "get": { "tags": [ "Traces" ], "summary": "List model calls", "description": "Paginated flat list of calls (one row per model turn), with the same filter and search syntax as the session list but applied to each call. Numeric filters (`cost_usd`, `duration_ms`, `input_tokens`, `output_tokens`) compare the call's own value rather than a session total. Each row includes `sessionId`, `taskId`, `taskTitle`, `agentName`, `userName`, and `workspaceName`, so the response is self-contained for bulk export. Rows carry `inputPreview` and `outputPreview` (first 2048 characters) instead of full `input` and `output`; use the session detail endpoint for the full text. `totalCount` stops at 10,000; `totalCountIsCapped: true` marks a capped count.", "operationId": "listTraceCalls", "parameters": [ { "in": "query", "name": "from", "required": true, "schema": { "type": "string", "format": "date-time", "description": "Start of the date range (ISO 8601)" } }, { "in": "query", "name": "to", "required": true, "schema": { "type": "string", "format": "date-time", "description": "End of the date range (ISO 8601)" } }, { "in": "query", "name": "page", "required": false, "schema": { "type": "integer", "minimum": 1, "description": "Page number, starting at 1", "default": 1 } }, { "in": "query", "name": "pageSize", "required": false, "schema": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Calls per page", "default": 15 } }, { "in": "query", "name": "search", "required": false, "schema": { "type": "string", "maxLength": 500, "description": "Token search over trace text. Terms match as whole tokens (runs of letters, digits, or non-ASCII characters; punctuation and underscores split terms). Matching ignores ASCII case, and every term must match. A `*` widens a term: `zip*` matches words starting with zip (zip, zipped, zipping), `*ip` matches words ending in `ip`, and `*ognit*` matches any text containing `ognit`. A match covers the whole conversation: every turn of a conversation whose text contains all the terms is returned, not only the turn that introduced them, and the terms may appear in different turns. Terms must occur inside the queried time range. Input with no searchable tokens (punctuation only) returns 400. Some time ranges are served from an older data source that matches terms by substring instead; this response does not say which (`GET /traces/sessions` does, in `readSource`)." } }, { "in": "query", "name": "filters", "required": false, "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "field": { "type": "string", "enum": [ "success", "cost_usd", "duration_ms", "model", "model_provider", "input_tokens", "output_tokens", "error_message", "agent_id", "user_id", "workspace_id", "chat_id", "skill_id", "origin" ], "description": "Field to filter on" }, "operator": { "type": "string", "enum": [ "equals", "not_equals", "greater_than", "less_than", "gte", "lte", "contains", "in", "not_in" ], "description": "Comparison operator for a filter condition" }, "value": { "type": "string", "minLength": 1, "description": "Filter value as a string, interpreted according to the field's type" } }, "required": [ "field", "operator", "value" ], "additionalProperties": false, "description": "One filter condition: a field, an operator, and a value" }, "maxItems": 20, "description": "Filter conditions, applied per call: numeric filters compare each call's own values" } } }, "description": "Send it as JSON in one value, for example `filters=[{\"field\":\"success\",\"operator\":\"equals\",\"value\":\"false\"}]`. Any other encoding is ignored." }, { "in": "query", "name": "matchAll", "required": false, "schema": { "type": "boolean", "description": "Combine filter conditions with AND (true) or with OR (false)", "default": true } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "data": { "type": "array", "items": { "type": "object", "properties": { "traceId": { "type": "string", "description": "Composite trace ID: `sessionId:messageId`" }, "timestamp": { "type": "string", "description": "Turn timestamp (ISO 8601)" }, "durationMs": { "type": "number", "minimum": 0, "description": "Turn duration in milliseconds" }, "model": { "type": "string", "description": "Model used" }, "modelProvider": { "type": "string", "description": "Model provider" }, "inputTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Input token count" }, "outputTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Output token count" }, "reasoningTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Reasoning token count" }, "cacheReadTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Cache read token count" }, "cacheWriteTokens": { "type": [ "integer", "null" ], "minimum": 0, "description": "Cache write token count" }, "costUsd": { "type": [ "number", "null" ], "minimum": 0, "description": "Cost in USD" }, "success": { "type": "boolean", "description": "Whether this turn succeeded" }, "errorMessage": { "type": [ "string", "null" ], "description": "Error message if the turn failed" }, "workspaceId": { "type": "string", "description": "Workspace that owns the chat for this call; empty when the task had no workspace" }, "skillsUsed": { "type": "array", "items": { "type": "string" }, "description": "Skill IDs invoked in this turn" }, "parts": { "type": "array", "items": { "type": "object", "properties": { "messageId": { "type": "string", "description": "ID of the message the part belongs to" }, "partIndex": { "type": "integer", "minimum": 0, "description": "Position of the part within its source array" }, "role": { "type": "string", "enum": [ "user", "assistant" ], "description": "Message role the part was stored under" }, "kind": { "type": "string", "enum": [ "user_text", "assistant_text", "tool_call", "tool_result", "compaction", "legacy_snapshot" ], "description": "Stored content kind" }, "toolName": { "type": "string", "description": "Tool name for `tool_call` and `tool_result` parts; empty string otherwise" }, "text": { "type": "string", "description": "Searchable text of the part, capped at 64 KB (see `truncated`)" }, "truncated": { "type": "boolean", "description": "`true` when the text was capped; fetch the payload for the full content" }, "payloadKey": { "type": "object", "properties": { "chatId": { "type": "string", "description": "Stable conversation ID" }, "messageId": { "type": "string", "description": "ID of the message that owns the part" }, "partIndex": { "type": "integer", "minimum": 0, "description": "Position of the part (an identifier, not a dense sequence)" }, "day": { "type": "string", "description": "Date of the trace row that introduced the part (YYYY-MM-DD)" } }, "required": [ "chatId", "messageId", "partIndex", "day" ], "additionalProperties": false, "description": "Pass these fields to `GET /traces/payloads` to fetch the part's full raw JSON" } }, "required": [ "messageId", "partIndex", "role", "kind", "toolName", "text", "truncated", "payloadKey" ], "additionalProperties": false, "description": "One stored message part: `text` is the capped searchable text, and `payloadKey` fetches the full content" }, "description": "This call's own message parts; present only when the session is stored in the current trace format" }, "turnIndex": { "type": "integer", "minimum": 0, "description": "Index into the response's `turns` array; present only when the session is stored in the current trace format" }, "inputPreview": { "type": [ "string", "null" ], "description": "First 2048 characters of the input (empty string when the call had no input). The full text is available from the session detail endpoint." }, "outputPreview": { "type": [ "string", "null" ], "description": "First 2048 characters of the output (empty string when the call had no output). The full text is available from the session detail endpoint." }, "sessionId": { "type": "string", "description": "Session ID this call belongs to" }, "chatId": { "type": "string", "description": "Stable chat (conversation) ID" }, "taskId": { "type": [ "string", "null" ], "description": "ID of the task linked to this chat; null when no task is linked" }, "taskTitle": { "type": [ "string", "null" ], "description": "Title of the linked task; null when no task is linked" }, "agentId": { "type": "string", "description": "ID of the agent that made this call" }, "agentName": { "type": [ "string", "null" ], "description": "Agent display name" }, "userId": { "type": [ "string", "null" ], "description": "User ID of the agent owner" }, "userName": { "type": [ "string", "null" ], "description": "User display name" }, "workspaceName": { "type": [ "string", "null" ], "description": "Workspace display name; null when `workspaceId` is empty or unknown" } }, "required": [ "traceId", "timestamp", "durationMs", "model", "modelProvider", "inputTokens", "outputTokens", "reasoningTokens", "cacheReadTokens", "cacheWriteTokens", "costUsd", "success", "errorMessage", "workspaceId", "skillsUsed", "inputPreview", "outputPreview", "sessionId", "chatId", "taskId", "taskTitle", "agentId", "agentName", "userId", "userName", "workspaceName" ], "additionalProperties": false, "description": "One model call with its related names and IDs" } }, "totalCount": { "type": "integer", "minimum": 0, "description": "Number of matching calls for pagination, counted up to a cap of 10,000" }, "totalCountIsCapped": { "type": "boolean", "description": "`true` when more calls matched than the cap; `totalCount` is then the cap, not the exact total" } }, "required": [ "data", "totalCount", "totalCountIsCapped" ], "additionalProperties": false, "description": "Paginated flat list of calls" }, "example": { "data": [ { "traceId": "3cd81d34-6c57-46ba-844a-64899bb8721d", "timestamp": "text", "durationMs": 0.75, "model": "anthropic/claude-sonnet-5", "modelProvider": "text", "inputTokens": 1, "outputTokens": 1, "reasoningTokens": 1, "cacheReadTokens": 1, "cacheWriteTokens": 1, "costUsd": 0.75, "success": true, "errorMessage": "text", "workspaceId": "09707fa5-aaae-4493-8a23-827111000f89", "skillsUsed": [ "text" ], "parts": [ { "messageId": "ae2bea39-b23d-4c35-81aa-f3f821c61382", "partIndex": 1, "role": "user", "kind": "user_text", "toolName": "text", "text": "text", "truncated": true, "payloadKey": { "chatId": "18f0adc1-7acd-4c24-86e2-35315d7f8e76", "messageId": "ae2bea39-b23d-4c35-81aa-f3f821c61382", "partIndex": 1, "day": "text" } } ], "turnIndex": 1, "inputPreview": "text", "outputPreview": "text", "sessionId": "09724085-1664-4183-8554-bacab2816579", "chatId": "18f0adc1-7acd-4c24-86e2-35315d7f8e76", "taskId": "de707028-6ea2-4616-8864-c2ebd92b87f5", "taskTitle": "text", "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "agentName": "text", "userId": "6ccb2121-4ffd-40b0-8c2c-1607cf6a05be", "userName": "text", "workspaceName": "text" } ], "totalCount": 1, "totalCountIsCapped": true } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`), or its owner is not a member of the organization. A key limited to specific agents is not refused here: it reads those agents' traces only.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Trace data can't be read: this deployment has no trace store configured (code `CLICKHOUSE_UNAVAILABLE`), or the server could not yet tell which store holds this range (`TRACE_READ_SOURCE_UNRESOLVED`, retry). A trace store that fails during the query answers 500 with code `QUERY_FAILED`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "traces", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/traces/calls\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode 'from=2026-10-01T00:00:00Z' \\\n --data-urlencode 'to=2026-10-31T00:00:00Z'" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/traces/calls\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"from\": \"2026-10-01T00:00:00Z\", \"to\": \"2026-10-31T00:00:00Z\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/traces/calls`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"from\", \"2026-10-01T00:00:00Z\")\nurl.searchParams.set(\"to\", \"2026-10-31T00:00:00Z\")\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/traces/payloads": { "get": { "tags": [ "Traces" ], "summary": "Get a message part's full content", "description": "Looks up the raw JSON of one stored message part by key (complete tool results, every nested field). The session detail endpoint returns the key fields (`chatId`, `messageId`, `partIndex`, `day`) on each part's `payloadKey`; pass them back unchanged. Returns 404 for unknown keys.", "operationId": "getTracePartPayload", "parameters": [ { "in": "query", "name": "chatId", "required": true, "schema": { "type": "string", "minLength": 1, "description": "Stable conversation ID the part belongs to" } }, { "in": "query", "name": "messageId", "required": true, "schema": { "type": "string", "minLength": 1, "description": "ID of the message (`msg_...`) that owns the part" } }, { "in": "query", "name": "partIndex", "required": true, "schema": { "type": "integer", "minimum": 0, "description": "Position of the part within its source array (an identifier, not a dense sequence)" } }, { "in": "query", "name": "day", "required": true, "schema": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$", "description": "Date of the trace row that introduced the part (YYYY-MM-DD); narrows the lookup to one day" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "payload": { "type": "string", "description": "Raw part JSON exactly as stored (complete tool results, every nested field)" }, "partType": { "type": "string", "description": "Raw part type: `text`, `tool`, `file`, `step-start`, `compaction`, and others" } }, "required": [ "payload", "partType" ], "additionalProperties": false, "description": "Full payload for one stored message part" }, "example": { "payload": "text", "partType": "text" } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`), or its owner is not a member of the organization. A key limited to specific agents is not refused here: it reads those agents' traces only.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "404": { "description": "Not found in your organization", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Trace data can't be read: this deployment has no trace store configured (code `CLICKHOUSE_UNAVAILABLE`), or the server could not yet tell which store holds this range (`TRACE_READ_SOURCE_UNRESOLVED`, retry). A trace store that fails during the query answers 500 with code `QUERY_FAILED`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "traces", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\n# chatId, messageId, partIndex and day: a part's payloadKey from GET /traces/sessions/{sessionId}\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/traces/payloads\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode \"chatId=$CHAT_ID\" \\\n --data-urlencode \"messageId=$MESSAGE_ID\" \\\n --data-urlencode \"partIndex=$PART_INDEX\" \\\n --data-urlencode \"day=$DAY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\n# chatId, messageId, partIndex and day: a part's payloadKey from GET /traces/sessions/{sessionId}\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/traces/payloads\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"chatId\": os.environ[\"CHAT_ID\"], \"messageId\": os.environ[\"MESSAGE_ID\"], \"partIndex\": int(os.environ[\"PART_INDEX\"]), \"day\": os.environ[\"DAY\"]},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\n// chatId, messageId, partIndex and day: a part's payloadKey from GET /traces/sessions/{sessionId}\nconst url = new URL(`/api/v1/evals/traces/payloads`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"chatId\", env(\"CHAT_ID\"))\nurl.searchParams.set(\"messageId\", env(\"MESSAGE_ID\"))\nurl.searchParams.set(\"partIndex\", env(\"PART_INDEX\"))\nurl.searchParams.set(\"day\", env(\"DAY\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/traces/histogram": { "get": { "tags": [ "Traces" ], "summary": "Get the session histogram", "operationId": "getTraceHistogram", "parameters": [ { "in": "query", "name": "from", "required": true, "schema": { "type": "string", "format": "date-time", "description": "Start of the date range (ISO 8601)" } }, { "in": "query", "name": "to", "required": true, "schema": { "type": "string", "format": "date-time", "description": "End of the date range (ISO 8601)" } }, { "in": "query", "name": "search", "required": false, "schema": { "type": "string", "maxLength": 500, "description": "Token search over trace text. Terms match as whole tokens (runs of letters, digits, or non-ASCII characters; punctuation and underscores split terms). Matching ignores ASCII case, and every term must match. A `*` widens a term: `zip*` matches words starting with zip (zip, zipped, zipping), `*ip` matches words ending in `ip`, and `*ognit*` matches any text containing `ognit`. A match covers the whole conversation: every turn of a conversation whose text contains all the terms is returned, not only the turn that introduced them, and the terms may appear in different turns. Terms must occur inside the queried time range. Input with no searchable tokens (punctuation only) returns 400. Some time ranges are served from an older data source that matches terms by substring instead; this response does not say which (`GET /traces/sessions` does, in `readSource`)." } }, { "in": "query", "name": "filters", "required": false, "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "field": { "type": "string", "enum": [ "success", "cost_usd", "duration_ms", "model", "model_provider", "input_tokens", "output_tokens", "error_message", "agent_id", "user_id", "workspace_id", "chat_id", "skill_id", "origin" ], "description": "Field to filter on" }, "operator": { "type": "string", "enum": [ "equals", "not_equals", "greater_than", "less_than", "gte", "lte", "contains", "in", "not_in" ], "description": "Comparison operator for a filter condition" }, "value": { "type": "string", "minLength": 1, "description": "Filter value as a string, interpreted according to the field's type" } }, "required": [ "field", "operator", "value" ], "additionalProperties": false, "description": "One filter condition: a field, an operator, and a value" }, "maxItems": 20 } } }, "description": "Send it as JSON in one value, for example `filters=[{\"field\":\"success\",\"operator\":\"equals\",\"value\":\"false\"}]`. Any other encoding is ignored." }, { "in": "query", "name": "matchAll", "required": false, "schema": { "type": "boolean", "default": true } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "buckets": { "type": "array", "items": { "type": "object", "properties": { "bucket": { "type": "string", "description": "Start of the time bucket (ISO 8601)" }, "total": { "type": "integer", "minimum": 0, "description": "Total sessions in the bucket" }, "errors": { "type": "integer", "minimum": 0, "description": "Sessions with at least one failure" } }, "required": [ "bucket", "total", "errors" ], "additionalProperties": false, "description": "One histogram time bucket" } } }, "required": [ "buckets" ], "additionalProperties": false, "description": "Session counts over time" }, "example": { "buckets": [ { "bucket": "text", "total": 1, "errors": 1 } ] } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`), or its owner is not a member of the organization. A key limited to specific agents is not refused here: it reads those agents' traces only.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Trace data can't be read: this deployment has no trace store configured (code `CLICKHOUSE_UNAVAILABLE`), or the server could not yet tell which store holds this range (`TRACE_READ_SOURCE_UNRESOLVED`, retry). A trace store that fails during the query answers 500 with code `QUERY_FAILED`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "traces", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/traces/histogram\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode 'from=2026-10-01T00:00:00Z' \\\n --data-urlencode 'to=2026-10-31T00:00:00Z'" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/traces/histogram\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"from\": \"2026-10-01T00:00:00Z\", \"to\": \"2026-10-31T00:00:00Z\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/traces/histogram`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"from\", \"2026-10-01T00:00:00Z\")\nurl.searchParams.set(\"to\", \"2026-10-31T00:00:00Z\")\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } }, "/traces/filter-options": { "get": { "tags": [ "Traces" ], "summary": "List trace filter values", "operationId": "getTraceFilterOptions", "parameters": [], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "models": { "type": "array", "items": { "type": "string" }, "description": "Distinct model names from recent traces" }, "providers": { "type": "array", "items": { "type": "string" }, "description": "Distinct provider names from recent traces" }, "agents": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "name": { "type": "string", "minLength": 1 } }, "required": [ "id", "name" ], "additionalProperties": false, "description": "Filter option with an ID and display name" }, "description": "Agents with trace data" }, "users": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "name": { "type": "string", "minLength": 1 } }, "required": [ "id", "name" ], "additionalProperties": false, "description": "Filter option with an ID and display name" }, "description": "Users who own agents with trace data" }, "workspaces": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "name": { "type": "string", "minLength": 1 } }, "required": [ "id", "name" ], "additionalProperties": false, "description": "Filter option with an ID and display name" }, "description": "First page of workspaces that appear in recent traces, with names (capped; see `workspacesTruncated`). Search for workspaces in the app to find the full set." }, "workspacesTruncated": { "type": "boolean", "description": "`true` when more workspaces appear in recent traces than this list holds, so the list is incomplete" }, "skills": { "type": "array", "items": { "type": "object", "properties": { "id": { "type": "string", "minLength": 1 }, "name": { "type": "string", "minLength": 1 } }, "required": [ "id", "name" ], "additionalProperties": false, "description": "Filter option with an ID and display name" }, "description": "Skills invoked in recent traces, with names" } }, "required": [ "models", "providers", "agents", "users", "workspaces", "workspacesTruncated", "skills" ], "additionalProperties": false, "description": "Available values for filters" }, "example": { "models": [ "text" ], "providers": [ "text" ], "agents": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" } ], "users": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" } ], "workspaces": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" } ], "workspacesTruncated": true, "skills": [ { "id": "a5614527-0ce6-43be-8d1d-d012b7394867", "name": "Refund questions" } ] } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`), or its owner is not a member of the organization. A key limited to specific agents is not refused here: it reads those agents' traces only.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Trace data can't be read: this deployment has no trace store configured (code `CLICKHOUSE_UNAVAILABLE`), or the server could not yet tell which store holds this range (`TRACE_READ_SOURCE_UNRESOLVED`, retry). A trace store that fails during the query answers 500 with code `QUERY_FAILED`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "traces", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/traces/filter-options\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\"" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/traces/filter-options\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/traces/filter-options`, env(\"CONTEXT_APP_ORIGIN\"))\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Find agents by name\ncontext evals filter-options | jq '.agents[] | select(.name | contains(\"marketing\"))'\n\n# Snapshot every agent's monthly profile to its own file\ncontext evals filter-options | jq -r '.agents[].id' | xargs -I {} context evals agent {} --from 30d --out runs/agent-{}.json" } ] } }, "/traces/agents/{agentId}/profile": { "get": { "tags": [ "Traces" ], "summary": "Get an agent's usage profile", "operationId": "getAgentProfile", "parameters": [ { "in": "path", "name": "agentId", "required": true, "description": "Agent ID", "schema": { "type": "string", "minLength": 1, "description": "Agent ID" } }, { "in": "query", "name": "since", "required": true, "schema": { "type": "string", "format": "date-time" } }, { "in": "query", "name": "until", "required": false, "schema": { "type": "string", "format": "date-time" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "totalSessions": { "type": "integer", "minimum": 0, "description": "Total number of sessions" }, "totalCost": { "type": "number", "minimum": 0, "description": "Total cost in USD" }, "totalDurationMs": { "type": "number", "minimum": 0, "description": "Total duration in milliseconds" }, "avgSessionDurationMs": { "type": "number", "minimum": 0, "description": "Average session duration in milliseconds" }, "avgCostPerSession": { "type": "number", "minimum": 0, "description": "Average cost per session in USD" }, "totalTokens": { "type": "integer", "minimum": 0, "description": "Billed tokens (input, output, reasoning, cache write, and cache read)" }, "models": { "type": "array", "items": { "type": "string" }, "description": "Distinct models used" }, "dailyCost": { "type": "array", "items": { "type": "object", "properties": { "date": { "type": "string", "description": "Date (YYYY-MM-DD)" }, "cost": { "type": "number", "minimum": 0, "description": "Total cost in USD for this day" }, "tokens": { "type": "integer", "minimum": 0, "description": "Billed tokens for this day" } }, "required": [ "date", "cost" ], "additionalProperties": false }, "description": "Daily cost breakdown, oldest date first" }, "modelBreakdown": { "type": "array", "items": { "type": "object", "properties": { "model": { "type": "string", "description": "Model ID" }, "totalCost": { "type": "number", "minimum": 0, "description": "Total cost in USD for this model" }, "totalTokens": { "type": "integer", "minimum": 0, "description": "Billed tokens for this model" } }, "required": [ "model", "totalCost", "totalTokens" ], "additionalProperties": false }, "description": "Per-model cost and token usage" } }, "required": [ "totalSessions", "totalCost", "totalDurationMs", "avgSessionDurationMs", "avgCostPerSession", "totalTokens", "models", "dailyCost" ], "additionalProperties": false }, "example": { "totalSessions": 1, "totalCost": 0.75, "totalDurationMs": 0.75, "avgSessionDurationMs": 0.75, "avgCostPerSession": 0.75, "totalTokens": 1, "models": [ "text" ], "dailyCost": [ { "date": "text", "cost": 0.75, "tokens": 1 } ], "modelBreakdown": [ { "model": "anthropic/claude-sonnet-5", "totalCost": 0.75, "totalTokens": 1 } ] } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`), or its owner is not a member of the organization. A key limited to specific agents is not refused here: it reads those agents' traces only.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Trace data can't be read: this deployment has no trace store configured (code `CLICKHOUSE_UNAVAILABLE`), or the server could not yet tell which store holds this range (`TRACE_READ_SOURCE_UNRESOLVED`, retry). A trace store that fails during the query answers 500 with code `QUERY_FAILED`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "traces", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/traces/agents/$AGENT_ID/profile\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode 'since=2026-10-01T00:00:00Z'" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nagent_id = os.environ[\"AGENT_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/traces/agents/{agent_id}/profile\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"since\": \"2026-10-01T00:00:00Z\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/traces/agents/${encodeURIComponent(env(\"AGENT_ID\"))}/profile`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"since\", \"2026-10-01T00:00:00Z\")\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Last-30-day cost / token / daily-cost profile for one agent\ncontext evals agent $AGENT_ID --from 30d" } ] } }, "/traces/users/{userId}/profile": { "get": { "tags": [ "Traces" ], "summary": "Get a user's usage profile", "operationId": "getUserProfile", "parameters": [ { "in": "path", "name": "userId", "required": true, "description": "User ID", "schema": { "type": "string", "minLength": 1, "description": "User ID" } }, { "in": "query", "name": "since", "required": true, "schema": { "type": "string", "format": "date-time" } }, { "in": "query", "name": "until", "required": false, "schema": { "type": "string", "format": "date-time" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "totalSessions": { "type": "integer", "minimum": 0, "description": "Total number of sessions" }, "totalCost": { "type": "number", "minimum": 0, "description": "Total cost in USD" }, "totalDurationMs": { "type": "number", "minimum": 0, "description": "Total duration in milliseconds" }, "avgSessionDurationMs": { "type": "number", "minimum": 0, "description": "Average session duration in milliseconds" }, "avgCostPerSession": { "type": "number", "minimum": 0, "description": "Average cost per session in USD" }, "totalTokens": { "type": "integer", "minimum": 0, "description": "Billed tokens (input, output, reasoning, cache write, and cache read)" }, "models": { "type": "array", "items": { "type": "string" }, "description": "Distinct models used" }, "dailyCost": { "type": "array", "items": { "type": "object", "properties": { "date": { "type": "string", "description": "Date (YYYY-MM-DD)" }, "cost": { "type": "number", "minimum": 0, "description": "Total cost in USD for this day" }, "tokens": { "type": "integer", "minimum": 0, "description": "Billed tokens for this day" } }, "required": [ "date", "cost" ], "additionalProperties": false }, "description": "Daily cost breakdown, oldest date first" }, "modelBreakdown": { "type": "array", "items": { "type": "object", "properties": { "model": { "type": "string", "description": "Model ID" }, "totalCost": { "type": "number", "minimum": 0, "description": "Total cost in USD for this model" }, "totalTokens": { "type": "integer", "minimum": 0, "description": "Billed tokens for this model" } }, "required": [ "model", "totalCost", "totalTokens" ], "additionalProperties": false }, "description": "Per-model cost and token usage" }, "agentBreakdown": { "type": "array", "items": { "type": "object", "properties": { "agentId": { "type": "string", "description": "Agent ID" }, "agentName": { "type": [ "string", "null" ], "description": "Agent display name" }, "sessionCount": { "type": "integer", "minimum": 0, "description": "Number of sessions for this agent" }, "totalCost": { "type": "number", "minimum": 0, "description": "Total cost in USD for this agent" }, "totalTokens": { "type": "integer", "minimum": 0, "description": "Billed tokens used by this agent" } }, "required": [ "agentId", "agentName", "sessionCount", "totalCost", "totalTokens" ], "additionalProperties": false }, "description": "Per-agent usage breakdown" } }, "required": [ "totalSessions", "totalCost", "totalDurationMs", "avgSessionDurationMs", "avgCostPerSession", "totalTokens", "models", "dailyCost", "agentBreakdown" ], "additionalProperties": false }, "example": { "totalSessions": 1, "totalCost": 0.75, "totalDurationMs": 0.75, "avgSessionDurationMs": 0.75, "avgCostPerSession": 0.75, "totalTokens": 1, "models": [ "text" ], "dailyCost": [ { "date": "text", "cost": 0.75, "tokens": 1 } ], "modelBreakdown": [ { "model": "anthropic/claude-sonnet-5", "totalCost": 0.75, "totalTokens": 1 } ], "agentBreakdown": [ { "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "agentName": "text", "sessionCount": 1, "totalCost": 0.75, "totalTokens": 1 } ] } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`), or its owner is not a member of the organization. A key limited to specific agents is not refused here: it reads those agents' traces only.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "503": { "description": "Trace data can't be read: this deployment has no trace store configured (code `CLICKHOUSE_UNAVAILABLE`), or the server could not yet tell which store holds this range (`TRACE_READ_SOURCE_UNRESOLVED`, retry). A trace store that fails during the query answers 500 with code `QUERY_FAILED`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "traces", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/traces/users/$USER_ID/profile\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode 'since=2026-10-01T00:00:00Z'" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\nuser_id = os.environ[\"USER_ID\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/traces/users/{user_id}/profile\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"since\": \"2026-10-01T00:00:00Z\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/traces/users/${encodeURIComponent(env(\"USER_ID\"))}/profile`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"since\", \"2026-10-01T00:00:00Z\")\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" }, { "lang": "shell", "label": "CLI", "source": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Last-7-day profile for one user, including per-agent breakdown\ncontext evals user $USER_ID --from 7d" } ] } }, "/traces/cost-dashboard": { "get": { "tags": [ "Traces" ], "summary": "Get the cost dashboard", "operationId": "getOrgCostDashboard", "parameters": [ { "in": "query", "name": "since", "required": true, "schema": { "type": "string", "format": "date-time", "description": "Start date (ISO 8601)" } }, { "in": "query", "name": "until", "required": false, "schema": { "type": "string", "format": "date-time", "description": "End date (ISO 8601); defaults to now" } } ], "responses": { "200": { "description": "Successful response", "content": { "application/json": { "schema": { "type": "object", "properties": { "agents": { "type": "array", "items": { "type": "object", "properties": { "agentId": { "type": "string", "description": "Agent ID" }, "agentName": { "type": [ "string", "null" ], "description": "Agent display name" }, "workspaceName": { "type": [ "string", "null" ], "description": "Workspace (team) name" }, "totalSessions": { "type": "integer", "minimum": 0, "description": "Total sessions in the period" }, "totalCost": { "type": "number", "minimum": 0, "description": "Total cost in USD" }, "avgCostPerSession": { "type": "number", "minimum": 0, "description": "Average cost per session in USD" }, "avgDurationMs": { "type": "number", "minimum": 0, "description": "Average session duration in milliseconds" }, "totalTokens": { "type": "integer", "minimum": 0, "description": "Billed tokens (input, output, reasoning, cache write, and cache read)" }, "trend": { "type": [ "number", "null" ], "description": "Percent change from the previous period of the same length; null when there is no previous data" } }, "required": [ "agentId", "agentName", "workspaceName", "totalSessions", "totalCost", "avgCostPerSession", "avgDurationMs", "totalTokens", "trend" ], "additionalProperties": false }, "description": "Per-agent cost rows, highest `totalCost` first (includes media generation cost)" }, "orgTotalCost": { "type": "number", "minimum": 0, "description": "Organization-wide total cost in USD for the period, including media and image generation" }, "orgTotalSessions": { "type": "integer", "minimum": 0, "description": "Organization-wide total sessions for the period" }, "orgAvgCostPerSession": { "type": "number", "minimum": 0, "description": "Organization-wide average cost per session in USD" }, "orgTotalDurationMs": { "type": "number", "minimum": 0, "description": "Organization-wide total session duration in milliseconds" }, "dailyCost": { "type": "array", "items": { "type": "object", "properties": { "date": { "type": "string", "description": "Date (YYYY-MM-DD)" }, "cost": { "type": "number", "minimum": 0, "description": "Total cost in USD for this day" }, "tokens": { "type": "integer", "minimum": 0, "description": "Billed tokens for this day" } }, "required": [ "date", "cost" ], "additionalProperties": false }, "description": "Daily cost breakdown, oldest date first (includes media generation cost)" }, "mediaGeneration": { "type": "object", "properties": { "totalCost": { "type": "number", "minimum": 0, "description": "Total billed media and image generation cost in USD for the period" }, "generationCount": { "type": "integer", "minimum": 0, "description": "Total billed media and image generations in the period" }, "byModel": { "type": "array", "items": { "type": "object", "properties": { "modelId": { "type": "string", "description": "Provider model or endpoint ID (for example, a fal endpoint or gpt-image model); `unknown` when none was recorded" }, "totalCost": { "type": "number", "minimum": 0, "description": "Billed cost in USD for this model in the period" }, "generationCount": { "type": "integer", "minimum": 0, "description": "Number of billed generations for this model" } }, "required": [ "modelId", "totalCost", "generationCount" ], "additionalProperties": false }, "description": "Per-model breakdown, highest `totalCost` first" } }, "required": [ "totalCost", "generationCount", "byModel" ], "additionalProperties": false, "description": "Media and image generation share of the period's spend" }, "skills": { "type": "array", "items": { "type": "object", "properties": { "skillId": { "type": "string", "description": "Skill ID as recorded when the call was traced" }, "skillName": { "type": [ "string", "null" ], "description": "Skill display name; null when the skill no longer exists" }, "uses": { "type": "integer", "minimum": 0, "description": "Model steps in the period whose output invoked the skill" }, "sessions": { "type": "integer", "minimum": 0, "description": "Distinct sessions in which the skill was invoked" }, "totalCost": { "type": "number", "minimum": 0, "description": "Estimated provider cost in USD summed over the steps that invoked the skill, not billed credits" }, "avgCostPerUse": { "type": "number", "minimum": 0, "description": "`totalCost` divided by `uses`" } }, "required": [ "skillId", "skillName", "uses", "sessions", "totalCost", "avgCostPerUse" ], "additionalProperties": false }, "description": "Per-skill cost for the current period, highest `totalCost` first. Estimated provider cost only. Empty when the period's costs come from billing records, which do not track skills." } }, "required": [ "agents", "orgTotalCost", "orgTotalSessions", "orgAvgCostPerSession", "orgTotalDurationMs", "dailyCost", "mediaGeneration", "skills" ], "additionalProperties": false }, "example": { "agents": [ { "agentId": "c62cefd9-9018-4d1b-8a0f-9381d6549ff6", "agentName": "text", "workspaceName": "text", "totalSessions": 1, "totalCost": 0.75, "avgCostPerSession": 0.75, "avgDurationMs": 0.75, "totalTokens": 1, "trend": 0.75 } ], "orgTotalCost": 0.75, "orgTotalSessions": 1, "orgAvgCostPerSession": 0.75, "orgTotalDurationMs": 0.75, "dailyCost": [ { "date": "text", "cost": 0.75, "tokens": 1 } ], "mediaGeneration": { "totalCost": 0.75, "generationCount": 1, "byModel": [ { "modelId": "9561deaa-738e-4e8d-80da-90cd6e737a45", "totalCost": 0.75, "generationCount": 1 } ] }, "skills": [ { "skillId": "3bb7a9ef-8bba-4c48-8bc8-0cccec5c0a00", "skillName": "text", "uses": 1, "sessions": 1, "totalCost": 0.75, "avgCostPerUse": 0.75 } ] } } } }, "400": { "description": "A parameter is missing or invalid", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "401": { "description": "The key is missing, unknown, revoked or expired, or the secret is wrong: `{\"error\":\"Unauthorized\"}`. A request signed in some other way (a browser session, a `ctx_` key) gets `{\"error\":\"Task API key required (Authorization: Bearer tsk_...)\"}`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "403": { "description": "Refused: the key lacks `evals:read` (code `TASK_CAPABILITY_MISSING`); or the key's owner can't read this in the app; or, on this endpoint, the key is limited to specific agents. Only the first carries a `code`.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } }, "429": { "description": "The key went over its request rate", "headers": { "Retry-After": { "description": "Seconds to wait before sending again", "schema": { "type": "integer", "minimum": 1 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/RateLimited" } } } }, "500": { "description": "Server error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "x-access": "evals", "x-codeSamples": [ { "lang": "shell", "label": "cURL", "source": "#!/usr/bin/env bash\nset -euo pipefail\ncurl -sS --fail-with-body --get \"$CONTEXT_APP_ORIGIN/api/v1/evals/traces/cost-dashboard\" \\\n -H \"Authorization: Bearer $CONTEXT_API_KEY\" \\\n --data-urlencode 'since=2026-10-01T00:00:00Z'" }, { "lang": "python", "label": "Python", "source": "import json\nimport os\n\nimport requests\n\norigin = os.environ[\"CONTEXT_APP_ORIGIN\"]\napi_key = os.environ[\"CONTEXT_API_KEY\"]\n\nresp = requests.get(\n f\"{origin}/api/v1/evals/traces/cost-dashboard\",\n headers={\"Authorization\": f\"Bearer {api_key}\"},\n params={\"since\": \"2026-10-01T00:00:00Z\"},\n timeout=60,\n)\nresp.raise_for_status()\nprint(json.dumps(resp.json(), indent=2))" }, { "lang": "typescript", "label": "TypeScript", "source": "function env(name: string): string {\n const value = process.env[name]\n if (!value) throw new Error(`Set ${name} first`)\n return value\n}\n\nconst url = new URL(`/api/v1/evals/traces/cost-dashboard`, env(\"CONTEXT_APP_ORIGIN\"))\nurl.searchParams.set(\"since\", \"2026-10-01T00:00:00Z\")\nconst res = await fetch(url, { headers: { Authorization: `Bearer ${env(\"CONTEXT_API_KEY\")}` } })\nif (!res.ok) throw new Error(`${res.status} ${await res.text()}`)\nconsole.log(JSON.stringify(await res.json(), null, 2))" } ] } } } }