{"openapi":"3.1.0","info":{"title":"OpenAI evaluations and runs (your key) API","version":"0.1.0","description":"OpenAI evaluations and runs (your key) lets an AI assistant read your own OpenAI Evals data through the OpenAI Evals API with a key you supply (OpenAI API key). Read-only: 5 tools (list_evals, get_eval, list_eval_runs, get_eval_run, list_eval_run_output_items). The BYOK key travels with each request, is used for that one call only and is never stored, logged or cached; it is only ever sent to...\n\nEvery operation is read-only or compute-only. The same tools are available over MCP (streamable HTTP) at https://byok-openai-evals.connectorbay.com/mcp.","contact":{"name":"Unify Dynamics LLC","email":"agent@connectorbay.com","url":"https://connectorbay.com/byok-openai-evals/docs"},"termsOfService":"https://connectorbay.com/byok-openai-evals/terms"},"servers":[{"url":"https://byok-openai-evals.connectorbay.com"}],"externalDocs":{"url":"https://connectorbay.com/byok-openai-evals/docs"},"security":[{"upstreamKey":[]}],"components":{"securitySchemes":{"bearer":{"type":"http","scheme":"bearer","description":"Optional. A Pro API key raises rate limits; no credentials are required."},"upstreamKey":{"type":"apiKey","in":"header","name":"X-Upstream-Key","description":"Your own key for the service this connector wraps (bring your own key). Used for that one call only; never stored, logged or cached."}},"schemas":{"Error":{"type":"object","properties":{"ok":{"const":false},"error":{"type":"object"}}}}},"paths":{"/v1/tools/list_evals":{"post":{"operationId":"list_evals","summary":"List the evaluations of the key's project with name and creation time, ordered by creation or update time. Reads evaluat","description":"List the evaluations of the key's project with name and creation time, ordered by creation or update time. Reads evaluation metadata only: it starts no run and sends no prompt. Use for \"Which evaluations have I defined in my OpenAI project\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many items to return (1 to 100).","title":"Limit"},"after":{"anyOf":[{"maxLength":100,"type":"string"},{"type":"null"}],"default":null,"description":"Cursor for paging: the last_id of the previous page.","title":"After"},"order":{"anyOf":[{"enum":["asc","desc"],"type":"string"},{"type":"null"}],"default":"asc","description":"Sort order by creation time: asc (default) or desc.","title":"Order"},"order_by":{"anyOf":[{"enum":["created_at","updated_at"],"type":"string"},{"type":"null"}],"default":"created_at","description":"Sort field: created_at (default) or updated_at.","title":"Order By"}},"title":"list_evalsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_eval":{"post":{"operationId":"get_eval","summary":"Get one evaluation: name, creation time, metadata, and the data source configuration type. Reads evaluation metadata onl","description":"Get one evaluation: name, creation time, metadata, and the data source configuration type. Reads evaluation metadata only: it starts no run and sends no prompt. Use for \"Show the details of OpenAI evaluation eval_abc123\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"eval_id":{"description":"Evaluation id, for example 'eval_abc123'.","maxLength":100,"title":"Eval Id","type":"string"}},"required":["eval_id"],"title":"get_evalArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_eval_runs":{"post":{"operationId":"list_eval_runs","summary":"List the runs of one evaluation with status, model, report link and pass, fail and error counts. Reads evaluation metada","description":"List the runs of one evaluation with status, model, report link and pass, fail and error counts. Reads evaluation metadata only: it starts no run and sends no prompt. Use for \"How did the runs of OpenAI evaluation eval_abc123 turn out\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"eval_id":{"description":"Evaluation id, for example 'eval_abc123'.","maxLength":100,"title":"Eval Id","type":"string"},"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many items to return (1 to 100).","title":"Limit"},"after":{"anyOf":[{"maxLength":100,"type":"string"},{"type":"null"}],"default":null,"description":"Cursor for paging: the last_id of the previous page.","title":"After"},"order":{"anyOf":[{"enum":["asc","desc"],"type":"string"},{"type":"null"}],"default":"asc","description":"Sort order by creation time: asc (default) or desc.","title":"Order"},"status":{"anyOf":[{"enum":["queued","in_progress","completed","canceled","failed"],"type":"string"},{"type":"null"}],"default":null,"description":"Only runs with this status.","title":"Status"}},"required":["eval_id"],"title":"list_eval_runsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_eval_run":{"post":{"operationId":"get_eval_run","summary":"Get one evaluation run: status, model, creation time, report link, pass, fail and error counts, and any error. Reads eva","description":"Get one evaluation run: status, model, creation time, report link, pass, fail and error counts, and any error. Reads evaluation metadata only: it starts no run and sends no prompt. Use for \"What were the pass and fail counts of OpenAI eval run evalrun_abc123\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"eval_id":{"description":"Evaluation id, for example 'eval_abc123'.","maxLength":100,"title":"Eval Id","type":"string"},"run_id":{"description":"Evaluation run id, for example 'evalrun_abc123'.","maxLength":100,"title":"Run Id","type":"string"}},"required":["eval_id","run_id"],"title":"get_eval_runArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_eval_run_output_items":{"post":{"operationId":"list_eval_run_output_items","summary":"List the output items of an evaluation run with pass or fail status and ids (the item contents are not picked). Reads ev","description":"List the output items of an evaluation run with pass or fail status and ids (the item contents are not picked). Reads evaluation metadata only: it starts no run and sends no prompt. Use for \"Which items failed in OpenAI eval run evalrun_abc123\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"eval_id":{"description":"Evaluation id, for example 'eval_abc123'.","maxLength":100,"title":"Eval Id","type":"string"},"run_id":{"description":"Evaluation run id, for example 'evalrun_abc123'.","maxLength":100,"title":"Run Id","type":"string"},"limit":{"anyOf":[{"maximum":100,"minimum":1,"type":"integer"},{"type":"null"}],"default":20,"description":"How many items to return (1 to 100).","title":"Limit"},"after":{"anyOf":[{"maxLength":100,"type":"string"},{"type":"null"}],"default":null,"description":"Cursor for paging: the last_id of the previous page.","title":"After"},"order":{"anyOf":[{"enum":["asc","desc"],"type":"string"},{"type":"null"}],"default":"asc","description":"Sort order by creation time: asc (default) or desc.","title":"Order"},"status":{"anyOf":[{"enum":["fail","pass"],"type":"string"},{"type":"null"}],"default":null,"description":"Only items with this result.","title":"Status"}},"required":["eval_id","run_id"],"title":"list_eval_run_output_itemsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}}}}