{
"openapi": "3.0.0",
"info": {
"title": "Traceloop API",
"version": "1.0.0",
"contact": {}
},
"paths": {
"/v2/auto-monitor-setups": {
"get": {
"description": "List all auto monitor setups for the organization with optional filters",
"parameters": [
{
"description": "Filter by status (e.g., pending, completed)",
"in": "query",
"name": "status",
"schema": {
"type": "string"
}
},
{
"description": "Filter by external ID",
"in": "query",
"name": "external_id",
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"items": {
"$ref": "#/components/schemas/response.AutoMonitorSetupResponse"
},
"type": "array"
}
}
}
},
"500": {
"description": "Internal error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"summary": "List auto monitor setups",
"tags": [
"auto-monitor-setups"
]
},
"post": {
"description": "Create a new auto monitor setup for automatic monitor creation.\n\nThe `selector` field is a map of span attribute key-value pairs used to filter which spans\nthis monitor applies to. Keys are span attribute names and values can be strings, numbers, or booleans.\n\nCommon selector keys:\n- `gen_ai.system` \u2014 LLM provider (e.g. \"openai\", \"anthropic\")\n- `gen_ai.request.model` \u2014 model name (e.g. \"gpt-4o\", \"claude-3-5-sonnet\")\n- `gen_ai.request.max_tokens` \u2014 max tokens (number)\n- Any other span attribute from your traces",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.CreateAutoMonitorSetupInput"
}
}
},
"description": "Auto monitor setup configuration",
"required": true
},
"responses": {
"201": {
"description": "Created",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AutoMonitorSetupResponse"
}
}
}
},
"400": {
"description": "Invalid input",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"summary": "Create an auto monitor setup",
"tags": [
"auto-monitor-setups"
]
}
},
"/v2/auto-monitor-setups/{external_id}": {
"delete": {
"description": "Delete an auto monitor setup by its external ID",
"parameters": [
{
"description": "Auto monitor setup external ID",
"in": "path",
"name": "external_id",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"204": {
"description": "No content"
},
"404": {
"description": "Not found",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"summary": "Delete an auto monitor setup by external ID",
"tags": [
"auto-monitor-setups"
]
},
"get": {
"description": "Get a specific auto monitor setup by its external ID",
"parameters": [
{
"description": "Auto monitor setup external ID",
"in": "path",
"name": "external_id",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AutoMonitorSetupResponse"
}
}
}
},
"404": {
"description": "Not found",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"summary": "Get an auto monitor setup by external ID",
"tags": [
"auto-monitor-setups"
]
},
"put": {
"description": "Update an existing auto monitor setup by its external ID",
"parameters": [
{
"description": "Auto monitor setup external ID",
"in": "path",
"name": "external_id",
"required": true,
"schema": {
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.UpdateAutoMonitorSetupInput"
}
}
},
"description": "Fields to update",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AutoMonitorSetupResponse"
}
}
}
},
"400": {
"description": "Invalid input",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"404": {
"description": "Not found",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"summary": "Update an auto monitor setup by external ID",
"tags": [
"auto-monitor-setups"
]
}
},
"/v2/evaluators": {
"post": {
"description": "Create a new custom evaluator at the organization level without project or environment bindings. The evaluator uses LLM-as-a-judge to evaluate inputs based on the provided configuration. The response format is automatically generated from the output schema.",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.CreateCustomEvaluatorInput"
}
}
},
"description": "Custom evaluator creation input",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.CreateEvaluatorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"summary": "Create a custom LLM-as-a-judge evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/agent-efficiency/execute": {
"post": {
"description": "Evaluate agent efficiency - detect redundant calls, unnecessary follow-ups\n\n**Request Body:**\n- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory\n- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.AgentEfficiencyRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AgentEfficiencyResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute agent-efficiency evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/agent-flow-quality/execute": {
"post": {
"description": "Validate agent trajectory against user-defined conditions\n\n**Request Body:**\n- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory\n- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory\n- `config.conditions` (array of strings, required): Array of evaluation conditions/rules to validate against\n- `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.AgentFlowQualityRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AgentFlowQualityResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute agent-flow-quality evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/agent-goal-accuracy/execute": {
"post": {
"description": "Evaluate agent goal accuracy\n\n**Request Body:**\n- `input.question` (string, required): The original question or goal\n- `input.completion` (string, required): The agent's completion/response\n- `input.reference` (string, required): The expected reference answer",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.AgentGoalAccuracyRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AgentGoalAccuracyResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute agent-goal-accuracy evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/agent-goal-completeness/execute": {
"post": {
"description": "Measure if agent accomplished all user goals\n\n**Request Body:**\n- `input.trajectory_prompts` (string, required): JSON array of prompts in the agent trajectory\n- `input.trajectory_completions` (string, required): JSON array of completions in the agent trajectory\n- `config.threshold` (number, required): Score threshold for pass/fail determination (0.0-1.0)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.AgentGoalCompletenessRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AgentGoalCompletenessResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute agent-goal-completeness evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/agent-tool-error-detector/execute": {
"post": {
"description": "Detect errors or failures during tool execution\n\n**Request Body:**\n- `input.tool_input` (string, required): JSON string of the tool input\n- `input.tool_output` (string, required): JSON string of the tool output",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.AgentToolErrorDetectorRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AgentToolErrorDetectorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute agent-tool-error-detector evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/agent-tool-selection-quality/execute": {
"post": {
"description": "Evaluate whether the agent selected the most appropriate tool(s) for the task at hand\n\n**Request Body:**\n- `input.input_messages` (array of objects, required): The agent's input messages — system prompt + conversation history as a JSON array of message objects (`role`, `content`, optional `tool_calls`, etc.)\n- `input.output_messages` (array of objects, required): The agent's output messages for this step as a JSON array of message objects (may contain tool calls or plain text)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.AgentToolSelectionQualityRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AgentToolSelectionQualityResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute agent-tool-selection-quality evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/agent-tool-trajectory/execute": {
"post": {
"description": "Compare actual tool calls against expected reference tool calls\n\n**Request Body:**\n- `input.executed_tool_calls` (string, required): JSON array of actual tool calls made by the agent\n- `input.expected_tool_calls` (string, required): JSON array of expected/reference tool calls\n- `config.threshold` (float, optional): Score threshold for pass/fail determination (default: 0.5)\n- `config.mismatch_sensitive` (bool, optional): Whether tool calls must match exactly (default: false)\n- `config.order_sensitive` (bool, optional): Whether order of tool calls matters (default: false)\n- `config.input_params_sensitive` (bool, optional): Whether to compare input parameters (default: true)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.AgentToolTrajectoryRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AgentToolTrajectoryResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute agent-tool-trajectory evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/action-advancement/execute": {
"post": {
"description": "Measure whether the assistant made progress toward at least one user goal in a conversation\n\n**Request Body:**\n- `input.messages` (string, required): JSON array of the conversation messages assembled from the trace",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.ActionAdvancementRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ActionAdvancementResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute action-advancement evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/answer-completeness/execute": {
"post": {
"description": "Evaluate whether the answer is complete and contains all the necessary information\n\n**Request Body:**\n- `input.question` (string, required): The original question\n- `input.completion` (string, required): The completion to evaluate for completeness\n- `input.context` (string, required): The context that provides the complete information",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.AnswerCompletenessRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AnswerCompletenessResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute answer-completeness evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/answer-correctness/execute": {
"post": {
"description": "Evaluate factual accuracy by comparing answers against ground truth\n\n**Request Body:**\n- `input.question` (string, required): The original question\n- `input.completion` (string, required): The completion to evaluate\n- `input.ground_truth` (string, required): The expected correct answer",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.AnswerCorrectnessRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AnswerCorrectnessResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute answer-correctness evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/answer-relevancy/execute": {
"post": {
"description": "Check if an answer is relevant to a question\n\n**Request Body:**\n- `input.answer` (string, required): The answer to evaluate for relevancy\n- `input.question` (string, required): The question that the answer should be relevant to",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.AnswerRelevancyRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.AnswerRelevancyResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute answer-relevancy evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/char-count/execute": {
"post": {
"description": "Count the number of characters in text\n\n**Request Body:**\n- `input.text` (string, required): The text to count characters in",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.CharCountRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.CharCountResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute char-count evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/char-count-ratio/execute": {
"post": {
"description": "Calculate the ratio of characters between two texts\n\n**Request Body:**\n- `input.numerator_text` (string, required): The numerator text (will be divided by denominator)\n- `input.denominator_text` (string, required): The denominator text (divides the numerator)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.CharCountRatioRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.CharCountRatioResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute char-count-ratio evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/context-relevance/execute": {
"post": {
"description": "Evaluate whether retrieved context contains sufficient information to answer the query\n\n**Request Body:**\n- `input.query` (string, required): The query/question to evaluate context relevance for\n- `input.completion` (string, required): The completion to evaluate for context relevance\n- `input.context` (string, required): The context to evaluate for relevance to the query\n- `config.model` (string, optional): Model to use for evaluation (default: gpt-4o)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.ContextRelevanceRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ContextRelevanceResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute context-relevance evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/conversation-quality/execute": {
"post": {
"description": "Evaluate conversation quality based on tone, clarity, flow, responsiveness, and transparency\n\n**Request Body:**\n- `input.prompts` (string, required): JSON array of prompts in the conversation\n- `input.completions` (string, required): JSON array of completions in the conversation",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.ConversationQualityRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ConversationQualityResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute conversation-quality evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/faithfulness/execute": {
"post": {
"description": "Check if a completion is faithful to the provided context\n\n**Request Body:**\n- `input.completion` (string, required): The LLM completion to check for faithfulness\n- `input.context` (string, required): The context that the completion should be faithful to\n- `input.question` (string, required): The original question asked",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.FaithfulnessRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.FaithfulnessResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute faithfulness evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/html-comparison/execute": {
"post": {
"description": "Compare two HTML documents for structural and content similarity\n\n**Request Body:**\n- `input.html1` (string, required): The first HTML document to compare\n- `input.html2` (string, required): The second HTML document to compare",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.HtmlComparisonRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.HtmlComparisonResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute html-comparison evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/instruction-adherence/execute": {
"post": {
"description": "Evaluate how well responses follow given instructions\n\n**Request Body:**\n- `input.instructions` (string, required): The instructions that should be followed\n- `input.response` (string, required): The response to evaluate for instruction adherence",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.InstructionAdherenceRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.InstructionAdherenceResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute instruction-adherence evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/intent-change/execute": {
"post": {
"description": "Detect changes in user intent between prompts and completions\n\n**Request Body:**\n- `input.prompts` (string, required): JSON array of prompts in the conversation\n- `input.completions` (string, required): JSON array of completions in the conversation",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.IntentChangeRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.IntentChangeResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute intent-change evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/json-validator/execute": {
"post": {
"description": "Validate JSON syntax\n\n**Request Body:**\n- `input.text` (string, required): The text to validate as JSON\n- `config.enable_schema_validation` (bool, optional): Enable JSON schema validation\n- `config.schema_string` (string, optional): JSON schema to validate against",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.JSONValidatorRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.JSONValidatorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute json-validator evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/perplexity/execute": {
"post": {
"description": "Measure text perplexity from logprobs\n\n**Request Body:**\n- `input.logprobs` (string, required): JSON array of log probabilities from the model",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.PerplexityRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.PerplexityResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute perplexity evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/pii-detector/execute": {
"post": {
"description": "Detect personally identifiable information in text\n\n**Request Body:**\n- `input.text` (string, required): The text to scan for personally identifiable information\n- `config.probability_threshold` (float, optional): Detection threshold (default: 0.8)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.PIIDetectorRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.PIIDetectorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute pii-detector evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/placeholder-regex/execute": {
"post": {
"description": "Validate text against a placeholder regex pattern\n\n**Request Body:**\n- `input.placeholder_name` (string, required): The name of the placeholder to substitute\n- `input.placeholder_value` (string, required): The value to substitute into the regex placeholder\n- `input.text` (string, required): The text to validate against the regex pattern\n- `config.should_match` (bool, optional): Whether the text should match the regex\n- `config.case_sensitive` (bool, optional): Case-sensitive matching\n- `config.dot_include_nl` (bool, optional): Dot matches newlines\n- `config.multi_line` (bool, optional): Multi-line mode",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.PlaceholderRegexRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.PlaceholderRegexResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute placeholder-regex evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/profanity-detector/execute": {
"post": {
"description": "Detect profanity in text\n\n**Request Body:**\n- `input.text` (string, required): The text to scan for profanity",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.ProfanityDetectorRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ProfanityDetectorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute profanity-detector evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/prompt-injection/execute": {
"post": {
"description": "Detect prompt injection attempts\n\n**Request Body:**\n- `input.prompt` (string, required): The prompt to check for injection attempts\n- `config.threshold` (float, optional): Detection threshold (default: 0.5)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.PromptInjectionRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.PromptInjectionResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute prompt-injection evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/prompt-perplexity/execute": {
"post": {
"description": "Measure prompt perplexity to detect potential injection attempts\n\n**Request Body:**\n- `input.prompt` (string, required): The prompt to calculate perplexity for",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.PromptPerplexityRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.PromptPerplexityResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute prompt-perplexity evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/regex-validator/execute": {
"post": {
"description": "Validate text against a regex pattern\n\n**Request Body:**\n- `input.text` (string, required): The text to validate against a regex pattern\n- `config.regex` (string, optional): The regex pattern to match against\n- `config.should_match` (bool, optional): Whether the text should match the regex\n- `config.case_sensitive` (bool, optional): Case-sensitive matching\n- `config.dot_include_nl` (bool, optional): Dot matches newlines\n- `config.multi_line` (bool, optional): Multi-line mode",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.RegexValidatorRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.RegexValidatorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute regex-validator evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/secrets-detector/execute": {
"post": {
"description": "Detect secrets and credentials in text\n\n**Request Body:**\n- `input.text` (string, required): The text to scan for secrets (API keys, passwords, etc.)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.SecretsDetectorRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.SecretsDetectorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute secrets-detector evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/semantic-similarity/execute": {
"post": {
"description": "Calculate semantic similarity between completion and reference\n\n**Request Body:**\n- `input.completion` (string, required): The completion text to compare\n- `input.reference` (string, required): The reference text to compare against",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.SemanticSimilarityRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.SemanticSimilarityResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute semantic-similarity evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/sexism-detector/execute": {
"post": {
"description": "Detect sexist language and bias\n\n**Request Body:**\n- `input.text` (string, required): The text to scan for sexist content\n- `config.threshold` (float, optional): Detection threshold (default: 0.5)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.SexismDetectorRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.SexismDetectorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute sexism-detector evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/sql-validator/execute": {
"post": {
"description": "Validate SQL query syntax\n\n**Request Body:**\n- `input.text` (string, required): The text to validate as SQL",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.SQLValidatorRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.SQLValidatorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute sql-validator evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/tone-detection/execute": {
"post": {
"description": "Detect the tone of the text\n\n**Request Body:**\n- `input.text` (string, required): The text to detect the tone of",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.ToneDetectionRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ToneDetectionResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute tone-detection evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/topic-adherence/execute": {
"post": {
"description": "Evaluate topic adherence\n\n**Request Body:**\n- `input.question` (string, required): The original question\n- `input.completion` (string, required): The completion to evaluate\n- `input.reference_topics` (string, required): Comma-separated list of expected topics",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.TopicAdherenceRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.TopicAdherenceResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute topic-adherence evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/toxicity-detector/execute": {
"post": {
"description": "Detect toxic or harmful language\n\n**Request Body:**\n- `input.text` (string, required): The text to scan for toxic content\n- `config.threshold` (float, optional): Detection threshold (default: 0.5)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.ToxicityDetectorRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ToxicityDetectorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute toxicity-detector evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/uncertainty-detector/execute": {
"post": {
"description": "Detect uncertainty in the text\n\n**Request Body:**\n- `input.prompt` (string, required): The text to detect uncertainty in",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.UncertaintyDetectorRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.UncertaintyDetectorResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute uncertainty-detector evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/word-count/execute": {
"post": {
"description": "Count the number of words in text\n\n**Request Body:**\n- `input.text` (string, required): The text to count words in",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.WordCountRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.WordCountResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute word-count evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/evaluators/word-count-ratio/execute": {
"post": {
"description": "Calculate the ratio of words between two texts\n\n**Request Body:**\n- `input.numerator_text` (string, required): The numerator text (will be divided by denominator)\n- `input.denominator_text` (string, required): The denominator text (divides the numerator)",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.WordCountRatioRequest"
}
}
},
"description": "Request body",
"required": true
},
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.WordCountRatioResponse"
}
}
}
},
"400": {
"description": "Bad Request",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal Server Error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"security": [
{
"BearerAuth": []
}
],
"summary": "Execute word-count-ratio evaluator",
"tags": [
"evaluators"
]
}
},
"/v2/metrics": {
"post": {
"description": "Retrieves metrics data with support for filtering, sorting, and pagination. Metrics are grouped by metric name with individual data points. Supports filtering by direct column fields (bool_value, trace_id, etc.), label fields (labels.agent_name, labels.trace_id), and attribute fields (attributes.*).",
"operationId": "get-metrics",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.GetMetricsRequest"
}
}
},
"description": "Metrics query parameters including filters, environments, and pagination",
"required": true
},
"responses": {
"200": {
"description": "Grouped metrics with data points",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.GetMetricsResponse"
}
}
}
},
"400": {
"description": "Invalid request parameters",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"404": {
"description": "Not Found",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal server error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"summary": "Get metrics with filtering and grouping",
"tags": [
"metrics"
]
}
},
"/v2/metrics_hwm": {
"get": {
"description": "Returns the timestamp of the last successfully processed evaluation (high water mark)",
"operationId": "get-metrics-hwm",
"responses": {
"200": {
"description": "High water mark timestamp in milliseconds",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.MetricsHWMResponse"
}
}
}
},
"500": {
"description": "Internal server error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"summary": "Get metrics high water mark",
"tags": [
"metrics"
]
}
},
"/v2/organizations": {
"post": {
"description": "Create a new organization with environments and API keys.",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/request.CreateOrganizationRequest"
}
}
},
"description": "Organization creation request",
"required": true
},
"responses": {
"201": {
"description": "Created",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.CreateOrganizationResponse"
}
}
}
},
"400": {
"description": "Invalid request body or validation error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"403": {
"description": "Not allowed to create organizations",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
},
"500": {
"description": "Internal server error",
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/response.ErrorResponse"
}
}
}
}
},
"summary": "Create a new organization",
"tags": [
"organizations"
]
}
}
},
"servers": [
{
"url": "https://api.traceloop.com"
}
],
"components": {
"securitySchemes": {
"BearerAuth": {
"description": "Type \"Bearer\" followed by a space and JWT token.",
"in": "header",
"name": "Authorization",
"type": "apiKey"
}
},
"schemas": {
"evaluator.Property": {
"properties": {
"description": {
"type": "string"
},
"enum_values": {
"description": "The allowed values for this property. Only applicable when `type` is `enum`.",
"items": {
"type": "string"
},
"type": "array"
},
"label": {
"type": "string"
},
"name": {
"type": "string"
},
"type": {
"$ref": "#/components/schemas/evaluator.PropertyType"
}
},
"required": [
"name",
"type"
],
"type": "object"
},
"evaluator.PropertyType": {
"enum": [
"string",
"int",
"float",
"boolean",
"enum"
],
"type": "string",
"x-enum-varnames": [
"StringType",
"IntType",
"FloatType",
"BooleanType",
"EnumType"
]
},
"model.LLMMessage": {
"properties": {
"content": {
"type": "string"
},
"role": {
"type": "string"
},
"tool_call_id": {
"type": "string"
}
},
"type": "object"
},
"request.ActionAdvancementInput": {
"properties": {
"messages": {
"example": "[{\"role\":\"user\",\"content\":\"Book me a flight to Paris\"},{\"role\":\"assistant\",\"content\":\"I found 3 flights to Paris. Booking the cheapest one now.\"}]",
"type": "string"
}
},
"required": [
"messages"
],
"type": "object"
},
"request.ActionAdvancementRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.ActionAdvancementInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.AgentEfficiencyInput": {
"properties": {
"trajectory_completions": {
"example": "[\"User found\", \"Email updated\", \"Changes saved\"]",
"type": "string"
},
"trajectory_prompts": {
"example": "[\"Find user info\", \"Update email\", \"Save changes\"]",
"type": "string"
}
},
"required": [
"trajectory_completions",
"trajectory_prompts"
],
"type": "object"
},
"request.AgentEfficiencyRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AgentEfficiencyInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.AgentFlowQualityConfigRequest": {
"properties": {
"conditions": {
"example": [
"no tools called",
"agent completed task"
],
"items": {
"type": "string"
},
"type": "array"
},
"threshold": {
"example": 0.5,
"type": "number"
}
},
"required": [
"conditions",
"threshold"
],
"type": "object"
},
"request.AgentFlowQualityInput": {
"properties": {
"trajectory_completions": {
"example": "[\"Found 5 flights\", \"Selected $299 flight\", \"Booking confirmed\"]",
"type": "string"
},
"trajectory_prompts": {
"example": "[\"Search for flights\", \"Select the cheapest option\", \"Confirm booking\"]",
"type": "string"
}
},
"required": [
"trajectory_completions",
"trajectory_prompts"
],
"type": "object"
},
"request.AgentFlowQualityRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.AgentFlowQualityConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.AgentFlowQualityInput"
}
},
"required": [
"config",
"input"
],
"type": "object"
},
"request.AgentGoalAccuracyInput": {
"properties": {
"completion": {
"example": "I have booked your flight from New York to Los Angeles departing Monday at 9am.",
"type": "string"
},
"question": {
"example": "Book a flight from NYC to LA for next Monday",
"type": "string"
},
"reference": {
"example": "Flight booked: NYC to LA, Monday departure",
"type": "string"
}
},
"required": [
"completion",
"question",
"reference"
],
"type": "object"
},
"request.AgentGoalAccuracyRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AgentGoalAccuracyInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.AgentGoalCompletenessConfigRequest": {
"properties": {
"threshold": {
"example": 0.5,
"type": "number"
}
},
"required": [
"threshold"
],
"type": "object"
},
"request.AgentGoalCompletenessInput": {
"properties": {
"trajectory_completions": {
"example": "[\"Account created\", \"Preferences saved\", \"Notifications enabled\"]",
"type": "string"
},
"trajectory_prompts": {
"example": "[\"Create new account\", \"Set preferences\", \"Enable notifications\"]",
"type": "string"
}
},
"required": [
"trajectory_completions",
"trajectory_prompts"
],
"type": "object"
},
"request.AgentGoalCompletenessRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.AgentGoalCompletenessConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.AgentGoalCompletenessInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.AgentToolErrorDetectorInput": {
"properties": {
"tool_input": {
"example": "{\"action\": \"search\", \"query\": \"flights to Paris\"}",
"type": "string"
},
"tool_output": {
"example": "{\"status\": \"success\", \"results\": [{\"flight\": \"AF123\", \"price\": 450}]}",
"type": "string"
}
},
"required": [
"tool_input",
"tool_output"
],
"type": "object"
},
"request.AgentToolErrorDetectorRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AgentToolErrorDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.AgentToolTrajectoryConfigRequest": {
"properties": {
"input_params_sensitive": {
"example": true,
"type": "boolean"
},
"mismatch_sensitive": {
"example": false,
"type": "boolean"
},
"order_sensitive": {
"example": false,
"type": "boolean"
},
"threshold": {
"example": 0.5,
"type": "number"
}
},
"type": "object"
},
"request.AgentToolTrajectoryInput": {
"properties": {
"executed_tool_calls": {
"example": "[{\"name\": \"search\", \"input\": {\"query\": \"weather\"}}]",
"type": "string"
},
"expected_tool_calls": {
"example": "[{\"name\": \"search\", \"input\": {\"query\": \"weather\"}}]",
"type": "string"
}
},
"required": [
"executed_tool_calls",
"expected_tool_calls"
],
"type": "object"
},
"request.AgentToolSelectionQualityInput": {
"properties": {
"input_messages": {
"example": "[{\"role\":\"system\",\"content\":\"You are a helpful assistant with access to tools: 1. get_weather(city) - Get current weather. 2. search_web(query) - Search the web.\"},{\"role\":\"user\",\"content\":\"What is the weather like in Paris?\"}]",
"type": "string"
},
"output_messages": {
"example": "[{\"role\":\"assistant\",\"tool_calls\":[{\"name\":\"get_weather\",\"arguments\":{\"city\":\"Paris\"}}]}]",
"type": "string"
}
},
"required": [
"input_messages",
"output_messages"
],
"type": "object"
},
"request.AgentToolSelectionQualityRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AgentToolSelectionQualityInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.AgentToolTrajectoryRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.AgentToolTrajectoryConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.AgentToolTrajectoryInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.AnswerCompletenessInput": {
"properties": {
"completion": {
"example": "Paris.",
"type": "string"
},
"context": {
"example": "The capital of France is Paris.",
"type": "string"
},
"question": {
"example": "What is the capital of France?",
"type": "string"
}
},
"required": [
"completion",
"context",
"question"
],
"type": "object"
},
"request.AnswerCompletenessRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AnswerCompletenessInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.AnswerCorrectnessInput": {
"properties": {
"completion": {
"example": "World War II ended in 1945.",
"type": "string"
},
"ground_truth": {
"example": "1945",
"type": "string"
},
"question": {
"example": "What year did World War II end?",
"type": "string"
}
},
"required": [
"completion",
"ground_truth",
"question"
],
"type": "object"
},
"request.AnswerCorrectnessRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AnswerCorrectnessInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.AnswerRelevancyInput": {
"properties": {
"answer": {
"example": "The capital of France is Paris.",
"type": "string"
},
"question": {
"example": "What is the capital of France?",
"type": "string"
}
},
"required": [
"answer",
"question"
],
"type": "object"
},
"request.AnswerRelevancyRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.AnswerRelevancyInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.CharCountInput": {
"properties": {
"text": {
"example": "Hello, world! This is a sample text.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.CharCountRatioInput": {
"properties": {
"denominator_text": {
"example": "This is a longer text for comparison",
"type": "string"
},
"numerator_text": {
"example": "Short text",
"type": "string"
}
},
"required": [
"denominator_text",
"numerator_text"
],
"type": "object"
},
"request.CharCountRatioRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.CharCountRatioInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.CharCountRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.CharCountInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.ContextRelevanceConfigRequest": {
"properties": {
"model": {
"example": "gpt-4o",
"type": "string"
}
},
"type": "object"
},
"request.ContextRelevanceInput": {
"properties": {
"completion": {
"example": "Our store is open Monday to Friday from 9am to 6pm, and Saturday from 10am to 4pm. We are closed on Sundays.",
"type": "string"
},
"context": {
"example": "Our store is open Monday to Friday from 9am to 6pm, and Saturday from 10am to 4pm. We are closed on Sundays.",
"type": "string"
},
"query": {
"example": "What are the business hours?",
"type": "string"
}
},
"required": [
"completion",
"context",
"query"
],
"type": "object"
},
"request.ContextRelevanceRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.ContextRelevanceConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.ContextRelevanceInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.ConversationQualityInput": {
"properties": {
"completions": {
"example": "[\"Hi! I'd be happy to assist you today.\", \"We offer consulting, development, and support services.\"]",
"type": "string"
},
"prompts": {
"example": "[\"Hello, how can I help?\", \"What services do you offer?\"]",
"type": "string"
}
},
"required": [
"completions",
"prompts"
],
"type": "object"
},
"request.ConversationQualityRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.ConversationQualityInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.CreateAutoMonitorSetupInput": {
"properties": {
"evaluators": {
"description": "List of evaluator slugs to run on matched spans",
"example": [
"hallucination",
"toxicity"
],
"items": {
"type": "string"
},
"minItems": 1,
"type": "array"
},
"external_id": {
"description": "Unique identifier for the auto monitor setup, used to reference it in future requests",
"example": "my-agent-monitor-1",
"type": "string"
},
"selector": {
"description": "Map of span attributes to filter which spans this monitor applies to.\nKeys are span attribute names (e.g. gen_ai.system, gen_ai.request.model) and\nvalues can be strings, numbers, or booleans.\nExample: {\"gen_ai.system\": \"openai\", \"gen_ai.request.model\": \"gpt-4o\", \"gen_ai.request.max_tokens\": 1000}",
"type": "object"
}
},
"required": [
"evaluators",
"external_id"
],
"type": "object"
},
"request.CreateCustomEvaluatorInput": {
"properties": {
"description": {
"type": "string"
},
"frequency_penalty": {
"type": "number"
},
"input_schema": {
"description": "Schemas",
"items": {
"$ref": "#/components/schemas/evaluator.Property"
},
"type": "array"
},
"logprobs": {
"type": "boolean"
},
"max_tokens": {
"type": "integer"
},
"messages": {
"description": "LLM Message configuration",
"items": {
"$ref": "#/components/schemas/model.LLMMessage"
},
"type": "array"
},
"mode": {
"type": "string"
},
"model": {
"description": "LLM parameters (from BaseLLMConfig)",
"type": "string"
},
"name": {
"description": "Basic fields",
"type": "string"
},
"output_schema": {
"items": {
"$ref": "#/components/schemas/evaluator.Property"
},
"type": "array"
},
"presence_penalty": {
"type": "number"
},
"provider": {
"description": "Provider configuration",
"type": "string"
},
"slug": {
"type": "string"
},
"stop": {
"items": {
"type": "string"
},
"type": "array"
},
"temperature": {
"type": "number"
},
"top_p": {
"type": "number"
}
},
"required": [
"input_schema",
"messages",
"model",
"name",
"output_schema",
"provider"
],
"type": "object"
},
"request.CreateOrganizationRequest": {
"properties": {
"envs": {
"items": {
"type": "string"
},
"type": "array"
},
"org_name": {
"type": "string"
},
"retention_hours": {
"type": "integer",
"description": "Data retention period in hours (1-720). Defaults to 720 (1 month).",
"minimum": 1,
"maximum": 720,
"default": 720
}
},
"required": [
"org_name"
],
"type": "object"
},
"request.FaithfulnessInput": {
"properties": {
"completion": {
"example": "The Eiffel Tower is located in Paris and was built in 1889.",
"type": "string"
},
"context": {
"example": "The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris, France. It was constructed from 1887 to 1889.",
"type": "string"
},
"question": {
"example": "When was the Eiffel Tower built?",
"type": "string"
}
},
"required": [
"completion",
"context",
"question"
],
"type": "object"
},
"request.FaithfulnessRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.FaithfulnessInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.GetMetricsRequest": {
"properties": {
"from_timestamp_sec": {
"type": "integer",
"description": "Start time in Unix seconds timestamp.",
"example": 1772905218
},
"to_timestamp_sec": {
"type": "integer",
"description": "End time in Unix seconds timestamp.",
"example": 1772905218
},
"cursor": {
"type": "integer",
"description": "Pagination cursor for fetching the next page of results. The cursor is returned in the response to the previous request.",
"example": 1772905218000
},
"environments": {
"items": {
"type": "string"
},
"type": "array",
"description": "List of environments to filter by",
"example": [
"prd",
"stg",
"dev"
]
},
"filters": {
"items": {
"$ref": "#/components/schemas/shared.FilterCondition"
},
"type": "array"
},
"limit": {
"type": "integer",
"description": "Maximum number of metrics to return per page.",
"example": 50
},
"logical_operator": {
"$ref": "#/components/schemas/types.LogicalOperator"
},
"metric_name": {
"type": "string",
"description": "Metric name to filter by. The metric_name is a metric attribute that is set on the metric object.",
"example": "answer relevancy"
},
"metric_source": {
"type": "string",
"description": "Metric source to filter by. The metric_source is a metric attribute that is set on the metric object.",
"example": "llm_as_a_judge"
},
"sort_by": {
"type": "string",
"description": "Field to sort by.",
"example": "timestamp"
},
"sort_order": {
"type": "string",
"description": "Sort order.",
"example": "ASC"
}
},
"type": "object"
},
"request.HtmlComparisonInput": {
"properties": {
"html1": {
"example": "Hello, world!
",
"type": "string"
},
"html2": {
"example": "Hello, world!
",
"type": "string"
}
},
"required": [
"html1",
"html2"
],
"type": "object"
},
"request.HtmlComparisonRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.HtmlComparisonInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.InstructionAdherenceInput": {
"properties": {
"instructions": {
"example": "Respond in exactly 3 bullet points and use formal language.",
"type": "string"
},
"response": {
"example": "- First point about the topic\n- Second relevant consideration\n- Final concluding thought",
"type": "string"
}
},
"required": [
"instructions",
"response"
],
"type": "object"
},
"request.InstructionAdherenceRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.InstructionAdherenceInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.IntentChangeInput": {
"properties": {
"completions": {
"example": "[\"Sure, I can help with hotel booking\", \"No problem, let me search for flights\"]",
"type": "string"
},
"prompts": {
"example": "[\"I want to book a hotel\", \"Actually, I need a flight instead\"]",
"type": "string"
}
},
"required": [
"completions",
"prompts"
],
"type": "object"
},
"request.IntentChangeRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.IntentChangeInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.JSONValidatorConfigRequest": {
"properties": {
"enable_schema_validation": {
"example": true,
"type": "boolean"
},
"schema_string": {
"example": "{}",
"type": "string"
}
},
"type": "object"
},
"request.JSONValidatorInput": {
"properties": {
"text": {
"example": "{\"name\": \"John\", \"age\": 30}",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.JSONValidatorRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.JSONValidatorConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.JSONValidatorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.PIIDetectorConfigRequest": {
"properties": {
"probability_threshold": {
"example": 0.8,
"type": "number"
}
},
"type": "object"
},
"request.PIIDetectorInput": {
"properties": {
"text": {
"example": "Please contact John Smith at john.smith@email.com or call 555-123-4567.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.PIIDetectorRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.PIIDetectorConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.PIIDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.PerplexityInput": {
"properties": {
"logprobs": {
"example": "[-2.3, -1.5, -0.8, -1.2, -0.5]",
"type": "string"
}
},
"required": [
"logprobs"
],
"type": "object"
},
"request.PerplexityRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.PerplexityInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.PlaceholderRegexConfigRequest": {
"properties": {
"case_sensitive": {
"example": true,
"type": "boolean"
},
"dot_include_nl": {
"example": true,
"type": "boolean"
},
"multi_line": {
"example": true,
"type": "boolean"
},
"should_match": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"request.PlaceholderRegexInput": {
"properties": {
"placeholder_name": {
"example": "name",
"type": "string"
},
"placeholder_value": {
"example": "Hello ${name}",
"type": "string"
},
"text": {
"example": "Hello John Doe",
"type": "string"
}
},
"required": [
"placeholder_name",
"placeholder_value",
"text"
],
"type": "object"
},
"request.PlaceholderRegexRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.PlaceholderRegexConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.PlaceholderRegexInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.ProfanityDetectorInput": {
"properties": {
"text": {
"example": "This is a clean and professional message.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.ProfanityDetectorRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.ProfanityDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.PromptInjectionConfigRequest": {
"properties": {
"threshold": {
"example": 0.5,
"type": "number"
}
},
"type": "object"
},
"request.PromptInjectionInput": {
"properties": {
"prompt": {
"example": "What is the weather like today?",
"type": "string"
}
},
"required": [
"prompt"
],
"type": "object"
},
"request.PromptInjectionRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.PromptInjectionConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.PromptInjectionInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.PromptPerplexityInput": {
"properties": {
"prompt": {
"example": "What is the capital of France?",
"type": "string"
}
},
"required": [
"prompt"
],
"type": "object"
},
"request.PromptPerplexityRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.PromptPerplexityInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.RegexValidatorConfigRequest": {
"properties": {
"case_sensitive": {
"example": true,
"type": "boolean"
},
"dot_include_nl": {
"example": true,
"type": "boolean"
},
"multi_line": {
"example": true,
"type": "boolean"
},
"regex": {
"example": ".*",
"type": "string"
},
"should_match": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"request.RegexValidatorInput": {
"properties": {
"text": {
"example": "user@example.com",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.RegexValidatorRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.RegexValidatorConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.RegexValidatorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.SQLValidatorInput": {
"properties": {
"text": {
"example": "SELECT * FROM users WHERE id = 1;",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.SQLValidatorRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.SQLValidatorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.SecretsDetectorInput": {
"properties": {
"text": {
"example": "Here is some text without any API keys or passwords.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.SecretsDetectorRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.SecretsDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.SemanticSimilarityInput": {
"properties": {
"completion": {
"example": "The cat sat on the mat.",
"type": "string"
},
"reference": {
"example": "A feline was resting on the rug.",
"type": "string"
}
},
"required": [
"completion",
"reference"
],
"type": "object"
},
"request.SemanticSimilarityRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.SemanticSimilarityInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.SexismDetectorConfigRequest": {
"properties": {
"threshold": {
"example": 0.5,
"type": "number"
}
},
"type": "object"
},
"request.SexismDetectorInput": {
"properties": {
"text": {
"example": "All team members should be treated equally regardless of gender.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.SexismDetectorRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.SexismDetectorConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.SexismDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.ToneDetectionInput": {
"properties": {
"text": {
"example": "The capital of France is Paris.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.ToneDetectionRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.ToneDetectionInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.TopicAdherenceInput": {
"properties": {
"completion": {
"example": "Machine learning is a subset of AI that enables systems to learn from data.",
"type": "string"
},
"question": {
"example": "Tell me about machine learning",
"type": "string"
},
"reference_topics": {
"example": "artificial intelligence, data science, algorithms",
"type": "string"
}
},
"required": [
"completion",
"question",
"reference_topics"
],
"type": "object"
},
"request.TopicAdherenceRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.TopicAdherenceInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.ToxicityDetectorConfigRequest": {
"properties": {
"threshold": {
"example": 0.5,
"type": "number"
}
},
"type": "object"
},
"request.ToxicityDetectorInput": {
"properties": {
"text": {
"example": "Thank you for your help with this project.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.ToxicityDetectorRequest": {
"properties": {
"config": {
"$ref": "#/components/schemas/request.ToxicityDetectorConfigRequest"
},
"input": {
"$ref": "#/components/schemas/request.ToxicityDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.UncertaintyDetectorInput": {
"properties": {
"prompt": {
"example": "I am not sure, I think the capital of France is Paris.",
"type": "string"
}
},
"required": [
"prompt"
],
"type": "object"
},
"request.UncertaintyDetectorRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.UncertaintyDetectorInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.UpdateAutoMonitorSetupInput": {
"properties": {
"evaluators": {
"description": "List of evaluator slugs to run on matched spans",
"example": [
"hallucination",
"toxicity"
],
"items": {
"type": "string"
},
"type": "array"
},
"selector": {
"description": "Map of span attributes to filter which spans this monitor applies to.\nKeys are span attribute names (e.g. gen_ai.system, gen_ai.request.model) and\nvalues can be strings, numbers, or booleans.\nExample: {\"gen_ai.system\": \"openai\", \"gen_ai.request.model\": \"gpt-4o\", \"gen_ai.request.max_tokens\": 1000}",
"type": "object"
}
},
"type": "object"
},
"request.WordCountInput": {
"properties": {
"text": {
"example": "This is a sample text with several words.",
"type": "string"
}
},
"required": [
"text"
],
"type": "object"
},
"request.WordCountRatioInput": {
"properties": {
"denominator_text": {
"example": "This is a longer input text for comparison",
"type": "string"
},
"numerator_text": {
"example": "Short response",
"type": "string"
}
},
"required": [
"denominator_text",
"numerator_text"
],
"type": "object"
},
"request.WordCountRatioRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.WordCountRatioInput"
}
},
"required": [
"input"
],
"type": "object"
},
"request.WordCountRequest": {
"properties": {
"input": {
"$ref": "#/components/schemas/request.WordCountInput"
}
},
"required": [
"input"
],
"type": "object"
},
"response.ActionAdvancementResponse": {
"properties": {
"pass": {
"example": true,
"type": "boolean"
},
"reason": {
"example": "The assistant advanced the user's goal by booking the requested flight",
"type": "string"
}
},
"type": "object"
},
"response.AgentEfficiencyResponse": {
"properties": {
"step_efficiency_reason": {
"example": "Agent completed task with minimal redundant steps",
"type": "string"
},
"step_efficiency_score": {
"example": 0.85,
"type": "number"
},
"task_completion_reason": {
"example": "All required tasks were completed successfully",
"type": "string"
},
"task_completion_score": {
"example": 0.92,
"type": "number"
}
},
"type": "object"
},
"response.AgentFlowQualityResponse": {
"properties": {
"reason": {
"example": "Agent followed the expected flow correctly",
"type": "string"
},
"score": {
"example": 0.89,
"type": "number"
},
"success": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.AgentGoalAccuracyResponse": {
"properties": {
"accuracy_score": {
"example": 0.88,
"type": "number"
}
},
"type": "object"
},
"response.AgentGoalCompletenessResponse": {
"properties": {
"reason": {
"example": "All user goals were accomplished",
"type": "string"
},
"score": {
"example": 0.95,
"type": "number"
},
"success": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.AgentToolErrorDetectorResponse": {
"properties": {
"reason": {
"example": "Tool executed successfully without errors",
"type": "string"
},
"success": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.AgentToolSelectionQualityResponse": {
"properties": {
"correct_args": {
"example": true,
"type": "boolean"
},
"correct_args_reason": {
"example": "The city argument matches the user's request",
"type": "string"
},
"tool_selection": {
"example": "correct",
"type": "string"
},
"tool_selection_reason": {
"example": "The agent correctly chose get_weather for the weather query",
"type": "string"
}
},
"type": "object"
},
"response.AgentToolTrajectoryResponse": {
"properties": {
"reason": {
"example": "Tool calls match the expected trajectory",
"type": "string"
},
"score": {
"example": 0.85,
"type": "number"
},
"success": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.AnswerCompletenessResponse": {
"properties": {
"answer_completeness_score": {
"example": 0.95,
"type": "number"
}
},
"type": "object"
},
"response.AnswerCorrectnessResponse": {
"properties": {
"correctness_score": {
"example": 0.91,
"type": "number"
}
},
"type": "object"
},
"response.AnswerRelevancyResponse": {
"properties": {
"is_relevant": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.AutoMonitorEvaluatorResponse": {
"properties": {
"binding_id": {
"type": "string"
},
"error_message": {
"type": "string"
},
"evaluator_id": {
"type": "string"
},
"evaluator_type": {
"type": "string"
},
"input_schema": {
"items": {
"$ref": "#/components/schemas/evaluator.Property"
},
"type": "array"
},
"output_schema": {
"items": {
"$ref": "#/components/schemas/evaluator.Property"
},
"type": "array"
},
"processed_at": {
"type": "string"
},
"status": {
"type": "string"
}
},
"type": "object"
},
"response.AutoMonitorSetupResponse": {
"properties": {
"created_at": {
"type": "string"
},
"env_project_id": {
"type": "string"
},
"evaluators": {
"items": {
"$ref": "#/components/schemas/response.AutoMonitorEvaluatorResponse"
},
"type": "array"
},
"external_id": {
"type": "string"
},
"id": {
"type": "string"
},
"init_rules": {
"items": {
"$ref": "#/components/schemas/types.Rule"
},
"type": "array"
},
"org_id": {
"type": "string"
},
"project_id": {
"type": "string"
},
"status": {
"type": "string"
},
"updated_at": {
"type": "string"
}
},
"type": "object"
},
"response.CharCountRatioResponse": {
"properties": {
"char_ratio": {
"example": 0.75,
"type": "number"
}
},
"type": "object"
},
"response.CharCountResponse": {
"properties": {
"char_count": {
"example": 42,
"type": "integer"
}
},
"type": "object"
},
"response.ContextRelevanceResponse": {
"properties": {
"relevance_score": {
"example": 0.88,
"type": "number"
}
},
"type": "object"
},
"response.ConversationQualityResponse": {
"properties": {
"conversation_quality_score": {
"example": 0.82,
"type": "number"
}
},
"type": "object"
},
"response.CreateEvaluatorResponse": {
"properties": {
"bindings": {
"items": {
"type": "string"
},
"type": "array"
},
"evaluator_id": {
"type": "string"
},
"slug": {
"type": "string"
},
"type": {
"$ref": "#/components/schemas/types.EvaluatorType"
},
"version_id": {
"type": "string"
}
},
"type": "object"
},
"response.CreateOrganizationResponse": {
"properties": {
"environments": {
"items": {
"$ref": "#/components/schemas/response.EnvironmentWithKeyResponse"
},
"type": "array"
},
"org_id": {
"type": "string"
}
},
"type": "object"
},
"response.EnvironmentWithKeyResponse": {
"properties": {
"api_key": {
"type": "string"
},
"slug": {
"type": "string"
}
},
"type": "object"
},
"response.ErrorResponse": {
"description": "Standard error response structure",
"properties": {
"error": {
"example": "error message",
"type": "string"
}
},
"type": "object"
},
"response.FaithfulnessResponse": {
"properties": {
"is_faithful": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.GetMetricsResponse": {
"properties": {
"metrics": {
"$ref": "#/components/schemas/response.PaginatedMetricsResponse"
}
},
"type": "object"
},
"response.HtmlComparisonResponse": {
"properties": {
"similarity_score": {
"example": 0.92,
"type": "number"
}
},
"type": "object"
},
"response.InstructionAdherenceResponse": {
"properties": {
"instruction_adherence_score": {
"example": 0.87,
"type": "number"
}
},
"type": "object"
},
"response.IntentChangeResponse": {
"properties": {
"reason": {
"example": "User intent remained consistent throughout the conversation",
"type": "string"
},
"score": {
"example": 1,
"type": "integer"
},
"success": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.JSONValidatorResponse": {
"properties": {
"is_valid_json": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.MetricGroup": {
"properties": {
"metric_name": {
"type": "string",
"description": "Metric name.",
"example": "answer relevancy"
},
"organization_id": {
"type": "string"
},
"points": {
"items": {
"$ref": "#/components/schemas/response.MetricPoint"
},
"type": "array",
"description": "List of data points for this metric. The `unit` field in each point's `labels` indicates the data type and determines which value field is populated: `bool_value` for booleans, `numeric_value` for numbers, or `enum_value` for strings."
}
},
"type": "object"
},
"response.MetricPoint": {
"description": "A single metric data point. The `unit` field in `labels` indicates the data type and determines which value field is populated: `bool_value` for booleans unitless, `numeric_value` for number units, or `enum_value` for strings.",
"properties": {
"bool_value": {
"type": "boolean"
},
"enum_value": {
"type": "string"
},
"numeric_value": {
"type": "number"
},
"event_time": {
"type": "integer"
},
"labels": {
"description": "The labels are the attributes of the metric point. The labels are key-value pairs that are used to identify the metric point.",
"example": {
"agent_name": "Travel Planner Agent",
"entity_type": "span",
"env_project_id": "1111",
"environment": "dev",
"evaluator_name": "",
"insert_time": "1772616490000",
"metric_source": "word_count",
"metric_type": "numeric",
"model": "gpt-4o-2024-08-06",
"org_id": "12345",
"run_id": "54321",
"service_name": "travel-agent-demo",
"span_id": "e118a1985c9cff23",
"trace_id": "2c98a42d1225cb786f3395d97ca3014d",
"unit": "words",
"vendor": "openai"
},
"additionalProperties": {
"type": "string"
},
"type": "object"
}
},
"type": "object"
},
"response.MetricsHWMResponse": {
"properties": {
"high_water_mark": {
"type": "integer"
}
},
"type": "object"
},
"response.PIIDetectorResponse": {
"properties": {
"has_pii": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response.PaginatedMetricsResponse": {
"properties": {
"data": {
"items": {
"$ref": "#/components/schemas/response.MetricGroup"
},
"type": "array"
},
"next_cursor": {
"type": "string"
},
"total_points": {
"type": "integer"
},
"total_results": {
"type": "integer"
}
},
"type": "object"
},
"response.PerplexityResponse": {
"properties": {
"perplexity_score": {
"example": 12.5,
"type": "number"
}
},
"type": "object"
},
"response.PlaceholderRegexResponse": {
"properties": {
"is_valid_regex": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.ProfanityDetectorResponse": {
"properties": {
"is_safe": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response.PromptInjectionResponse": {
"properties": {
"has_injection": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response.PromptPerplexityResponse": {
"properties": {
"perplexity_score": {
"example": 8.3,
"type": "number"
}
},
"type": "object"
},
"response.RegexValidatorResponse": {
"properties": {
"is_valid_regex": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.SQLValidatorResponse": {
"properties": {
"is_valid_sql": {
"example": true,
"type": "boolean"
}
},
"type": "object"
},
"response.SecretsDetectorResponse": {
"properties": {
"has_secret": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response.SemanticSimilarityResponse": {
"properties": {
"similarity_score": {
"example": 0.92,
"type": "number"
}
},
"type": "object"
},
"response.SexismDetectorResponse": {
"properties": {
"is_safe": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response.ToneDetectionResponse": {
"properties": {
"score": {
"example": 0.95,
"type": "number"
},
"tone": {
"example": "neutral",
"type": "string"
}
},
"type": "object"
},
"response.TopicAdherenceResponse": {
"properties": {
"adherence_score": {
"example": 0.95,
"type": "number"
}
},
"type": "object"
},
"response.ToxicityDetectorResponse": {
"properties": {
"is_safe": {
"example": false,
"type": "boolean"
}
},
"type": "object"
},
"response.UncertaintyDetectorResponse": {
"properties": {
"answer": {
"example": "Paris",
"type": "string"
},
"uncertainty": {
"example": 0.95,
"type": "number"
}
},
"type": "object"
},
"response.WordCountRatioResponse": {
"properties": {
"word_ratio": {
"example": 0.85,
"type": "number"
}
},
"type": "object"
},
"response.WordCountResponse": {
"properties": {
"word_count": {
"example": 10,
"type": "integer"
}
},
"type": "object"
},
"shared.FilterCondition": {
"properties": {
"field": {
"type": "string",
"description": "Field key to filter by. The key is labels attribute on the metric point from the response.metrics object",
"example": "labels.agent_name"
},
"operator": {
"type": "string",
"description": "Comparison operator.",
"enum": [
"equals",
"not_equals",
"contains",
"not_contains",
"in",
"not_in"
],
"example": "equals"
},
"value": {
"type": "string",
"description": "Value to filter by.",
"example": "Travel Planner Agent"
},
"values": {
"items": {
"type": "string"
},
"type": "array",
"description": "List of values to filter by. This is only used for the in and not_in operators.",
"example": [
"Travel Planner Agent",
"Calendar Planner Agent"
]
}
},
"type": "object"
},
"types.ComparisonOperator": {
"enum": [
"equals",
"not_equals",
"contains",
"not_contains",
"in",
"not_in"
],
"type": "string",
"x-enum-varnames": [
"ComparisonOperatorEquals",
"ComparisonOperatorNotEquals",
"ComparisonOperatorContains",
"ComparisonOperatorNotContains",
"ComparisonOperatorIn",
"ComparisonOperatorNotIn"
]
},
"types.EvaluatorType": {
"enum": [
"AGENT_GOAL_ACCURACY",
"CHAR_COUNT",
"CHAR_COUNT_RATIO",
"ANSWER_COMPLETENESS",
"FAITHFULNESS",
"JSON_VALIDATOR",
"PERPLEXITY",
"PII_DETECTOR",
"PLACEHOLDER_REGEX",
"PROFANITY_DETECTOR",
"SEXISM_DETECTOR",
"PROMPT_INJECTION",
"TONE_DETECTION",
"PROMPT_PERPLEXITY",
"TOXICITY_DETECTOR",
"ANSWER_RELEVANCY",
"ANSWER_CORRECTNESS",
"REGEX_VALIDATOR",
"SECRETS_DETECTOR",
"SQL_VALIDATOR",
"SEMANTIC_SIMILARITY",
"TOPIC_ADHERENCE",
"UNCERTAINTY_DETECTOR",
"WORD_COUNT",
"WORD_COUNT_RATIO",
"INSTRUCTION_ADHERENCE",
"AGENT_FLOW_QUALITY",
"AGENT_EFFICIENCY",
"AGENT_GOAL_COMPLETENESS",
"CONVERSATION_QUALITY",
"INTENT_CHANGE",
"CONTEXT_RELEVANCE",
"AGENT_TOOL_ERROR_DETECTOR",
"AGENT_TOOL_TRAJECTORY",
"HTML_COMPARISON",
"LLM_AS_A_JUDGE"
],
"type": "string",
"x-enum-varnames": [
"AgentGoalAccuracy",
"CharCount",
"CharCountRatio",
"AnswerCompleteness",
"Faithfulness",
"JsonValidator",
"Perplexity",
"PIIDetector",
"PlaceholderRegex",
"ProfanityDetector",
"SexismDetector",
"PromptInjection",
"ToneDetection",
"PromptPerplexity",
"ToxicityDetector",
"AnswerRelevancy",
"AnswerCorrectness",
"RegexValidator",
"SecretsDetector",
"SQLValidator",
"SemanticSimilarity",
"TopicAdherence",
"UncertaintyDetector",
"WordCount",
"WordCountRatio",
"InstructionAdherence",
"AgentFlowQuality",
"AgentEfficiency",
"AgentGoalCompleteness",
"ConversationQuality",
"IdentIntentChange",
"ContextRelevance",
"AgentToolErrorDetector",
"AgentToolTrajectory",
"HtmlComparison",
"LLMAsAJudge"
]
},
"types.LogicalOperator": {
"enum": [
"AND",
"OR"
],
"type": "string",
"x-enum-varnames": [
"LogicalOperatorAnd",
"LogicalOperatorOr"
]
},
"types.Rule": {
"properties": {
"key": {
"type": "string"
},
"op": {
"$ref": "#/components/schemas/types.ComparisonOperator"
},
"source": {
"type": "string"
},
"value": {
"type": "string"
},
"value_type": {
"type": "string"
}
},
"type": "object"
}
}
}
}