{ "openapi": "3.1.0", "info": { "title": "AI Crucible API", "version": "1.0.0", "description": "OpenAI-compatible REST API for running AI Crucible ensemble strategies (multiple LLMs orchestrated through a strategy and judged or synthesized by an arbiter model), evaluating model responses with AI judges, and reading usage.\n\n**API access is Pro-only.** API keys can only be created, listed and rotated by users on the Pro subscription tier (`requireProTier` in the key-management functions), and every API request re-checks that the key owner still has an active Pro subscription; otherwise it is rejected with 403. Keys are managed in the dashboard under Settings > API Keys; they are not created through this API.\n\nAuthentication: send the key as `Authorization: Bearer `. Keys have the form `sk-aicruc_`.\n\nErrors from authenticated endpoints are JSON of the form `{\"error\": {\"message\": string, \"type\": string}}`. There is no `code` field.\n\nLong-running multi-model strategies can take several minutes; the server timeout is 540 seconds.", "contact": { "name": "AI Crucible Support", "email": "support@ai-crucible.com", "url": "https://ai-crucible.com" } }, "servers": [ { "url": "https://ai-crucible.com", "description": "Production. Firebase Hosting rewrites `/v1/**` to the `api` Cloud Function. Paths below include the `/v1` prefix, so OpenAI SDK clients should use `https://ai-crucible.com/v1` as their base URL." } ], "security": [ { "ApiKeyAuth": [] } ], "tags": [ { "name": "Chat", "description": "OpenAI-compatible chat completions with ensemble orchestration." }, { "name": "Responses", "description": "OpenAI Responses-style endpoint with single-string input." }, { "name": "Evaluations", "description": "Score model responses with AI judge models." }, { "name": "Usage", "description": "Token and cost usage for the authenticated user." } ], "paths": { "/v1/chat/completions": { "post": { "tags": ["Chat"], "operationId": "createChatCompletion", "summary": "Create a chat completion using an ensemble strategy", "description": "Runs the selected ensemble strategy. The root `model` is the arbiter/judge; `ai_crucible.models` are the participants. The last message is the prompt; earlier non-system messages become conversation history; the first system message is used as the arbiter system prompt unless `ai_crucible.systemPrompt` is set. With `stream: true` the response is Server-Sent Events.", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ChatCompletionRequest" }, "example": { "model": "claude-sonnet-5", "messages": [ { "role": "system", "content": "You are a precise technical editor. Pick the most accurate answer." }, { "role": "user", "content": "Explain the difference between optimistic and pessimistic locking in two paragraphs." } ], "temperature": 0.7, "stream": false, "ai_crucible": { "strategy": "competitive_refinement", "rounds": 2, "models": ["gemini-3.8-flash", "gpt-5.4-mini"], "includeCandidates": true, "includeReasoning": true } } } } }, "responses": { "200": { "description": "Completion. JSON when `stream` is false or omitted; an SSE stream of `ChatCompletionChunk` objects terminated by `data: [DONE]` when `stream` is true.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ChatCompletionResponse" } }, "text/event-stream": { "schema": { "type": "string", "description": "Server-Sent Events. Each event is `data: ` followed by a blank line. Payloads are `ChatCompletionChunk` objects (or `StreamError`), and the stream ends with `data: [DONE]`." }, "x-itemSchema": { "oneOf": [ { "$ref": "#/components/schemas/ChatCompletionChunk" }, { "$ref": "#/components/schemas/StreamError" } ] } } } }, "400": { "description": "Invalid request (empty/missing `messages`, `temperature` outside 0-2, non-positive `max_tokens`).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "error": { "message": "Invalid messages array", "type": "invalid_request_error" } } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" }, "429": { "$ref": "#/components/responses/RateLimited" }, "500": { "$ref": "#/components/responses/InternalError" } } } }, "/v1/responses": { "post": { "tags": ["Responses"], "operationId": "createResponse", "summary": "Create a response from a single input string", "description": "Responses-style endpoint. `input` must be a string. Pass a previous `id` as `previous_response_id` to continue a conversation. If `ai_crucible.strategy` is omitted, the strategy is taken from `model` when it contains \"ensemble\" or \"panel\", otherwise a single-model run is requested. With `stream: true` the response is SSE using the same chat.completion.chunk format as /v1/chat/completions.", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ResponsesRequest" }, "example": { "model": "gemini-3.8-flash", "input": "Summarize the trade-offs between SQLite and PostgreSQL for a small SaaS app.", "stream": false, "ai_crucible": { "strategy": "collaborative_synthesis" } } } } }, "responses": { "200": { "description": "Response object, or an SSE stream when `stream` is true.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ResponsesResponse" } }, "text/event-stream": { "schema": { "type": "string", "description": "Server-Sent Events carrying `ChatCompletionChunk` objects (not Responses-API event types), ending with `data: [DONE]`." }, "x-itemSchema": { "oneOf": [ { "$ref": "#/components/schemas/ChatCompletionChunk" }, { "$ref": "#/components/schemas/StreamError" } ] } } } }, "400": { "description": "`input` missing or not a string.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "error": { "message": "Invalid input string", "type": "invalid_request_error" } } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" }, "429": { "$ref": "#/components/responses/RateLimited" }, "500": { "$ref": "#/components/responses/InternalError" } } } }, "/v1/evaluations": { "post": { "tags": ["Evaluations"], "operationId": "createEvaluation", "summary": "Evaluate model responses with AI judges", "description": "Scores each response on accuracy, creativity, clarity, completeness and usefulness using one or more judge models. Multiple judges produce consensus scores.", "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvaluationRequest" }, "example": { "prompt": "What is the capital of France?", "responses": [ { "modelId": "gemini-3.8-flash", "modelName": "Gemini 3.8 Flash", "response": "The capital of France is Paris, home of the Eiffel Tower." }, { "modelId": "gpt-5.4-mini", "modelName": "GPT-5.4 Mini", "response": "Paris is the capital of France." } ], "judge_models": [ { "id": "claude-sonnet-5", "name": "Claude Sonnet 5" } ], "evaluation_mode": "standard", "weighted": false } } } }, "responses": { "200": { "description": "Evaluation results.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/EvaluationResponse" } } } }, "400": { "description": "Validation failed (missing prompt, empty responses or judge_models, missing fields, unsupported strategy).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "error": { "message": "Invalid request: judge_models is required and must be a non-empty array", "type": "invalid_request_error" } } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" }, "429": { "$ref": "#/components/responses/RateLimited" }, "500": { "$ref": "#/components/responses/InternalError" } } } }, "/v1/usage": { "get": { "tags": ["Usage"], "operationId": "getUsage", "summary": "Get usage statistics", "description": "Aggregates monthly usage documents covering the requested date range (at most 24 months). Defaults to the start of the current month through now.", "parameters": [ { "name": "start_date", "in": "query", "required": false, "description": "Start of range. Either a Unix timestamp in seconds (digits only) or any date string parseable by JavaScript `Date` (e.g. ISO 8601). Default: first day of the current month.", "schema": { "type": "string" }, "example": "2026-09-01" }, { "name": "end_date", "in": "query", "required": false, "description": "End of range, same formats as `start_date`. Default: now.", "schema": { "type": "string" }, "example": "1790000000" } ], "responses": { "200": { "description": "Usage list.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UsageResponse" } } } }, "400": { "description": "Invalid date format.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "error": { "message": "Invalid date format", "type": "invalid_request_error" } } } } }, "401": { "$ref": "#/components/responses/Unauthorized" }, "403": { "$ref": "#/components/responses/Forbidden" }, "429": { "$ref": "#/components/responses/RateLimited" }, "500": { "$ref": "#/components/responses/InternalError" } } } } }, "components": { "securitySchemes": { "ApiKeyAuth": { "type": "http", "scheme": "bearer", "bearerFormat": "sk-aicruc_", "description": "API key sent as `Authorization: Bearer sk-aicruc_...`. Keys are created in the dashboard (Pro tier only) and are stored hashed; revoked or inactive keys are rejected with 401." } }, "responses": { "Unauthorized": { "description": "Missing, malformed, invalid, revoked or inactive API key.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "error": { "message": "Unauthorized: Invalid API key", "type": "invalid_request_error" } } } } }, "Forbidden": { "description": "The API key is valid but its owner no longer has an active Pro subscription (tier `pro` with status `active` or `trialing`). The subscription is re-checked on every request, so keys stop working after a downgrade, cancellation or failed payment; changes take effect within about a minute. Other Firebase `permission-denied` errors also map to 403.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "error": { "message": "API access requires an active Pro subscription. Please upgrade or renew your plan.", "type": "permission_error" } } } } }, "RateLimited": { "description": "Rate limit exceeded. Limits are per user over a one-minute window; Pro allows 30 requests per minute.", "headers": { "Retry-After": { "description": "Seconds to wait before retrying.", "schema": { "type": "integer" } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" }, "example": { "error": { "message": "Rate limit exceeded", "type": "rate_limit_error" } } } } }, "InternalError": { "description": "Internal error, including orchestration failures.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Error" } } } } }, "schemas": { "Error": { "type": "object", "required": ["error"], "properties": { "error": { "type": "object", "required": ["message", "type"], "properties": { "message": { "type": "string", "description": "Human-readable error message." }, "type": { "type": "string", "description": "Error category. `invalid_request_error` unless the thrown error carries its own type (e.g. `rate_limit_error`).", "examples": ["invalid_request_error", "permission_error", "rate_limit_error"] } } } } }, "StreamError": { "type": "object", "description": "Emitted inside an SSE stream when orchestration fails after headers were sent. The stream then ends (no `[DONE]` for orchestration error events).", "required": ["error"], "properties": { "error": { "type": "string", "description": "Error message (a plain string, not an object)." } } }, "EnsembleStrategy": { "type": "string", "enum": [ "competitive_refinement", "collaborative_synthesis", "expert_panel", "debate_tournament", "hierarchical", "chain_of_thought", "red_team_blue_team" ], "description": "Ensemble strategy ID. On /v1/chat/completions an unknown value silently falls back to `competitive_refinement` (the default)." }, "ModelId": { "type": "string", "description": "Model identifier: an AI Crucible model id (lowercase, hyphen/dot separated, e.g. `gemini-3.8-flash`, `claude-sonnet-5`) or a provider model name from the model catalog. Unknown values are passed through as-is. The listed examples are the currently non-deprecated ids; the default participant model is `gemini-3.8-flash`.", "examples": [ "gemini-3.8-flash", "gemini-3.5-flash", "gemini-3.1-pro", "gemini-3.5-flash-lite", "gemini-3.1-flash-lite", "gemini-3-flash", "gpt", "gpt-4-mini", "gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano", "gpt-6-astra", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna", "gpt-5.5", "gpt-5.5-pro", "claude-fable-5-1", "claude-opus-5", "claude-sonnet-5", "claude-haiku-4-5", "grok-4.6", "grok-4.5", "grok-4.3", "grok-4.20-beta", "mistral-large-3", "mistral-medium-3-5", "mistral-small-4", "ministral-3-14b", "ministral-3-8b", "ministral-3-3b", "glm-5-3", "glm-5-3-flash", "glm-5-2", "qwen-3.8-max", "qwen-3.8-flash", "qwen-3.7-max", "qwen-3.7-plus", "qwen-3.6-flash", "qwen-3.5", "qwen-flash", "kimi-k3", "kimi-k2.6", "deepseek-v4-pro", "deepseek-v4-flash" ] }, "ChatMessage": { "type": "object", "required": ["role", "content"], "properties": { "role": { "type": "string", "enum": ["system", "user", "assistant", "tool"] }, "content": { "type": "string", "description": "Message text." }, "name": { "type": "string" }, "tool_calls": { "type": "array", "items": { "type": "object" }, "description": "Accepted for compatibility; not used." } } }, "AiCrucibleOptions": { "type": "object", "description": "AI Crucible extension object. Controls ensemble orchestration. Additional unknown properties are accepted and ignored.", "additionalProperties": true, "properties": { "strategy": { "$ref": "#/components/schemas/EnsembleStrategy" }, "rounds": { "type": "integer", "minimum": 1, "maximum": 10, "default": 1, "description": "Number of refinement rounds. Non-integers are floored; values outside 1-10 are clamped; non-numbers fall back to 1." }, "iterations": { "type": "integer", "minimum": 1, "maximum": 10, "default": 1, "description": "Alias for `rounds`. If both are sent, `iterations` wins." }, "models": { "type": "array", "items": { "$ref": "#/components/schemas/ModelId" }, "description": "Participant models that generate competing responses. Model identifiers may be passed as strings (or `{id}`/`{name}` objects, which are also resolved)." }, "includeFollowUpPrompts": { "type": "boolean", "default": false, "description": "Generate follow-up prompt suggestions." }, "includeCandidates": { "type": "boolean", "default": false, "description": "Include individual participant responses in `choices[].candidates`." }, "includeReasoning": { "type": "boolean", "default": false, "description": "Include the arbiter's explanation. If omitted, it is enabled whenever `reasoning.effort` is set." }, "systemPrompt": { "type": "string", "maxLength": 100000, "description": "Instructions for the arbiter/synthesis model. Truncated to 100,000 characters; empty string is treated as absent." } } }, "Reasoning": { "type": "object", "description": "OpenAI-style reasoning control. Setting `effort` (any value) enables the explanation field unless `ai_crucible.includeReasoning` is set explicitly. The effort level itself is not used.", "additionalProperties": true, "properties": { "effort": { "type": "string", "enum": ["low", "medium", "high"] } } }, "ChatCompletionRequest": { "type": "object", "required": ["model", "messages"], "properties": { "model": { "$ref": "#/components/schemas/ModelId", "description": "Arbiter/judge model. If `ai_crucible.models` is omitted, this model is also the sole participant." }, "messages": { "type": "array", "minItems": 1, "items": { "$ref": "#/components/schemas/ChatMessage" }, "description": "Conversation. The last message is the prompt." }, "stream": { "type": "boolean", "default": false, "description": "Return Server-Sent Events." }, "temperature": { "type": "number", "minimum": 0, "maximum": 2, "description": "Applied to the final arbiter/synthesis step." }, "top_p": { "type": "number", "description": "Applied to the final arbiter/synthesis step." }, "max_tokens": { "type": "integer", "exclusiveMinimum": 0, "description": "Validated (must be > 0) but not forwarded to models." }, "n": { "type": "integer", "description": "Accepted for OpenAI compatibility; ignored (always one choice)." }, "stop": { "oneOf": [ { "type": "string" }, { "type": "array", "items": { "type": "string" } } ], "description": "Accepted for compatibility; ignored." }, "presence_penalty": { "type": "number", "description": "Accepted for compatibility; ignored." }, "frequency_penalty": { "type": "number", "description": "Accepted for compatibility; ignored." }, "logit_bias": { "type": "object", "additionalProperties": { "type": "number" }, "description": "Accepted for compatibility; ignored." }, "user": { "type": "string", "description": "Accepted for compatibility; ignored." }, "reasoning": { "$ref": "#/components/schemas/Reasoning" }, "ai_crucible": { "$ref": "#/components/schemas/AiCrucibleOptions" } } }, "Candidate": { "type": "object", "required": ["modelId", "modelName", "response"], "properties": { "modelId": { "type": "string" }, "modelName": { "type": "string" }, "response": { "type": "string" }, "tokenUsage": { "type": "object", "properties": { "promptTokens": { "type": "integer" }, "completionTokens": { "type": "integer" }, "totalTokens": { "type": "integer" } } }, "executionTimeMs": { "type": "number" }, "cost": { "type": "number", "description": "Cost in USD." } } }, "ChatCompletionResponse": { "type": "object", "required": ["id", "object", "created", "model", "choices", "usage"], "properties": { "id": { "type": "string", "description": "`chatcmpl-`." }, "object": { "type": "string", "const": "chat.completion" }, "created": { "type": "integer", "description": "Unix seconds." }, "model": { "type": "string", "description": "Echo of the request `model`." }, "choices": { "type": "array", "items": { "type": "object", "required": ["index", "message", "finish_reason"], "properties": { "index": { "type": "integer" }, "message": { "type": "object", "required": ["role", "content"], "properties": { "role": { "type": "string", "const": "assistant" }, "content": { "type": "string", "description": "Best answer or synthesis." }, "explanation": { "type": "string", "description": "Arbiter's explanation (when reasoning is enabled and the strategy produces one)." } } }, "finish_reason": { "type": "string", "const": "stop" }, "candidates": { "type": "array", "items": { "$ref": "#/components/schemas/Candidate" }, "description": "Participant responses, when available from the strategy." } } } }, "usage": { "type": "object", "required": ["prompt_tokens", "completion_tokens", "total_tokens"], "properties": { "prompt_tokens": { "type": "integer" }, "completion_tokens": { "type": "integer" }, "total_tokens": { "type": "integer" }, "input_tokens": { "type": "integer", "description": "Same as prompt_tokens." }, "output_tokens": { "type": "integer", "description": "Same as completion_tokens." }, "input_token_details": { "type": "object", "properties": { "text_tokens": { "type": "integer" }, "audio_tokens": { "type": "integer" }, "image_tokens": { "type": "integer" }, "cached_tokens": { "type": "integer" }, "cached_tokens_details": { "type": "object", "properties": { "text_tokens": { "type": "integer" }, "audio_tokens": { "type": "integer" }, "image_tokens": { "type": "integer" } } } } }, "output_token_details": { "type": "object", "properties": { "text_tokens": { "type": "integer" }, "audio_tokens": { "type": "integer" } } } } } } }, "ChatCompletionChunk": { "type": "object", "required": ["id", "object", "created", "model", "choices"], "description": "SSE payload. Content deltas carry the final synthesized/best answer. Progress chunks carry an empty delta plus `status`. A `candidates` chunk may be sent before the end. A usage chunk (empty `choices`) is sent when tokens were consumed, followed by a final chunk with `finish_reason: \"stop\"`, then `data: [DONE]`. `stream_options` is not read.", "properties": { "id": { "type": "string" }, "object": { "type": "string", "const": "chat.completion.chunk" }, "created": { "type": "integer" }, "model": { "type": "string" }, "choices": { "type": "array", "items": { "type": "object", "properties": { "index": { "type": "integer" }, "delta": { "type": "object", "properties": { "content": { "type": "string" } } }, "finish_reason": { "type": ["string", "null"], "enum": ["stop", null] }, "explanation": { "type": "string", "description": "Present on the final chunk when the arbiter produced an explanation." } } } }, "status": { "type": "string", "description": "Progress message (progress chunks only)." }, "candidates": { "type": "array", "items": { "type": "object", "properties": { "model": { "type": "string" }, "text": { "type": "string" }, "reasoning": { "type": "string" } } } }, "usage": { "type": "object", "properties": { "prompt_tokens": { "type": "integer" }, "completion_tokens": { "type": "integer" }, "total_tokens": { "type": "integer" }, "unifiedTokens": { "type": "number", "description": "AI Crucible billing units." }, "input_tokens": { "type": "integer" }, "output_tokens": { "type": "integer" }, "input_token_details": { "type": "object" }, "output_token_details": { "type": "object" } } } } }, "ResponsesRequest": { "type": "object", "required": ["model", "input"], "properties": { "model": { "$ref": "#/components/schemas/ModelId", "description": "Model to run; also the arbiter. Required in practice (strategy detection reads it)." }, "input": { "type": "string", "minLength": 1, "description": "Prompt text. Arrays are rejected." }, "previous_response_id": { "type": "string", "description": "Accepted for compatibility but has no effect: /v1/responses does not store conversations, so each call runs without earlier history." }, "stream": { "type": "boolean", "default": false }, "temperature": { "type": "number" }, "top_p": { "type": "number" }, "reasoning": { "$ref": "#/components/schemas/Reasoning" }, "ai_crucible": { "$ref": "#/components/schemas/AiCrucibleOptions" } } }, "ResponsesResponse": { "type": "object", "required": ["id", "object", "created", "model", "output", "usage"], "properties": { "id": { "type": "string", "description": "`resp_` when a chat id is available, otherwise `resp_`." }, "object": { "type": "string", "const": "response" }, "created": { "type": "integer" }, "model": { "type": "string" }, "output": { "type": "array", "items": { "type": "object", "required": ["type", "id"], "properties": { "type": { "type": "string", "enum": ["message", "reasoning", "function_call", "tool_call"], "description": "Currently always `message`." }, "id": { "type": "string" }, "role": { "type": "string", "const": "assistant" }, "content": { "type": "array", "items": { "type": "object", "required": ["type", "text"], "properties": { "type": { "type": "string", "const": "text" }, "text": { "type": "string" } } } } } } }, "usage": { "type": "object", "required": ["input_tokens", "output_tokens", "total_tokens"], "properties": { "input_tokens": { "type": "integer" }, "output_tokens": { "type": "integer" }, "total_tokens": { "type": "integer" } } } } }, "EvaluationRequest": { "type": "object", "required": ["prompt", "responses", "judge_models"], "properties": { "prompt": { "type": "string", "description": "The original prompt the responses answer." }, "responses": { "type": "array", "minItems": 1, "items": { "type": "object", "required": ["modelId", "modelName", "response"], "properties": { "modelId": { "$ref": "#/components/schemas/ModelId" }, "modelName": { "type": "string" }, "response": { "type": "string", "minLength": 1 }, "role": { "type": "string", "enum": [ "blueTeam", "redTeam", "whiteTeam", "proposition", "opposition", "judge", "strategist", "implementer", "reviewer" ], "description": "Team role for role-aware evaluation." } } } }, "judge_models": { "type": "array", "minItems": 1, "items": { "type": "object", "required": ["id", "name"], "properties": { "id": { "$ref": "#/components/schemas/ModelId" }, "name": { "type": "string" } } } }, "strategy": { "type": "string", "enum": ["standard", "speed_pair", "round_robin", "competitive_refinement"], "description": "Strategy context. Other values are rejected with 400." }, "evaluation_mode": { "type": "string", "enum": ["standard", "pointwise"], "default": "standard", "description": "Any value other than `pointwise` is treated as `standard`." }, "weighted": { "type": "boolean", "default": false } } }, "EvaluationCriteria": { "type": "object", "description": "Scores 0-10. Role-aware evaluations may carry additional criteria.", "required": ["accuracy", "creativity", "clarity", "completeness", "usefulness"], "additionalProperties": { "type": "number" }, "properties": { "accuracy": { "type": "number" }, "creativity": { "type": "number" }, "clarity": { "type": "number" }, "completeness": { "type": "number" }, "usefulness": { "type": "number" } } }, "ModelEvaluation": { "type": "object", "required": ["modelId", "modelName", "response", "criteria", "overallScore", "reasoning"], "properties": { "modelId": { "type": "string" }, "modelName": { "type": "string" }, "response": { "type": "string" }, "criteria": { "$ref": "#/components/schemas/EvaluationCriteria" }, "overallScore": { "type": "number" }, "reasoning": { "type": "string" }, "role": { "type": "string" }, "roleSpecificCriteria": { "type": "object", "additionalProperties": { "type": "number" } }, "individualScores": { "type": "array", "items": { "type": "object", "required": ["judgeId", "judgeName", "criteria", "overallScore", "reasoning"], "properties": { "judgeId": { "type": "string" }, "judgeName": { "type": "string" }, "criteria": { "$ref": "#/components/schemas/EvaluationCriteria" }, "overallScore": { "type": "number" }, "reasoning": { "type": "string" } } } }, "overallScoreVariance": { "type": "number" }, "scoredByJudges": { "type": "integer" }, "totalJudges": { "type": "integer" }, "notScored": { "type": "boolean", "description": "True when no judge produced a usable score; scores are 0 placeholders." }, "divergencePercent": { "type": "number", "minimum": 0, "maximum": 100 }, "weighted": { "type": "boolean" }, "evaluationMode": { "type": "string", "enum": ["standard", "pointwise"] } } }, "EvaluationResponse": { "type": "object", "required": ["evaluations", "usage"], "properties": { "evaluations": { "type": "array", "items": { "$ref": "#/components/schemas/ModelEvaluation" } }, "usage": { "type": "object", "required": ["prompt_tokens", "completion_tokens", "total_tokens", "total_cost"], "properties": { "prompt_tokens": { "type": "integer" }, "completion_tokens": { "type": "integer" }, "total_tokens": { "type": "integer" }, "total_cost": { "type": "number", "description": "USD." } } } } }, "UsageResponse": { "type": "object", "required": ["object", "data", "has_more"], "properties": { "object": { "type": "string", "const": "list" }, "has_more": { "type": "boolean", "const": false }, "data": { "type": "array", "maxItems": 1, "items": { "type": "object", "required": [ "aggregation_timestamp", "n_requests", "n_context_tokens_total", "n_generated_tokens_total", "total_tokens", "total_cost", "period", "breakdown", "apiKeyBreakdown" ], "properties": { "aggregation_timestamp": { "type": "integer", "description": "Unix seconds." }, "n_requests": { "type": "integer" }, "n_context_tokens_total": { "type": "integer" }, "n_generated_tokens_total": { "type": "integer" }, "total_tokens": { "type": "integer" }, "total_cost": { "type": "number", "description": "USD." }, "period": { "type": "object", "required": ["start", "end"], "properties": { "start": { "type": "string", "format": "date-time" }, "end": { "type": "string", "format": "date-time" } } }, "breakdown": { "type": "array", "items": { "type": "object", "properties": { "model_id": { "type": "string" }, "model_name": { "type": "string" }, "n_requests": { "type": "integer" }, "cost": { "type": "number" }, "n_context_tokens": { "type": "integer" }, "n_generated_tokens": { "type": "integer" } } } }, "apiKeyBreakdown": { "type": "array", "items": { "type": "object", "properties": { "keyName": { "type": "string" }, "requests": { "type": "integer" }, "totalTokens": { "type": "integer" }, "cost": { "type": "number" } } } } } } } } } } } }