{ "openapi": "3.1.0", "info": { "title": "TurnScope public tools", "version": "0.3.0", "description": "Analyze recorded agent runs, token and cache accounting, and comparative outcome evidence. Explicitly curated tools. Import this document with the API base URL shown in servers; choose no authentication. Client schema support varies." }, "servers": [ { "url": "https://turnscope.agentextual.com" } ], "security": [], "paths": { "/api/v1/analyze": { "post": { "operationId": "turnscope_analyze_recorded_run", "summary": "Analyze recorded run", "description": "Analyze a user-provided recorded agent run: reported input/output/cache tokens, optional context fingerprints and selected tool calls. Supply the complete trace object, not a URL. Unknowns stay unknown; findings are overlapping observations, not guaranteed savings.", "security": [], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalyzeRequest" } } } }, "responses": { "200": { "description": "Complete JSON analysis report. Counts, prices and outcomes are caller-supplied; read the report limitations.", "content": { "application/json": { "schema": { "type": "object", "description": "Analysis report; output fields depend on the operation." } } } }, "400": { "description": "Malformed JSON, invalid Content-Type, or request body timeout (10 seconds).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "413": { "description": "Request exceeds the 2 MiB body limit.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "422": { "description": "Schema, identity, accounting or other semantic validation failed.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "429": { "description": "Another analysis is active. Honor Retry-After before retrying.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } }, "headers": { "Retry-After": { "description": "Seconds before retry (currently 1).", "schema": { "type": "integer", "minimum": 1 } } } }, "500": { "description": "Analysis or report serialization failed.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } } } } }, "/api/v1/compare": { "post": { "operationId": "turnscope_compare_recorded_runs", "summary": "Compare recorded runs", "description": "Compare user-provided baseline and candidate measurements paired by task_id and trial_id, with an explicit resource objective. Outcomes and evaluator/scope identity are caller claims. Returns within_budget, regression or insufficient_evidence; no causal or statistical proof.", "security": [], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CompareRequest" } } } }, "responses": { "200": { "description": "Complete JSON analysis report. Counts, prices and outcomes are caller-supplied; read the report limitations.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Compare_report" } } } }, "400": { "description": "Malformed JSON, invalid Content-Type, or request body timeout (10 seconds).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "413": { "description": "Request exceeds the 2 MiB body limit.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "422": { "description": "Schema, identity, accounting or other semantic validation failed.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "429": { "description": "Another analysis is active. Honor Retry-After before retrying.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } }, "headers": { "Retry-After": { "description": "Seconds before retry (currently 1).", "schema": { "type": "integer", "minimum": 1 } } } }, "500": { "description": "Analysis or report serialization failed.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } } } } }, "/api/v1/accounting": { "post": { "operationId": "turnscope_account_recorded_usage", "summary": "Account recorded usage", "description": "Account for user-provided normalized model events, cache reads/writes, optional rates and usage ownership. Supply recorded events with an explicit scope_id. Missing rates or usage remain unknown; provider pricing is never inferred. Does not fetch external telemetry.", "security": [], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AccountingRequest" } } } }, "responses": { "200": { "description": "Complete JSON analysis report. Counts, prices and outcomes are caller-supplied; read the report limitations.", "content": { "application/json": { "schema": { "type": "object", "description": "Analysis report; output fields depend on the operation." } } } }, "400": { "description": "Malformed JSON, invalid Content-Type, or request body timeout (10 seconds).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "413": { "description": "Request exceeds the 2 MiB body limit.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "422": { "description": "Schema, identity, accounting or other semantic validation failed.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "429": { "description": "Another analysis is active. Honor Retry-After before retrying.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } }, "headers": { "Retry-After": { "description": "Seconds before retry (currently 1).", "schema": { "type": "integer", "minimum": 1 } } } }, "500": { "description": "Analysis or report serialization failed.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } } } } }, "/api/v1/studies/compare": { "post": { "operationId": "turnscope_compare_recorded_study", "summary": "Compare recorded study", "description": "Compare user-provided recorded study arms against a declared task/trial manifest. Retains missing and failed trials in coverage and separates execution from outcome claims. Requires manifest, records and objective; does not run experiments or independently verify outcomes.", "security": [], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/StudyRequest" } } } }, "responses": { "200": { "description": "Complete JSON analysis report. Counts, prices and outcomes are caller-supplied; read the report limitations.", "content": { "application/json": { "schema": { "type": "object", "description": "Analysis report; output fields depend on the operation." } } } }, "400": { "description": "Malformed JSON, invalid Content-Type, or request body timeout (10 seconds).", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "413": { "description": "Request exceeds the 2 MiB body limit.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "422": { "description": "Schema, identity, accounting or other semantic validation failed.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } }, "429": { "description": "Another analysis is active. Honor Retry-After before retrying.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } }, "headers": { "Retry-After": { "description": "Seconds before retry (currently 1).", "schema": { "type": "integer", "minimum": 1 } } } }, "500": { "description": "Analysis or report serialization failed.", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/AnalysisError" } } } } } } } }, "components": { "schemas": { "AccountingRequest": { "title": "TurnScope normalized event accounting v1", "description": "2 MiB maximum. Caller-supplied normalized inclusive input and output usage. Unknown fields reject. Missing optional fields are unknown. Semantic validation also enforces acyclic references, exact usage reconciliation, explicit ownership, finite aggregate metrics <=1e12, and completeness assertions. No original prompts or tool arguments.", "type": "object", "additionalProperties": false, "required": [ "version", "scope_id", "events" ], "properties": { "version": { "const": 1, "type": "integer", "description": "Required value: 1." }, "scope_id": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "coverage": { "$ref": "#/components/schemas/Accounting_coverage" }, "root": { "anyOf": [ { "$ref": "#/components/schemas/Accounting_root" }, { "type": "null" } ] }, "rate_catalog": { "anyOf": [ { "$ref": "#/components/schemas/Accounting_catalog" }, { "type": "null" } ] }, "charges": { "type": "array", "items": { "$ref": "#/components/schemas/Accounting_charge" }, "maxItems": 1000 }, "events": { "type": "array", "items": { "$ref": "#/components/schemas/Accounting_event" }, "maxItems": 5000 } } }, "Accounting_card": { "type": "object", "additionalProperties": false, "required": [ "provider", "model" ], "properties": { "provider": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "model": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "input_per_million": { "anyOf": [ { "type": "number", "minimum": 0, "maximum": 1000000 }, { "type": "null" } ] }, "cache_read_per_million": { "anyOf": [ { "type": "number", "minimum": 0, "maximum": 1000000 }, { "type": "null" } ] }, "cache_write_per_million": { "anyOf": [ { "type": "number", "minimum": 0, "maximum": 1000000 }, { "type": "null" } ] }, "output_per_million": { "anyOf": [ { "type": "number", "minimum": 0, "maximum": 1000000 }, { "type": "null" } ] }, "cache_write_classes": { "type": "array", "items": { "$ref": "#/components/schemas/Accounting_write_rate" }, "maxItems": 32 } } }, "Accounting_catalog": { "type": "object", "additionalProperties": false, "required": [ "id", "currency", "source", "as_of", "cards" ], "properties": { "id": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "currency": { "type": "string", "pattern": "^[A-Z]{3}$" }, "source": { "type": "string", "minLength": 1, "maxLength": 256, "description": "Nonblank text, no control characters; maximum 256 UTF-8 bytes." }, "as_of": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "cards": { "type": "array", "items": { "$ref": "#/components/schemas/Accounting_card" }, "maxItems": 1000 } } }, "Accounting_charge": { "type": "object", "additionalProperties": false, "required": [ "id", "source" ], "properties": { "id": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "amount": { "anyOf": [ { "type": "number", "minimum": 0, "maximum": 1000000000000.0 }, { "type": "null" } ] }, "source": { "type": "string", "minLength": 1, "maxLength": 256, "description": "Nonblank text, no control characters; maximum 256 UTF-8 bytes." } } }, "Accounting_coverage": { "type": "object", "additionalProperties": false, "required": [], "properties": { "events_complete": { "type": "boolean", "default": false }, "model_calls_complete": { "type": "boolean", "default": false }, "charges_complete": { "type": "boolean", "default": false }, "dependencies_complete": { "type": "boolean", "default": false }, "timing_complete": { "type": "boolean", "default": false } } }, "Accounting_event": { "type": "object", "additionalProperties": false, "required": [ "id", "kind", "status", "usage_ownership" ], "properties": { "id": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "kind": { "enum": [ "model", "tool", "wait", "container" ], "type": "string" }, "status": { "enum": [ "completed", "failed", "cancelled", "timeout", "unknown" ], "type": "string" }, "usage_ownership": { "enum": [ "self", "inclusive", "none" ], "type": "string" }, "provider": { "anyOf": [ { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, { "type": "null" } ] }, "model": { "anyOf": [ { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, { "type": "null" } ] }, "parent_id": { "anyOf": [ { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, { "type": "null" } ] }, "agent_id": { "anyOf": [ { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, { "type": "null" } ] }, "retry_of": { "anyOf": [ { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, { "type": "null" } ] }, "depends_on": { "type": "array", "items": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "maxItems": 64 }, "start_ms": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 9000000000000000 }, { "type": "null" } ] }, "end_ms": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 9000000000000000 }, { "type": "null" } ] }, "purpose": { "anyOf": [ { "enum": [ "task", "compaction", "helper", "evaluation" ], "type": "string" }, { "type": "null" } ] }, "usage": { "anyOf": [ { "$ref": "#/components/schemas/Accounting_usage" }, { "type": "null" } ] }, "regions": { "anyOf": [ { "$ref": "#/components/schemas/Accounting_regions" }, { "type": "null" } ] }, "tool": { "anyOf": [ { "$ref": "#/components/schemas/Accounting_tool" }, { "type": "null" } ] } } }, "Accounting_region": { "type": "object", "additionalProperties": false, "required": [ "fingerprint" ], "properties": { "fingerprint": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, "tokens": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 1000000000 }, { "type": "null" } ] } } }, "Accounting_regions": { "type": "object", "additionalProperties": false, "required": [ "actual_order", "complete", "items" ], "properties": { "actual_order": { "type": "boolean" }, "complete": { "type": "boolean" }, "items": { "type": "array", "items": { "$ref": "#/components/schemas/Accounting_region" }, "maxItems": 128 } } }, "Accounting_root": { "type": "object", "additionalProperties": false, "required": [ "start_ms", "end_ms" ], "properties": { "start_ms": { "type": "integer", "minimum": 0, "maximum": 9000000000000000 }, "end_ms": { "type": "integer", "minimum": 0, "maximum": 9000000000000000 } } }, "Accounting_tool": { "type": "object", "additionalProperties": false, "required": [ "name", "operation" ], "properties": { "name": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "operation": { "enum": [ "read", "search", "other" ], "type": "string" }, "arguments_fingerprint": { "anyOf": [ { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, { "type": "null" } ] }, "source_revision": { "anyOf": [ { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, { "type": "null" } ] }, "permission_scope": { "anyOf": [ { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, { "type": "null" } ] } } }, "Accounting_usage": { "type": "object", "additionalProperties": false, "required": [], "properties": { "input_tokens": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 1000000000 }, { "type": "null" } ] }, "output_tokens": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 1000000000 }, { "type": "null" } ] }, "cache_read_tokens": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 1000000000 }, { "type": "null" } ] }, "cache_write_tokens": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 1000000000 }, { "type": "null" } ] }, "reasoning_tokens": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 1000000000 }, { "type": "null" } ] }, "cache_write_classes": { "type": "array", "items": { "$ref": "#/components/schemas/Accounting_write_usage" }, "maxItems": 32 } } }, "Accounting_write_rate": { "type": "object", "additionalProperties": false, "required": [ "class", "per_million" ], "properties": { "class": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "per_million": { "type": "number", "minimum": 0, "maximum": 1000000 } } }, "Accounting_write_usage": { "type": "object", "additionalProperties": false, "required": [ "class", "tokens" ], "properties": { "class": { "type": "string", "minLength": 1, "maxLength": 128, "description": "Nonblank text, no control characters; maximum 128 UTF-8 bytes." }, "tokens": { "type": "integer", "minimum": 0, "maximum": 1000000000 } } }, "AnalysisError": { "type": "object", "required": [ "error" ], "properties": { "error": { "type": "object", "required": [ "code", "message" ], "properties": { "code": { "type": "string" }, "message": { "type": "string" } } } } }, "AnalyzeRequest": { "title": "TurnScope recorded run, version 1", "description": "Ordered model turns with reported token counts and optional non-overlapping context inventories. UTF-8 byte limits and cross-field accounting are additionally validated by the API.", "type": "object", "additionalProperties": false, "required": [ "version", "turns" ], "properties": { "version": { "const": 1, "type": "integer", "description": "Required value: 1." }, "name": { "type": [ "string", "null" ], "minLength": 1, "maxLength": 256 }, "model": { "type": [ "string", "null" ], "minLength": 1, "maxLength": 256 }, "pricing": { "anyOf": [ { "$ref": "#/components/schemas/Analyze_pricing" }, { "type": "null" } ] }, "turns": { "type": "array", "minItems": 1, "maxItems": 1000, "items": { "$ref": "#/components/schemas/Analyze_turn" } } } }, "Analyze_block": { "type": "object", "additionalProperties": false, "required": [ "id", "kind", "tokens" ], "anyOf": [ { "required": [ "fingerprint" ], "properties": { "fingerprint": { "type": "string" } } }, { "required": [ "content" ], "properties": { "content": { "type": "string" } } } ], "properties": { "id": { "$ref": "#/components/schemas/Analyze_label", "description": "Display label; unique within a turn. Reusing it does not establish identity." }, "kind": { "enum": [ "static_document", "tool_result", "tool_schema", "history", "other" ], "type": "string" }, "tokens": { "$ref": "#/components/schemas/Analyze_tokens" }, "fingerprint": { "type": [ "string", "null" ], "pattern": "^sha256:[a-f0-9]{64}$", "description": "SHA-256 of the exact UTF-8 content. Must match supplied content. A fingerprint keeps the same kind, tokens, and tool_name in this run." }, "content": { "type": [ "string", "null" ], "maxLength": 32768, "description": "Optional exact content; at most 32 KiB UTF-8. Hashed locally by the service and never echoed. No tokenization is performed." }, "tool_name": { "type": [ "string", "null" ], "minLength": 1, "maxLength": 128, "description": "Exact call name; needed to evaluate schema selection." } } }, "Analyze_call": { "type": "object", "additionalProperties": false, "required": [ "name", "kind" ], "properties": { "name": { "type": "string", "minLength": 1, "maxLength": 128 }, "kind": { "enum": [ "discovery", "read", "search", "other" ], "type": "string" }, "target": { "type": [ "string", "null" ], "minLength": 1, "maxLength": 2048, "description": "Exact stable target or query. No normalization is performed; omit when unknown." } } }, "Analyze_label": { "type": "string", "minLength": 1, "maxLength": 256 }, "Analyze_pricing": { "type": "object", "additionalProperties": false, "required": [ "currency", "input_per_million", "cached_input_per_million", "output_per_million" ], "properties": { "currency": { "type": "string", "pattern": "^[A-Z]{3}$" }, "input_per_million": { "$ref": "#/components/schemas/Analyze_rate" }, "cached_input_per_million": { "$ref": "#/components/schemas/Analyze_rate" }, "output_per_million": { "$ref": "#/components/schemas/Analyze_rate" } } }, "Analyze_rate": { "type": "number", "minimum": 0, "maximum": 1000000 }, "Analyze_tokens": { "type": "integer", "minimum": 0, "maximum": 1000000000 }, "Analyze_turn": { "type": "object", "additionalProperties": false, "required": [ "id", "input_tokens", "output_tokens" ], "properties": { "id": { "$ref": "#/components/schemas/Analyze_label", "description": "Unique within this run; order comes from array position." }, "input_tokens": { "$ref": "#/components/schemas/Analyze_tokens", "description": "Total reported input including cached input." }, "output_tokens": { "$ref": "#/components/schemas/Analyze_tokens" }, "cached_input_tokens": { "anyOf": [ { "$ref": "#/components/schemas/Analyze_tokens" }, { "type": "null" } ], "description": "Subset of input_tokens; omit or use null when unknown. Zero means known zero." }, "context_blocks": { "type": "array", "maxItems": 256, "items": { "$ref": "#/components/schemas/Analyze_block" }, "description": "Optional partial inventory. Blocks must not overlap and their tokens must sum to no more than input_tokens." }, "tool_calls": { "type": [ "array", "null" ], "maxItems": 256, "items": { "$ref": "#/components/schemas/Analyze_call" }, "description": "Calls selected by the model in this turn, not tool outputs from earlier turns. Omit/null means unknown; [] means known no calls." } } }, "CompareRequest": { "title": "TurnScope paired measurement comparison, version 1", "description": "Request for POST /api/v1/compare. Maximum serialized body: 2 MiB. Unknown fields are rejected. UTF-8 byte limits, unique task/trial keys, and ordered cost bounds are additionally validated by the API. The response schema is #/$defs/report.", "type": "object", "additionalProperties": false, "required": [ "version", "objective", "baseline", "candidate" ], "properties": { "version": { "const": 1, "type": "integer", "description": "Required value: 1." }, "objective": { "$ref": "#/components/schemas/Compare_metric" }, "max_regression_pct": { "$ref": "#/components/schemas/Compare_percentage" }, "baseline": { "type": "array", "maxItems": 1000, "items": { "$ref": "#/components/schemas/Compare_measurement" }, "minItems": 1 }, "candidate": { "type": "array", "maxItems": 1000, "items": { "$ref": "#/components/schemas/Compare_measurement" }, "minItems": 1 } } }, "Compare_basis": { "enum": [ "modeled", "reported" ], "description": "Both bases are supplied claims. Reported does not mean TurnScope verified a provider invoice.", "type": "string" }, "Compare_cost": { "type": "object", "additionalProperties": false, "required": [ "lower", "upper", "currency", "basis", "assumptions_id" ], "properties": { "lower": { "$ref": "#/components/schemas/Compare_number" }, "upper": { "$ref": "#/components/schemas/Compare_number" }, "currency": { "$ref": "#/components/schemas/Compare_currency" }, "basis": { "$ref": "#/components/schemas/Compare_basis" }, "assumptions_id": { "$ref": "#/components/schemas/Compare_identifier" } }, "description": "Finite nonnegative bounds with lower <= upper. Cost aggregation requires matching currency, basis, and assumptions_id across all supplied records." }, "Compare_costMetadata": { "type": "object", "additionalProperties": false, "required": [ "currency", "basis", "assumptions_id" ], "properties": { "currency": { "$ref": "#/components/schemas/Compare_currency" }, "basis": { "$ref": "#/components/schemas/Compare_basis" }, "assumptions_id": { "$ref": "#/components/schemas/Compare_identifier" } } }, "Compare_count": { "type": "integer", "minimum": 0, "maximum": 1000000000000 }, "Compare_coverage": { "type": "object", "additionalProperties": false, "required": [ "baseline_records", "candidate_records", "matched_pairs", "unmatched_baseline", "unmatched_candidate", "unmatched_baseline_records", "unmatched_candidate_records" ], "properties": { "baseline_records": { "$ref": "#/components/schemas/Compare_recordCount" }, "candidate_records": { "$ref": "#/components/schemas/Compare_recordCount" }, "matched_pairs": { "$ref": "#/components/schemas/Compare_recordCount" }, "unmatched_baseline": { "$ref": "#/components/schemas/Compare_recordCount" }, "unmatched_candidate": { "$ref": "#/components/schemas/Compare_recordCount" }, "unmatched_baseline_records": { "type": "array", "maxItems": 1000, "items": { "$ref": "#/components/schemas/Compare_unmatchedRecord" } }, "unmatched_candidate_records": { "type": "array", "maxItems": 1000, "items": { "$ref": "#/components/schemas/Compare_unmatchedRecord" } } } }, "Compare_currency": { "type": "string", "pattern": "^[A-Z]{3}$" }, "Compare_delta": { "type": "object", "additionalProperties": false, "required": [ "absolute", "percentage" ], "properties": { "absolute": { "$ref": "#/components/schemas/Compare_interval" }, "percentage": { "anyOf": [ { "$ref": "#/components/schemas/Compare_interval" }, { "type": "null" } ] } }, "description": "Absolute candidate minus baseline interval; percentage bounds are null if the baseline includes zero or a finite ratio cannot be represented." }, "Compare_identifier": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^(?!\\s*$)[^\\u0000-\\u001f\\u007f-\\u009f]+$", "description": "Nonblank identifier, at most 128 UTF-8 bytes, without control characters. The API additionally enforces the UTF-8 byte limit.", "x-maxUtf8Bytes": 128 }, "Compare_index": { "type": "integer", "minimum": 0, "maximum": 999 }, "Compare_interval": { "type": "object", "additionalProperties": false, "required": [ "lower", "upper" ], "properties": { "lower": { "type": "number" }, "upper": { "type": "number" } }, "description": "Finite numeric bounds with lower <= upper. Deltas may be negative; resource totals are nonnegative." }, "Compare_measurement": { "type": "object", "additionalProperties": false, "required": [ "task_id", "trial_id", "evaluator_id", "outcome", "scope_id", "metrics" ], "properties": { "task_id": { "$ref": "#/components/schemas/Compare_identifier" }, "trial_id": { "$ref": "#/components/schemas/Compare_identifier" }, "evaluator_id": { "$ref": "#/components/schemas/Compare_identifier" }, "outcome": { "$ref": "#/components/schemas/Compare_outcome" }, "scope_id": { "$ref": "#/components/schemas/Compare_identifier" }, "metrics": { "$ref": "#/components/schemas/Compare_metrics" } }, "description": "Pair by the exact (task_id, trial_id) tuple. Keys must be unique within each side. Evaluator, scope, outcome, and cost attribution remain caller claims." }, "Compare_metric": { "enum": [ "input_tokens", "output_tokens", "turns", "duration_ms", "cost" ], "type": "string" }, "Compare_metrics": { "type": "object", "additionalProperties": false, "required": [], "properties": { "input_tokens": { "anyOf": [ { "$ref": "#/components/schemas/Compare_count" }, { "type": "null" } ] }, "output_tokens": { "anyOf": [ { "$ref": "#/components/schemas/Compare_count" }, { "type": "null" } ] }, "turns": { "anyOf": [ { "$ref": "#/components/schemas/Compare_count" }, { "type": "null" } ] }, "duration_ms": { "anyOf": [ { "$ref": "#/components/schemas/Compare_number" }, { "type": "null" } ] }, "cost": { "anyOf": [ { "$ref": "#/components/schemas/Compare_cost" }, { "type": "null" } ] } }, "description": "Run-level resource measurements. Missing and null mean unknown, never zero. Duration is in milliseconds and bounded to 10^12." }, "Compare_number": { "type": "number", "minimum": 0, "maximum": 1000000000000 }, "Compare_objectiveSummary": { "type": "object", "additionalProperties": false, "required": [ "name", "max_regression_pct", "matched_pairs", "measured_pairs", "complete", "baseline", "candidate", "absolute_delta", "percentage_delta", "budget", "cost_metadata" ], "properties": { "name": { "$ref": "#/components/schemas/Compare_metric" }, "max_regression_pct": { "$ref": "#/components/schemas/Compare_percentage" }, "matched_pairs": { "$ref": "#/components/schemas/Compare_recordCount" }, "measured_pairs": { "$ref": "#/components/schemas/Compare_recordCount" }, "complete": { "type": "boolean" }, "baseline": { "anyOf": [ { "$ref": "#/components/schemas/Compare_interval" }, { "type": "null" } ] }, "candidate": { "anyOf": [ { "$ref": "#/components/schemas/Compare_interval" }, { "type": "null" } ] }, "absolute_delta": { "anyOf": [ { "$ref": "#/components/schemas/Compare_interval" }, { "type": "null" } ] }, "percentage_delta": { "anyOf": [ { "$ref": "#/components/schemas/Compare_interval" }, { "type": "null" } ] }, "budget": { "$ref": "#/components/schemas/Compare_verdict" }, "cost_metadata": { "anyOf": [ { "$ref": "#/components/schemas/Compare_costMetadata" }, { "type": "null" } ] } }, "description": "Totals cover matched pairs only. measured_pairs counts pairs with present, comparable objective values. Aggregate bounds/deltas are null if any matched objective is absent or incomparable. complete additionally requires no unmatched records; outcomes are checked separately. Within-budget: candidate.upper <= baseline.lower * (1 + percentage/100). Regression: candidate.lower > baseline.upper * factor. Otherwise the budget is uncertain. One expensive pair does not override a complete aggregate that is within budget." }, "Compare_outcome": { "enum": [ "passed", "failed", "unknown" ], "type": "string" }, "Compare_outcomeCounts": { "type": "object", "additionalProperties": false, "required": [ "passed", "failed", "unknown" ], "properties": { "passed": { "$ref": "#/components/schemas/Compare_recordCount" }, "failed": { "$ref": "#/components/schemas/Compare_recordCount" }, "unknown": { "$ref": "#/components/schemas/Compare_recordCount" } } }, "Compare_outcomes": { "type": "object", "additionalProperties": false, "required": [ "scope", "baseline", "candidate", "transitions", "quality_regressions", "quality_improvements", "unresolved_candidate_failures", "unknown_pairs" ], "properties": { "scope": { "const": "matched_pairs", "type": "string", "description": "Required value: \"matched_pairs\"." }, "baseline": { "$ref": "#/components/schemas/Compare_outcomeCounts" }, "candidate": { "$ref": "#/components/schemas/Compare_outcomeCounts" }, "transitions": { "type": "object", "additionalProperties": false, "required": [ "passed_to_passed", "passed_to_failed", "passed_to_unknown", "failed_to_passed", "failed_to_failed", "failed_to_unknown", "unknown_to_passed", "unknown_to_failed", "unknown_to_unknown" ], "properties": { "passed_to_passed": { "$ref": "#/components/schemas/Compare_recordCount" }, "passed_to_failed": { "$ref": "#/components/schemas/Compare_recordCount" }, "passed_to_unknown": { "$ref": "#/components/schemas/Compare_recordCount" }, "failed_to_passed": { "$ref": "#/components/schemas/Compare_recordCount" }, "failed_to_failed": { "$ref": "#/components/schemas/Compare_recordCount" }, "failed_to_unknown": { "$ref": "#/components/schemas/Compare_recordCount" }, "unknown_to_passed": { "$ref": "#/components/schemas/Compare_recordCount" }, "unknown_to_failed": { "$ref": "#/components/schemas/Compare_recordCount" }, "unknown_to_unknown": { "$ref": "#/components/schemas/Compare_recordCount" } } }, "quality_regressions": { "$ref": "#/components/schemas/Compare_recordCount" }, "quality_improvements": { "$ref": "#/components/schemas/Compare_recordCount" }, "unresolved_candidate_failures": { "$ref": "#/components/schemas/Compare_recordCount" }, "unknown_pairs": { "$ref": "#/components/schemas/Compare_recordCount" } }, "description": "Transitions describe supplied claims, including pairs whose evaluator or scope differ. Quality regression/improvement counts require matching evaluator and scope; raw transitions alone do not establish a comparable quality change. No causal finding is asserted." }, "Compare_pair": { "type": "object", "additionalProperties": false, "required": [ "task_id", "trial_id", "baseline_index", "candidate_index", "baseline", "candidate", "evaluator_matches", "scope_matches", "outcome_transition", "metric_deltas", "objective_budget", "issues" ], "properties": { "task_id": { "$ref": "#/components/schemas/Compare_identifier" }, "trial_id": { "$ref": "#/components/schemas/Compare_identifier" }, "baseline_index": { "$ref": "#/components/schemas/Compare_index" }, "candidate_index": { "$ref": "#/components/schemas/Compare_index" }, "baseline": { "$ref": "#/components/schemas/Compare_measurement" }, "candidate": { "$ref": "#/components/schemas/Compare_measurement" }, "evaluator_matches": { "type": "boolean" }, "scope_matches": { "type": "boolean" }, "outcome_transition": { "enum": [ "passed_to_passed", "passed_to_failed", "passed_to_unknown", "failed_to_passed", "failed_to_failed", "failed_to_unknown", "unknown_to_passed", "unknown_to_failed", "unknown_to_unknown" ], "type": "string" }, "metric_deltas": { "type": "object", "additionalProperties": false, "required": [ "input_tokens", "output_tokens", "turns", "duration_ms", "cost" ], "properties": { "input_tokens": { "anyOf": [ { "$ref": "#/components/schemas/Compare_delta" }, { "type": "null" } ] }, "output_tokens": { "anyOf": [ { "$ref": "#/components/schemas/Compare_delta" }, { "type": "null" } ] }, "turns": { "anyOf": [ { "$ref": "#/components/schemas/Compare_delta" }, { "type": "null" } ] }, "duration_ms": { "anyOf": [ { "$ref": "#/components/schemas/Compare_delta" }, { "type": "null" } ] }, "cost": { "anyOf": [ { "$ref": "#/components/schemas/Compare_delta" }, { "type": "null" } ] } } }, "objective_budget": { "$ref": "#/components/schemas/Compare_verdict" }, "issues": { "type": "array", "maxItems": 12, "items": { "enum": [ "quality_regression", "budget_regression", "unmatched_records", "no_matched_records", "evaluator_mismatch", "scope_mismatch", "unknown_outcome", "candidate_failed", "missing_objective_metric", "incompatible_cost", "budget_uncertain", "within_budget" ], "type": "string" } } }, "description": "Pairs are sorted by task then trial; indices refer to the original arrays. Deltas and budget are unavailable across evaluator/scope mismatches. Cost deltas additionally require matching pair cost attribution. Unavailable non-objective metrics do not block a decision for a complete objective." }, "Compare_percentage": { "type": "number", "minimum": 0, "maximum": 1000, "default": 0 }, "Compare_reason": { "type": "object", "additionalProperties": false, "required": [ "code", "message" ], "properties": { "code": { "enum": [ "quality_regression", "budget_regression", "unmatched_records", "no_matched_records", "evaluator_mismatch", "scope_mismatch", "unknown_outcome", "candidate_failed", "missing_objective_metric", "incompatible_cost", "budget_uncertain", "within_budget" ], "type": "string" }, "message": { "type": "string" } } }, "Compare_recordCount": { "type": "integer", "minimum": 0, "maximum": 1000 }, "Compare_report": { "type": "object", "additionalProperties": false, "required": [ "version", "verdict", "reasons", "coverage", "objective", "outcomes", "pairs", "limitations" ], "properties": { "version": { "const": 1, "type": "integer", "description": "Required value: 1." }, "verdict": { "$ref": "#/components/schemas/Compare_verdict" }, "reasons": { "type": "array", "maxItems": 12, "items": { "$ref": "#/components/schemas/Compare_reason" }, "minItems": 1 }, "coverage": { "$ref": "#/components/schemas/Compare_coverage" }, "objective": { "$ref": "#/components/schemas/Compare_objectiveSummary" }, "outcomes": { "$ref": "#/components/schemas/Compare_outcomes" }, "pairs": { "type": "array", "maxItems": 1000, "items": { "$ref": "#/components/schemas/Compare_pair" } }, "limitations": { "type": "array", "maxItems": 16, "items": { "type": "string" } } }, "description": "Response schema. A passed-to-failed transition with matching evaluator and scope, or a comparable aggregate budget regression, yields regression. A positive verdict requires complete pairing, matching evaluator/scope, known outcomes, every candidate passed, a complete compatible objective, and a proven within-budget bound. Other cases are insufficient_evidence. No statistical significance, general equivalence, or causality is asserted." }, "Compare_unmatchedRecord": { "type": "object", "additionalProperties": false, "required": [ "index", "task_id", "trial_id" ], "properties": { "index": { "$ref": "#/components/schemas/Compare_index" }, "task_id": { "$ref": "#/components/schemas/Compare_identifier" }, "trial_id": { "$ref": "#/components/schemas/Compare_identifier" } } }, "Compare_verdict": { "enum": [ "within_budget", "regression", "insufficient_evidence" ], "type": "string" }, "StudyRequest": { "title": "TurnScope study version 1", "type": "object", "additionalProperties": false, "required": [ "version", "objective", "manifest", "records" ], "properties": { "version": { "const": 1, "type": "integer", "description": "Required value: 1." }, "objective": { "enum": [ "input_tokens", "output_tokens", "turns", "duration_ms", "cost" ], "type": "string" }, "max_regression_pct": { "type": "number", "minimum": 0, "maximum": 1000, "default": 0 }, "manifest": { "$ref": "#/components/schemas/Study_manifest" }, "records": { "type": "array", "maxItems": 2000, "items": { "$ref": "#/components/schemas/Study_record" } } }, "description": "At most 2 MiB UTF-8 JSON and 1000 expected task/trial pairs per arm. Unique task IDs and configuration IDs, record keys (config_id,task_id,trial_id), declared records only, and ordered cost bounds are validated by the server. IDs are bounded in UTF-8 bytes. Absent/null outcomes and metrics remain unknown. Provenance mismatches block comparability rather than erase records." }, "Study_configuration": { "type": "object", "additionalProperties": false, "required": [ "config_id", "revision", "models", "intervention" ], "properties": { "config_id": { "$ref": "#/components/schemas/Study_id" }, "revision": { "$ref": "#/components/schemas/Study_id" }, "models": { "type": "array", "items": { "$ref": "#/components/schemas/Study_id" }, "maxItems": 32, "uniqueItems": true }, "intervention": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Declared configuration/intervention provenance; at most 1024 UTF-8 bytes without control characters.", "pattern": "^(?!\\s*$)[^\\u0000-\\u001f\\u007f-\\u009f]+$" } } }, "Study_cost": { "type": "object", "additionalProperties": false, "required": [ "lower", "upper", "currency", "basis", "assumptions_id" ], "properties": { "lower": { "type": "number", "minimum": 0, "maximum": 1000000000000 }, "upper": { "type": "number", "minimum": 0, "maximum": 1000000000000 }, "currency": { "type": "string", "pattern": "^[A-Z]{3}$" }, "basis": { "enum": [ "modeled", "reported" ], "type": "string" }, "assumptions_id": { "$ref": "#/components/schemas/Study_id" } } }, "Study_fingerprint": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, "Study_id": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^(?!\\s*$)[^\\u0000-\\u001f\\u007f-\\u009f]+$", "description": "At most 128 UTF-8 bytes." }, "Study_manifest": { "type": "object", "additionalProperties": false, "required": [ "manifest_id", "evaluator_id", "scope_id", "environment_id", "baseline", "candidate", "tasks" ], "properties": { "manifest_id": { "$ref": "#/components/schemas/Study_id" }, "evaluator_id": { "$ref": "#/components/schemas/Study_id" }, "scope_id": { "$ref": "#/components/schemas/Study_id" }, "environment_id": { "$ref": "#/components/schemas/Study_id" }, "baseline": { "$ref": "#/components/schemas/Study_configuration" }, "candidate": { "$ref": "#/components/schemas/Study_configuration" }, "tasks": { "type": "array", "minItems": 1, "maxItems": 1000, "items": { "$ref": "#/components/schemas/Study_task" } } } }, "Study_metrics": { "type": "object", "additionalProperties": false, "required": [], "properties": { "input_tokens": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 1000000000000 }, { "type": "null" } ] }, "output_tokens": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 1000000000000 }, { "type": "null" } ] }, "turns": { "anyOf": [ { "type": "integer", "minimum": 0, "maximum": 1000000000000 }, { "type": "null" } ] }, "duration_ms": { "anyOf": [ { "type": "number", "minimum": 0, "maximum": 1000000000000 }, { "type": "null" } ] }, "cost": { "anyOf": [ { "$ref": "#/components/schemas/Study_cost" }, { "type": "null" } ] } } }, "Study_record": { "type": "object", "additionalProperties": false, "required": [ "manifest_id", "config_id", "config_revision", "task_id", "trial_id", "task_fingerprint", "evaluator_id", "scope_id", "environment_id", "status", "metrics" ], "properties": { "manifest_id": { "$ref": "#/components/schemas/Study_id" }, "config_id": { "$ref": "#/components/schemas/Study_id" }, "config_revision": { "$ref": "#/components/schemas/Study_id" }, "task_id": { "$ref": "#/components/schemas/Study_id" }, "trial_id": { "$ref": "#/components/schemas/Study_id" }, "task_fingerprint": { "$ref": "#/components/schemas/Study_fingerprint" }, "evaluator_id": { "$ref": "#/components/schemas/Study_id" }, "scope_id": { "$ref": "#/components/schemas/Study_id" }, "environment_id": { "$ref": "#/components/schemas/Study_id" }, "status": { "enum": [ "completed", "failed", "cancelled", "timeout", "evaluator_error" ], "type": "string" }, "outcome": { "anyOf": [ { "enum": [ "passed", "failed", "unknown" ], "type": "string" }, { "type": "null" } ] }, "metrics": { "$ref": "#/components/schemas/Study_metrics" } } }, "Study_task": { "type": "object", "additionalProperties": false, "required": [ "task_id", "task_fingerprint", "trial_ids" ], "properties": { "task_id": { "$ref": "#/components/schemas/Study_id" }, "task_fingerprint": { "$ref": "#/components/schemas/Study_fingerprint" }, "trial_ids": { "type": "array", "minItems": 1, "maxItems": 1000, "uniqueItems": true, "items": { "$ref": "#/components/schemas/Study_id" } } } } } } }