> ## Documentation Index > Fetch the complete documentation index at: https://evalgate.com/docs/llms.txt > Use this file to discover all available pages before exploring further. # Create scorers test > Requires scopes: runs:write ## OpenAPI ````yaml /api-reference/openapi.json post /api/scorers/{scorerId}/test openapi: 3.1.0 info: title: EvalGate API version: 3.7.6 description: EvalGate API. See docs/api-contract.md for stability commitments. servers: - url: https://evalgate.com description: Production - url: http://localhost:3000 description: Local security: [] paths: /api/scorers/{scorerId}/test: post: tags: - scorers summary: Create scorers test description: 'Requires scopes: runs:write' operationId: post_scorers_scorerId_test parameters: - name: scorerId in: path required: true schema: type: string requestBody: required: true content: application/json: schema: $ref: '#/components/schemas/PostApiScorersScorerIdTestRequest' responses: '200': description: Successful response content: application/json: schema: $ref: >- #/components/schemas/PostApiScorersScorerIdTestResponse200ApplicationJson '400': $ref: '#/components/responses/ValidationError' '401': $ref: '#/components/responses/Unauthorized' '403': $ref: '#/components/responses/Forbidden' '404': $ref: '#/components/responses/NotFound' '409': $ref: '#/components/responses/Conflict' '429': $ref: '#/components/responses/RateLimited' '500': $ref: '#/components/responses/InternalError' security: - bearerAuth: - runs:write components: schemas: PostApiScorersScorerIdTestRequest: type: object properties: input: type: object properties: input: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' output: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' expectedOutput: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - $ref: '#/components/schemas/JsonValue' context: type: object additionalProperties: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' latencyMs: type: number costUsd: type: number additionalProperties: false required: - input - output versionId: type: number resourceId: type: string idempotencyKey: type: string surface: oneOf: - type: string enum: - playground - type: string enum: - experiment - type: string enum: - online_eval - type: string enum: - gate additionalProperties: false required: - input - versionId - resourceId PostApiScorersScorerIdTestResponse200ApplicationJson: oneOf: - type: object properties: resultRecord: type: object properties: id: type: string organizationId: $ref: '#/components/schemas/OrganizationId' artifactId: type: number versionId: type: number contentHash: type: string surface: oneOf: - type: string enum: - playground - type: string enum: - experiment - type: string enum: - online_eval - type: string enum: - gate resourceId: type: string status: type: string rawScore: oneOf: - type: 'null' - type: string normalizedScore: oneOf: - type: 'null' - type: string passed: oneOf: - type: 'null' - type: boolean enum: - false - type: boolean enum: - true reasoning: oneOf: - type: 'null' - type: string evidence: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' logs: type: array items: type: string errorCode: oneOf: - type: 'null' - type: string errorMessage: oneOf: - type: 'null' - type: string latencyMs: type: number costUsd: oneOf: - type: 'null' - type: string modelCallId: oneOf: - type: 'null' - type: string comparabilityStatus: oneOf: - type: string enum: - not_comparable - type: string enum: - exact_comparable - type: string enum: - calibrated_comparable - type: string enum: - trend_discontinuity mappingVersionId: oneOf: - type: 'null' - type: string calibrationSetId: oneOf: - type: 'null' - type: string calibrationRunId: oneOf: - type: 'null' - type: string scoreMappingId: oneOf: - type: 'null' - type: string idempotencyKey: type: string createdAt: type: string format: date-time additionalProperties: false required: - id - organizationId - artifactId - versionId - contentHash - surface - resourceId - status - rawScore - normalizedScore - passed - reasoning - evidence - logs - errorCode - errorMessage - latencyMs - costUsd - modelCallId - comparabilityStatus - mappingVersionId - calibrationSetId - calibrationRunId - scoreMappingId - idempotencyKey - createdAt execution: type: object properties: status: type: string result: oneOf: - type: 'null' - type: object properties: score: type: number passed: type: boolean reasoning: type: string evidence: type: object additionalProperties: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' additionalProperties: false required: - score - passed - reasoning - evidence normalizedScore: oneOf: - type: 'null' - type: number comparabilityStatus: oneOf: - type: string enum: - not_comparable - type: string enum: - exact_comparable - type: string enum: - calibrated_comparable - type: string enum: - trend_discontinuity logs: type: array items: type: string latencyMs: type: number modelCallId: type: string costUsd: oneOf: - type: 'null' - type: string additionalProperties: false required: - status - result - normalizedScore - comparabilityStatus - logs - latencyMs - modelCallId - costUsd version: type: object properties: createdAt: type: string submittedAt: oneOf: - type: 'null' - type: string approvedAt: oneOf: - type: 'null' - type: string definition: type: object properties: description: type: string inputSchema: type: object properties: type: oneOf: - type: string - type: array items: type: string properties: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' required: type: array items: type: string additionalProperties: oneOf: - type: boolean enum: - false - type: boolean enum: - true - $ref: '#/components/schemas/JsonValue' items: $ref: '#/components/schemas/JsonValue' enum: type: array items: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' additionalProperties: false outputSchema: type: object properties: type: oneOf: - type: string - type: array items: type: string properties: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' required: type: array items: type: string additionalProperties: oneOf: - type: boolean enum: - false - type: boolean enum: - true - $ref: '#/components/schemas/JsonValue' items: $ref: '#/components/schemas/JsonValue' enum: type: array items: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' additionalProperties: false scoreScale: type: object properties: min: type: number max: type: number passThreshold: type: number labels: type: object additionalProperties: type: number additionalProperties: false required: - min - max - passThreshold implementation: oneOf: - type: object properties: type: type: string enum: - deterministic operator: oneOf: - type: string enum: - contains - type: string enum: - cost - type: string enum: - latency - type: string enum: - json_schema - type: string enum: - regex - type: string enum: - exact_match config: type: object properties: caseSensitive: oneOf: - type: boolean enum: - false - type: boolean enum: - true pattern: type: string maxLatencyMs: type: number maxCostUsd: type: number schema: type: object properties: type: oneOf: - type: string - type: array items: type: string properties: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' required: type: array items: type: string additionalProperties: oneOf: - type: boolean enum: - false - type: boolean enum: - true - $ref: '#/components/schemas/JsonValue' items: $ref: '#/components/schemas/JsonValue' enum: type: array items: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' additionalProperties: false additionalProperties: false additionalProperties: false required: - type - operator - config - type: object properties: type: type: string enum: - code language: oneOf: - type: string enum: - python - type: string enum: - typescript code: type: string entrypoint: type: string enum: - score limits: type: object properties: timeoutMs: type: number cpuMs: type: number memoryMb: type: number maxOutputBytes: type: number additionalProperties: false required: - timeoutMs - cpuMs - memoryMb - maxOutputBytes allowNetwork: type: boolean enum: - false additionalProperties: false required: - type - language - code - entrypoint - limits - allowNetwork - type: object properties: type: type: string enum: - composite aggregation: oneOf: - type: string enum: - minimum - type: string enum: - maximum - type: string enum: - any - type: string enum: - all - type: string enum: - weighted_average components: type: array items: type: object properties: scorerArtifactId: type: number scorerVersionId: type: number contentHash: type: string weight: type: number required: type: boolean additionalProperties: false required: - scorerArtifactId - scorerVersionId - contentHash - weight - required additionalProperties: false required: - type - aggregation - components - type: object properties: type: type: string enum: - llm_judge judgeConfigId: type: number rubric: type: string judgePrompt: type: string model: type: object properties: provider: type: string model: type: string routingKey: type: string additionalProperties: false promptBinding: type: object properties: promptArtifactId: type: number promptVersionId: type: number contentHash: type: string additionalProperties: false required: - promptArtifactId - promptVersionId - contentHash responseSchema: type: object properties: type: oneOf: - type: string - type: array items: type: string properties: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' required: type: array items: type: string additionalProperties: oneOf: - type: boolean enum: - false - type: boolean enum: - true - $ref: '#/components/schemas/JsonValue' items: $ref: '#/components/schemas/JsonValue' enum: type: array items: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' additionalProperties: false temperature: type: number enum: - 0 additionalProperties: false required: - type - rubric - judgePrompt - model - responseSchema - temperature calibration: type: object properties: requiredForPublishing: type: boolean minHumanAnchors: type: number minAgreementRate: type: number maxParseFailureRate: type: number requireApprovedMapping: type: boolean ownerId: type: string additionalProperties: false required: - requiredForPublishing - minHumanAnchors - minAgreementRate - maxParseFailureRate - requireApprovedMapping tags: type: array items: type: string additionalProperties: false required: - description - inputSchema - outputSchema - scoreScale - implementation - calibration - tags versionId: type: number artifactId: type: number organizationId: $ref: '#/components/schemas/OrganizationId' versionNumber: type: number payload: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' metadata: oneOf: - type: 'null' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' createdBy: type: string parentVersionId: oneOf: - type: 'null' - type: number contentHash: type: string packageHash: oneOf: - type: 'null' - type: string packageSignature: oneOf: - type: 'null' - type: string reviewStatus: type: string reviewRevision: type: number submittedBy: oneOf: - type: 'null' - type: string approvedBy: oneOf: - type: 'null' - type: string additionalProperties: false required: - createdAt - submittedAt - approvedAt - definition - versionId - artifactId - organizationId - versionNumber - payload - metadata - createdBy - parentVersionId - contentHash - packageHash - packageSignature - reviewStatus - reviewRevision - submittedBy - approvedBy idempotent: type: boolean additionalProperties: false required: - resultRecord - execution - version - idempotent - type: object properties: resultRecord: type: object properties: calibrationSetId: oneOf: - type: 'null' - type: string status: type: string id: type: string createdAt: type: string format: date-time organizationId: $ref: '#/components/schemas/OrganizationId' artifactId: type: number versionId: type: number contentHash: type: string evidence: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' idempotencyKey: type: string surface: oneOf: - type: string enum: - playground - type: string enum: - experiment - type: string enum: - online_eval - type: string enum: - gate costUsd: oneOf: - type: 'null' - type: string latencyMs: type: number errorCode: oneOf: - type: 'null' - type: string errorMessage: oneOf: - type: 'null' - type: string modelCallId: oneOf: - type: 'null' - type: string mappingVersionId: oneOf: - type: 'null' - type: string logs: type: array items: type: string reasoning: oneOf: - type: 'null' - type: string rawScore: oneOf: - type: 'null' - type: string normalizedScore: oneOf: - type: 'null' - type: string comparabilityStatus: oneOf: - type: string enum: - not_comparable - type: string enum: - exact_comparable - type: string enum: - calibrated_comparable - type: string enum: - trend_discontinuity calibrationRunId: oneOf: - type: 'null' - type: string scoreMappingId: oneOf: - type: 'null' - type: string resourceId: type: string passed: oneOf: - type: 'null' - type: boolean enum: - false - type: boolean enum: - true additionalProperties: false required: - calibrationSetId - status - id - createdAt - organizationId - artifactId - versionId - contentHash - evidence - idempotencyKey - surface - costUsd - latencyMs - errorCode - errorMessage - modelCallId - mappingVersionId - logs - reasoning - rawScore - normalizedScore - comparabilityStatus - calibrationRunId - scoreMappingId - resourceId - passed execution: type: object properties: status: oneOf: - type: string enum: - failed - type: string enum: - success - type: string enum: - blocked - type: string enum: - timeout result: oneOf: - type: 'null' - type: object properties: score: type: number passed: type: boolean reasoning: type: string evidence: type: object additionalProperties: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' additionalProperties: false required: - score - passed - reasoning - evidence logs: type: array items: type: string errorCode: type: string errorMessage: type: string latencyMs: type: number modelCallId: type: string costUsd: oneOf: - type: 'null' - type: string comparabilityStatus: oneOf: - type: string enum: - not_comparable - type: string enum: - exact_comparable - type: string enum: - calibrated_comparable - type: string enum: - trend_discontinuity normalizedScore: oneOf: - type: 'null' - type: number mappingVersionId: oneOf: - type: 'null' - type: string calibrationSetId: oneOf: - type: 'null' - type: string calibrationSetVersionId: oneOf: - type: 'null' - type: string calibrationRunId: oneOf: - type: 'null' - type: string scoreMappingId: oneOf: - type: 'null' - type: string additionalProperties: false required: - status - result - logs - latencyMs - comparabilityStatus - normalizedScore version: type: object properties: createdAt: type: string submittedAt: oneOf: - type: 'null' - type: string approvedAt: oneOf: - type: 'null' - type: string definition: type: object properties: description: type: string inputSchema: type: object properties: type: oneOf: - type: string - type: array items: type: string properties: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' required: type: array items: type: string additionalProperties: oneOf: - type: boolean enum: - false - type: boolean enum: - true - $ref: '#/components/schemas/JsonValue' items: $ref: '#/components/schemas/JsonValue' enum: type: array items: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' additionalProperties: false outputSchema: type: object properties: type: oneOf: - type: string - type: array items: type: string properties: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' required: type: array items: type: string additionalProperties: oneOf: - type: boolean enum: - false - type: boolean enum: - true - $ref: '#/components/schemas/JsonValue' items: $ref: '#/components/schemas/JsonValue' enum: type: array items: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' additionalProperties: false scoreScale: type: object properties: min: type: number max: type: number passThreshold: type: number labels: type: object additionalProperties: type: number additionalProperties: false required: - min - max - passThreshold implementation: oneOf: - type: object properties: type: type: string enum: - deterministic operator: oneOf: - type: string enum: - contains - type: string enum: - cost - type: string enum: - latency - type: string enum: - json_schema - type: string enum: - regex - type: string enum: - exact_match config: type: object properties: caseSensitive: oneOf: - type: boolean enum: - false - type: boolean enum: - true pattern: type: string maxLatencyMs: type: number maxCostUsd: type: number schema: type: object properties: type: oneOf: - type: string - type: array items: type: string properties: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' required: type: array items: type: string additionalProperties: oneOf: - type: boolean enum: - false - type: boolean enum: - true - $ref: '#/components/schemas/JsonValue' items: $ref: '#/components/schemas/JsonValue' enum: type: array items: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' additionalProperties: false additionalProperties: false additionalProperties: false required: - type - operator - config - type: object properties: type: type: string enum: - code language: oneOf: - type: string enum: - python - type: string enum: - typescript code: type: string entrypoint: type: string enum: - score limits: type: object properties: timeoutMs: type: number cpuMs: type: number memoryMb: type: number maxOutputBytes: type: number additionalProperties: false required: - timeoutMs - cpuMs - memoryMb - maxOutputBytes allowNetwork: type: boolean enum: - false additionalProperties: false required: - type - language - code - entrypoint - limits - allowNetwork - type: object properties: type: type: string enum: - composite aggregation: oneOf: - type: string enum: - minimum - type: string enum: - maximum - type: string enum: - any - type: string enum: - all - type: string enum: - weighted_average components: type: array items: type: object properties: scorerArtifactId: type: number scorerVersionId: type: number contentHash: type: string weight: type: number required: type: boolean additionalProperties: false required: - scorerArtifactId - scorerVersionId - contentHash - weight - required additionalProperties: false required: - type - aggregation - components - type: object properties: type: type: string enum: - llm_judge judgeConfigId: type: number rubric: type: string judgePrompt: type: string model: type: object properties: provider: type: string model: type: string routingKey: type: string additionalProperties: false promptBinding: type: object properties: promptArtifactId: type: number promptVersionId: type: number contentHash: type: string additionalProperties: false required: - promptArtifactId - promptVersionId - contentHash responseSchema: type: object properties: type: oneOf: - type: string - type: array items: type: string properties: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' required: type: array items: type: string additionalProperties: oneOf: - type: boolean enum: - false - type: boolean enum: - true - $ref: '#/components/schemas/JsonValue' items: $ref: '#/components/schemas/JsonValue' enum: type: array items: oneOf: - type: 'null' - type: string - type: number - type: boolean enum: - false - type: boolean enum: - true - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' additionalProperties: false temperature: type: number enum: - 0 additionalProperties: false required: - type - rubric - judgePrompt - model - responseSchema - temperature calibration: type: object properties: requiredForPublishing: type: boolean minHumanAnchors: type: number minAgreementRate: type: number maxParseFailureRate: type: number requireApprovedMapping: type: boolean ownerId: type: string additionalProperties: false required: - requiredForPublishing - minHumanAnchors - minAgreementRate - maxParseFailureRate - requireApprovedMapping tags: type: array items: type: string additionalProperties: false required: - description - inputSchema - outputSchema - scoreScale - implementation - calibration - tags versionId: type: number artifactId: type: number organizationId: $ref: '#/components/schemas/OrganizationId' versionNumber: type: number payload: type: object additionalProperties: $ref: '#/components/schemas/JsonValue' metadata: oneOf: - type: 'null' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' createdBy: type: string parentVersionId: oneOf: - type: 'null' - type: number contentHash: type: string packageHash: oneOf: - type: 'null' - type: string packageSignature: oneOf: - type: 'null' - type: string reviewStatus: type: string reviewRevision: type: number submittedBy: oneOf: - type: 'null' - type: string approvedBy: oneOf: - type: 'null' - type: string additionalProperties: false required: - createdAt - submittedAt - approvedAt - definition - versionId - artifactId - organizationId - versionNumber - payload - metadata - createdBy - parentVersionId - contentHash - packageHash - packageSignature - reviewStatus - reviewRevision - submittedBy - approvedBy idempotent: type: boolean additionalProperties: false required: - resultRecord - execution - version - idempotent JsonValue: oneOf: - type: 'null' - type: boolean - type: number - type: string - type: array items: $ref: '#/components/schemas/JsonValue' - type: object additionalProperties: $ref: '#/components/schemas/JsonValue' OrganizationId: type: string format: uuid ApiError: type: object properties: error: type: object properties: code: type: string enum: - UNAUTHORIZED - FORBIDDEN - NOT_FOUND - VALIDATION_ERROR - RATE_LIMITED - CONFLICT - INTERNAL_ERROR - SERVICE_UNAVAILABLE - QUOTA_EXCEEDED - NO_ORG_MEMBERSHIP - CROSS_ORG_REFERENCE - INCOMPLETE - MALFORMED message: type: string details: {} requestId: type: string format: uuid required: - code - message required: - error responses: ValidationError: description: Validation error content: application/json: schema: $ref: '#/components/schemas/ApiError' Unauthorized: description: Unauthorized content: application/json: schema: $ref: '#/components/schemas/ApiError' Forbidden: description: Forbidden content: application/json: schema: $ref: '#/components/schemas/ApiError' NotFound: description: Resource not found content: application/json: schema: $ref: '#/components/schemas/ApiError' Conflict: description: Conflict content: application/json: schema: $ref: '#/components/schemas/ApiError' RateLimited: description: Rate limit exceeded content: application/json: schema: $ref: '#/components/schemas/ApiError' InternalError: description: Internal server error content: application/json: schema: $ref: '#/components/schemas/ApiError' securitySchemes: bearerAuth: type: http scheme: bearer bearerFormat: API Key or Session Token ````