> ## Documentation Index
> Fetch the complete documentation index at: https://evalgate.com/docs/llms.txt
> Use this file to discover all available pages before exploring further.

# Create scorers scorers

> Requires scopes: eval:write



## OpenAPI

````yaml /api-reference/openapi.json post /api/scorers
openapi: 3.1.0
info:
  title: EvalGate API
  version: 3.7.6
  description: EvalGate API. See docs/api-contract.md for stability commitments.
servers:
  - url: https://evalgate.com
    description: Production
  - url: http://localhost:3000
    description: Local
security: []
paths:
  /api/scorers:
    post:
      tags:
        - scorers
      summary: Create scorers scorers
      description: 'Requires scopes: eval:write'
      operationId: post_scorers
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/PostApiScorersRequest'
      responses:
        '201':
          description: Successful response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/PostApiScorersResponse201ApplicationJson'
        '400':
          $ref: '#/components/responses/ValidationError'
        '401':
          $ref: '#/components/responses/Unauthorized'
        '403':
          $ref: '#/components/responses/Forbidden'
        '404':
          $ref: '#/components/responses/NotFound'
        '409':
          $ref: '#/components/responses/Conflict'
        '429':
          $ref: '#/components/responses/RateLimited'
        '500':
          $ref: '#/components/responses/InternalError'
      security:
        - bearerAuth:
            - eval:write
components:
  schemas:
    PostApiScorersRequest:
      type: object
      properties:
        name:
          type: string
        definition:
          type: object
          properties:
            description:
              type: string
            implementation:
              oneOf:
                - type: object
                  properties:
                    type:
                      type: string
                      enum:
                        - deterministic
                    config:
                      type: object
                      properties:
                        schema:
                          type: object
                          additionalProperties:
                            oneOf:
                              - type: 'null'
                              - type: string
                              - type: number
                              - type: boolean
                                enum:
                                  - false
                              - type: boolean
                                enum:
                                  - true
                              - type: array
                                items:
                                  $ref: '#/components/schemas/JsonValue'
                              - type: object
                                additionalProperties:
                                  $ref: '#/components/schemas/JsonValue'
                        maxCostUsd:
                          type: number
                        pattern:
                          type: string
                        caseSensitive:
                          oneOf:
                            - type: boolean
                              enum:
                                - false
                            - type: boolean
                              enum:
                                - true
                        maxLatencyMs:
                          type: number
                      additionalProperties: false
                    operator:
                      oneOf:
                        - type: string
                          enum:
                            - contains
                        - type: string
                          enum:
                            - cost
                        - type: string
                          enum:
                            - latency
                        - type: string
                          enum:
                            - json_schema
                        - type: string
                          enum:
                            - regex
                        - type: string
                          enum:
                            - exact_match
                  additionalProperties: false
                  required:
                    - type
                    - config
                    - operator
                - type: object
                  properties:
                    code:
                      type: string
                    type:
                      type: string
                      enum:
                        - code
                    entrypoint:
                      type: string
                      enum:
                        - score
                    language:
                      oneOf:
                        - type: string
                          enum:
                            - python
                        - type: string
                          enum:
                            - typescript
                    limits:
                      type: object
                      properties:
                        maxOutputBytes:
                          type: number
                        memoryMb:
                          type: number
                        timeoutMs:
                          type: number
                        cpuMs:
                          type: number
                      additionalProperties: false
                      required:
                        - maxOutputBytes
                        - memoryMb
                        - timeoutMs
                        - cpuMs
                    allowNetwork:
                      type: boolean
                      enum:
                        - false
                  additionalProperties: false
                  required:
                    - code
                    - type
                    - entrypoint
                    - language
                    - limits
                    - allowNetwork
                - type: object
                  properties:
                    type:
                      type: string
                      enum:
                        - composite
                    aggregation:
                      oneOf:
                        - type: string
                          enum:
                            - minimum
                        - type: string
                          enum:
                            - maximum
                        - type: string
                          enum:
                            - any
                        - type: string
                          enum:
                            - all
                        - type: string
                          enum:
                            - weighted_average
                    components:
                      type: array
                      items:
                        type: object
                        properties:
                          contentHash:
                            type: string
                          scorerVersionId:
                            type: number
                          weight:
                            type: number
                          required:
                            type: boolean
                          scorerArtifactId:
                            type: number
                        additionalProperties: false
                        required:
                          - contentHash
                          - scorerVersionId
                          - weight
                          - required
                          - scorerArtifactId
                  additionalProperties: false
                  required:
                    - type
                    - aggregation
                    - components
                - type: object
                  properties:
                    type:
                      type: string
                      enum:
                        - llm_judge
                    model:
                      type: object
                      properties:
                        provider:
                          type: string
                        model:
                          type: string
                        routingKey:
                          type: string
                      additionalProperties: false
                    temperature:
                      type: number
                      enum:
                        - 0
                    judgePrompt:
                      type: string
                    rubric:
                      type: string
                    responseSchema:
                      type: object
                      additionalProperties:
                        oneOf:
                          - type: 'null'
                          - type: string
                          - type: number
                          - type: boolean
                            enum:
                              - false
                          - type: boolean
                            enum:
                              - true
                          - type: array
                            items:
                              $ref: '#/components/schemas/JsonValue'
                          - type: object
                            additionalProperties:
                              $ref: '#/components/schemas/JsonValue'
                    judgeConfigId:
                      type: number
                    promptBinding:
                      type: object
                      properties:
                        contentHash:
                          type: string
                        promptVersionId:
                          type: number
                        promptArtifactId:
                          type: number
                      additionalProperties: false
                      required:
                        - contentHash
                        - promptVersionId
                        - promptArtifactId
                  additionalProperties: false
                  required:
                    - type
                    - model
                    - temperature
                    - judgePrompt
                    - rubric
                    - responseSchema
            outputSchema:
              type: object
              additionalProperties:
                oneOf:
                  - type: 'null'
                  - type: string
                  - type: number
                  - type: boolean
                    enum:
                      - false
                  - type: boolean
                    enum:
                      - true
                  - type: array
                    items:
                      $ref: '#/components/schemas/JsonValue'
                  - type: object
                    additionalProperties:
                      $ref: '#/components/schemas/JsonValue'
            inputSchema:
              type: object
              additionalProperties:
                oneOf:
                  - type: 'null'
                  - type: string
                  - type: number
                  - type: boolean
                    enum:
                      - false
                  - type: boolean
                    enum:
                      - true
                  - type: array
                    items:
                      $ref: '#/components/schemas/JsonValue'
                  - type: object
                    additionalProperties:
                      $ref: '#/components/schemas/JsonValue'
            calibration:
              type: object
              properties:
                ownerId:
                  type: string
                requiredForPublishing:
                  oneOf:
                    - type: boolean
                      enum:
                        - false
                    - type: boolean
                      enum:
                        - true
                minHumanAnchors:
                  type: number
                minAgreementRate:
                  type: number
                maxParseFailureRate:
                  type: number
                requireApprovedMapping:
                  oneOf:
                    - type: boolean
                      enum:
                        - false
                    - type: boolean
                      enum:
                        - true
              additionalProperties: false
            scoreScale:
              type: object
              properties:
                max:
                  type: number
                min:
                  type: number
                passThreshold:
                  type: number
                labels:
                  type: object
                  additionalProperties:
                    type: number
              additionalProperties: false
              required:
                - max
                - min
                - passThreshold
            tags:
              type: array
              items:
                type: string
          additionalProperties: false
          required:
            - description
            - implementation
            - outputSchema
            - inputSchema
            - calibration
            - scoreScale
        key:
          type: string
        description:
          type: string
        idempotencyKey:
          type: string
      additionalProperties: false
      required:
        - name
        - definition
        - key
    PostApiScorersResponse201ApplicationJson:
      type: object
      properties:
        artifactId:
          type: number
        key:
          type: string
        name:
          type: string
        version:
          type: object
          properties:
            createdAt:
              type: string
            submittedAt:
              oneOf:
                - type: 'null'
                - type: string
            approvedAt:
              oneOf:
                - type: 'null'
                - type: string
            definition:
              type: object
              properties:
                description:
                  type: string
                inputSchema:
                  type: object
                  properties:
                    type:
                      oneOf:
                        - type: string
                        - type: array
                          items:
                            type: string
                    properties:
                      type: object
                      additionalProperties:
                        $ref: '#/components/schemas/JsonValue'
                    required:
                      type: array
                      items:
                        type: string
                    additionalProperties:
                      oneOf:
                        - type: boolean
                          enum:
                            - false
                        - type: boolean
                          enum:
                            - true
                        - $ref: '#/components/schemas/JsonValue'
                    items:
                      $ref: '#/components/schemas/JsonValue'
                    enum:
                      type: array
                      items:
                        oneOf:
                          - type: 'null'
                          - type: string
                          - type: number
                          - type: boolean
                            enum:
                              - false
                          - type: boolean
                            enum:
                              - true
                          - type: array
                            items:
                              $ref: '#/components/schemas/JsonValue'
                          - type: object
                            additionalProperties:
                              $ref: '#/components/schemas/JsonValue'
                  additionalProperties: false
                outputSchema:
                  type: object
                  properties:
                    type:
                      oneOf:
                        - type: string
                        - type: array
                          items:
                            type: string
                    properties:
                      type: object
                      additionalProperties:
                        $ref: '#/components/schemas/JsonValue'
                    required:
                      type: array
                      items:
                        type: string
                    additionalProperties:
                      oneOf:
                        - type: boolean
                          enum:
                            - false
                        - type: boolean
                          enum:
                            - true
                        - $ref: '#/components/schemas/JsonValue'
                    items:
                      $ref: '#/components/schemas/JsonValue'
                    enum:
                      type: array
                      items:
                        oneOf:
                          - type: 'null'
                          - type: string
                          - type: number
                          - type: boolean
                            enum:
                              - false
                          - type: boolean
                            enum:
                              - true
                          - type: array
                            items:
                              $ref: '#/components/schemas/JsonValue'
                          - type: object
                            additionalProperties:
                              $ref: '#/components/schemas/JsonValue'
                  additionalProperties: false
                scoreScale:
                  type: object
                  properties:
                    min:
                      type: number
                    max:
                      type: number
                    passThreshold:
                      type: number
                    labels:
                      type: object
                      additionalProperties:
                        type: number
                  additionalProperties: false
                  required:
                    - min
                    - max
                    - passThreshold
                implementation:
                  oneOf:
                    - type: object
                      properties:
                        type:
                          type: string
                          enum:
                            - deterministic
                        operator:
                          oneOf:
                            - type: string
                              enum:
                                - contains
                            - type: string
                              enum:
                                - cost
                            - type: string
                              enum:
                                - latency
                            - type: string
                              enum:
                                - json_schema
                            - type: string
                              enum:
                                - regex
                            - type: string
                              enum:
                                - exact_match
                        config:
                          type: object
                          properties:
                            caseSensitive:
                              oneOf:
                                - type: boolean
                                  enum:
                                    - false
                                - type: boolean
                                  enum:
                                    - true
                            pattern:
                              type: string
                            maxLatencyMs:
                              type: number
                            maxCostUsd:
                              type: number
                            schema:
                              type: object
                              properties:
                                type:
                                  oneOf:
                                    - type: string
                                    - type: array
                                      items:
                                        type: string
                                properties:
                                  type: object
                                  additionalProperties:
                                    $ref: '#/components/schemas/JsonValue'
                                required:
                                  type: array
                                  items:
                                    type: string
                                additionalProperties:
                                  oneOf:
                                    - type: boolean
                                      enum:
                                        - false
                                    - type: boolean
                                      enum:
                                        - true
                                    - $ref: '#/components/schemas/JsonValue'
                                items:
                                  $ref: '#/components/schemas/JsonValue'
                                enum:
                                  type: array
                                  items:
                                    oneOf:
                                      - type: 'null'
                                      - type: string
                                      - type: number
                                      - type: boolean
                                        enum:
                                          - false
                                      - type: boolean
                                        enum:
                                          - true
                                      - type: array
                                        items:
                                          $ref: '#/components/schemas/JsonValue'
                                      - type: object
                                        additionalProperties:
                                          $ref: '#/components/schemas/JsonValue'
                              additionalProperties: false
                          additionalProperties: false
                      additionalProperties: false
                      required:
                        - type
                        - operator
                        - config
                    - type: object
                      properties:
                        type:
                          type: string
                          enum:
                            - code
                        language:
                          oneOf:
                            - type: string
                              enum:
                                - python
                            - type: string
                              enum:
                                - typescript
                        code:
                          type: string
                        entrypoint:
                          type: string
                          enum:
                            - score
                        limits:
                          type: object
                          properties:
                            timeoutMs:
                              type: number
                            cpuMs:
                              type: number
                            memoryMb:
                              type: number
                            maxOutputBytes:
                              type: number
                          additionalProperties: false
                          required:
                            - timeoutMs
                            - cpuMs
                            - memoryMb
                            - maxOutputBytes
                        allowNetwork:
                          type: boolean
                          enum:
                            - false
                      additionalProperties: false
                      required:
                        - type
                        - language
                        - code
                        - entrypoint
                        - limits
                        - allowNetwork
                    - type: object
                      properties:
                        type:
                          type: string
                          enum:
                            - composite
                        aggregation:
                          oneOf:
                            - type: string
                              enum:
                                - minimum
                            - type: string
                              enum:
                                - maximum
                            - type: string
                              enum:
                                - any
                            - type: string
                              enum:
                                - all
                            - type: string
                              enum:
                                - weighted_average
                        components:
                          type: array
                          items:
                            type: object
                            properties:
                              scorerArtifactId:
                                type: number
                              scorerVersionId:
                                type: number
                              contentHash:
                                type: string
                              weight:
                                type: number
                              required:
                                type: boolean
                            additionalProperties: false
                            required:
                              - scorerArtifactId
                              - scorerVersionId
                              - contentHash
                              - weight
                              - required
                      additionalProperties: false
                      required:
                        - type
                        - aggregation
                        - components
                    - type: object
                      properties:
                        type:
                          type: string
                          enum:
                            - llm_judge
                        judgeConfigId:
                          type: number
                        rubric:
                          type: string
                        judgePrompt:
                          type: string
                        model:
                          type: object
                          properties:
                            provider:
                              type: string
                            model:
                              type: string
                            routingKey:
                              type: string
                          additionalProperties: false
                        promptBinding:
                          type: object
                          properties:
                            promptArtifactId:
                              type: number
                            promptVersionId:
                              type: number
                            contentHash:
                              type: string
                          additionalProperties: false
                          required:
                            - promptArtifactId
                            - promptVersionId
                            - contentHash
                        responseSchema:
                          type: object
                          properties:
                            type:
                              oneOf:
                                - type: string
                                - type: array
                                  items:
                                    type: string
                            properties:
                              type: object
                              additionalProperties:
                                $ref: '#/components/schemas/JsonValue'
                            required:
                              type: array
                              items:
                                type: string
                            additionalProperties:
                              oneOf:
                                - type: boolean
                                  enum:
                                    - false
                                - type: boolean
                                  enum:
                                    - true
                                - $ref: '#/components/schemas/JsonValue'
                            items:
                              $ref: '#/components/schemas/JsonValue'
                            enum:
                              type: array
                              items:
                                oneOf:
                                  - type: 'null'
                                  - type: string
                                  - type: number
                                  - type: boolean
                                    enum:
                                      - false
                                  - type: boolean
                                    enum:
                                      - true
                                  - type: array
                                    items:
                                      $ref: '#/components/schemas/JsonValue'
                                  - type: object
                                    additionalProperties:
                                      $ref: '#/components/schemas/JsonValue'
                          additionalProperties: false
                        temperature:
                          type: number
                          enum:
                            - 0
                      additionalProperties: false
                      required:
                        - type
                        - rubric
                        - judgePrompt
                        - model
                        - responseSchema
                        - temperature
                calibration:
                  type: object
                  properties:
                    requiredForPublishing:
                      type: boolean
                    minHumanAnchors:
                      type: number
                    minAgreementRate:
                      type: number
                    maxParseFailureRate:
                      type: number
                    requireApprovedMapping:
                      type: boolean
                    ownerId:
                      type: string
                  additionalProperties: false
                  required:
                    - requiredForPublishing
                    - minHumanAnchors
                    - minAgreementRate
                    - maxParseFailureRate
                    - requireApprovedMapping
                tags:
                  type: array
                  items:
                    type: string
              additionalProperties: false
              required:
                - description
                - inputSchema
                - outputSchema
                - scoreScale
                - implementation
                - calibration
                - tags
            versionId:
              type: number
            artifactId:
              type: number
            organizationId:
              $ref: '#/components/schemas/OrganizationId'
            versionNumber:
              type: number
            payload:
              type: object
              additionalProperties:
                $ref: '#/components/schemas/JsonValue'
            metadata:
              oneOf:
                - type: 'null'
                - type: object
                  additionalProperties:
                    $ref: '#/components/schemas/JsonValue'
            createdBy:
              type: string
            parentVersionId:
              oneOf:
                - type: 'null'
                - type: number
            contentHash:
              type: string
            packageHash:
              oneOf:
                - type: 'null'
                - type: string
            packageSignature:
              oneOf:
                - type: 'null'
                - type: string
            reviewStatus:
              type: string
            reviewRevision:
              type: number
            submittedBy:
              oneOf:
                - type: 'null'
                - type: string
            approvedBy:
              oneOf:
                - type: 'null'
                - type: string
          additionalProperties: false
          required:
            - createdAt
            - submittedAt
            - approvedAt
            - definition
            - versionId
            - artifactId
            - organizationId
            - versionNumber
            - payload
            - metadata
            - createdBy
            - parentVersionId
            - contentHash
            - packageHash
            - packageSignature
            - reviewStatus
            - reviewRevision
            - submittedBy
            - approvedBy
      additionalProperties: false
      required:
        - artifactId
        - key
        - name
        - version
    JsonValue:
      oneOf:
        - type: 'null'
        - type: boolean
        - type: number
        - type: string
        - type: array
          items:
            $ref: '#/components/schemas/JsonValue'
        - type: object
          additionalProperties:
            $ref: '#/components/schemas/JsonValue'
    OrganizationId:
      type: string
      format: uuid
    ApiError:
      type: object
      properties:
        error:
          type: object
          properties:
            code:
              type: string
              enum:
                - UNAUTHORIZED
                - FORBIDDEN
                - NOT_FOUND
                - VALIDATION_ERROR
                - RATE_LIMITED
                - CONFLICT
                - INTERNAL_ERROR
                - SERVICE_UNAVAILABLE
                - QUOTA_EXCEEDED
                - NO_ORG_MEMBERSHIP
                - CROSS_ORG_REFERENCE
                - INCOMPLETE
                - MALFORMED
            message:
              type: string
            details: {}
            requestId:
              type: string
              format: uuid
          required:
            - code
            - message
      required:
        - error
  responses:
    ValidationError:
      description: Validation error
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ApiError'
    Unauthorized:
      description: Unauthorized
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ApiError'
    Forbidden:
      description: Forbidden
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ApiError'
    NotFound:
      description: Resource not found
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ApiError'
    Conflict:
      description: Conflict
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ApiError'
    RateLimited:
      description: Rate limit exceeded
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ApiError'
    InternalError:
      description: Internal server error
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ApiError'
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer
      bearerFormat: API Key or Session Token

````