DocumentationRecipesReferenceGraphQLChangelog
Log In

Kili label taxonomy (JSON response)

jsonResponse is a JSON character string that contains the list of annotations. It varies for the different asset types and jobs used.

For detailed descriptions of specific jsonResponse examples, refer to these sections:

Classification

Check the detailed JSON property descriptions
  • categories: List of categories
    • name: Name of the category
    • confidence: Confidence (100 by default when done by human)

Examples:

'json_response': {
    "JOB_0": {
        "categories": [{"name": "YES_IT_IS_SPAM", "confidence": 100 }]
    }
}
'json_response': {
    "JOB_0": {
        "categories": [
            { "name": "BAD_PRODUCT", "confidence": 100 },
            { "name": "ILLEGAL_PRODUCT", "confidence": 100 }
        ]
    }
}
'json_response': {
    "JOB_0": {
        "categories": [{"name": "YES_IT_IS_A_NEWS_ARTICLE", "confidence": 100 }],
        "children": {
            "NESTED_JOB": {
                "categories": [{ "name": "SPORTS", "confidence": 100 }]
            }
        }
    }
}

Object Detection

Check the detailed JSON property descriptions
  • annotationsList of annotations
    • boundingPoly: Polygon of the object contour
      • normalizedVertices: List of vertices of the polygon. In the case of a bounding box, 4 vertices
        • x: Abscissa of the vertex position from top left and expressed as a percentage of the total image size
        • y: Ordinate of the vertex position from top left and expressed as a percentage of the total image size
    • categories: Category of the object
      • name: Name of the category
      • confidence: Confidence (100 by default when done by human)
    • mid: A unique identifier of the object
    • score: When a pre-annotation model is used, the score is the confidence of the object detection
    • type: Type of tool used for the annotation (one of: "rectangle", "polygon", or "semantic")

Examples:

'json_response': {
    "JOB_0": {
        "annotations": [{
            "boundingPoly": [{
                "normalizedVertices": [
                    { "x": 0.16, "y": 0.82},
                    { "x": 0.16, "y": 0.32 },
                    { "x": 0.82, "y": 0.32 },
                    { "x": 0.82, "y": 0.82 }
                ]}
            ],
            "categories": [{ "name": "TESLA", "confidence": 100 }],
            "mid": "unique-tesla",
            "type": "rectangle",
        }]
    }
}
'json_response': {
    "JOB": {
        "annotations": [{
            "boundingPoly": [{
                "normalizedVertices": [
                    { "x": 0.09, "y": 0.84 },
                    { "x": 0.09, "y": 0.36 },
                    { "x": 0.92, "y": 0.36 },
                    { "x": 0.92, "y": 0.84 }
                ]
            }],
            "categories": [{ "name": "FERRARI", "confidence": 100 }],
            "mid": "unique-ferrari",
            "type": "rectangle",
            "children": {
                "CLASSIFICATION_JOB": {
                    "categories": [{ "name": "GREY", "confidence": 100 }]
                }
            }
        }]
    }
}
'json_response': {
    "JOB_0": {
        "annotations": [{
            "boundingPoly": [{
                "normalizedVertices": [
                    { "x": 0.16, "y": 0.82 },
                    { "x": 0.16, "y": 0.32 },
                    { "x": 0.82, "y": 0.32 },
                    { "x": 0.82, "y": 0.82 }
                ]}
            ],
            "mid": "car",
            "type": "rectangle",
            "categories": [{ "name": "WHOLE_CAR", "confidence": 100 }],
        },
        {
            "boundingPoly": [{
                "normalizedVertices": [
                    { "x": 0.54, "y": 0.59 },
                    { "x": 0.43, "y": 0.59 },
                    { "x": 0.43, "y": 0.83 },
                    { "x": 0.54, "y": 0.83 }
                ]}
            ],
            "mid": "left-front-wheel",
            "type": "rectangle",
            "categories": [{ "name": "WHEELS", "confidence": 100 }],
        },
        {
            "boundingPoly": [{
                "normalizedVertices": [
                    { "x": 0.81, "y": 0.57 },
                    { "x": 0.74, "y": 0.57 },
                    { "x": 0.74, "y": 0.77 },
                    { "x": 0.81, "y": 0.77 }
                ]}
            ],
            "mid": "left-back-wheel",
            "type": "rectangle",
            "categories": [{ "name": "WHEELS", "confidence": 100 }],
        }]
    },
    'RELATION_JOB': {
        'annotations': [
            {
                'categories': [{'name': 'WHOLE_CAR_AND_WHEELS', 'confidence': 100}],
                'startObjects': [{'mid': 'car'}],
                'endObjects': [{'mid': 'left-front-wheel'}, {'mid': 'left-back-wheel'}],
            },
        ]
    },
}
'json_response': {
    "JOB_0": {
        "annotations": [{
            "boundingPoly": [{
                "normalizedVertices": [
                    { "x": 0.16, "y": 0.52},
                    { "x": 0.16, "y": 0.76 },
                    { "x": 0.49, "y": 0.83 },
                    { "x": 0.82, "y": 0.76 },
                    { "x": 0.82, "y": 0.49 },
                    { "x": 0.70, "y": 0.32 },
                    { "x": 0.48, "y": 0.32 },
                ]}
            ],
            "mid": "unique-tesla",
            "type": "polygon",
            "categories": [{ "name": "TESLA", "confidence": 100 }],
        }]
    }
}
'json_response': {
    "JOB_0": {
        "annotations": [{
            "boundingPoly": [{
                "normalizedVertices": [
                    { "x": 0.16, "y": 0.52},
                    { "x": 0.16, "y": 0.76 },
                    { "x": 0.49, "y": 0.83 },
                    { "x": 0.82, "y": 0.76 },
                    { "x": 0.82, "y": 0.49 },
                    { "x": 0.70, "y": 0.32 },
                    { "x": 0.48, "y": 0.32 },
                ]}
            ],
            "mid": "unique-tesla",
            "type": "semantic",
            "categories": [{ "name": "TESLA", "confidence": 100 }],
        }]
    }
}

Object detection in video assets

📘

In object detection jobs, native video and frame assets use an additional isKeyFrame property. This is a Boolean indicating if the timestamp or frame is used for interpolation.

Examples:

"jsonResponse": {
  "0": { //Timestamp number
    "job_0": {
      "annotations": [
        {
          "boundingPoly": [
            {
              "normalizedVertices": [
                { "x": 0.24283568900708732, "y": 0.5538364851214209 },
                { "x": 0.24283568900708732, "y": 0.3020356974943339 },
                { "x": 0.3729654281518853, "y": 0.3020356974943339 },
                { "x": 0.3729654281518853, "y": 0.5538364851214209 }
              ]
            }
          ],
          "categories": [{ "confidence": 100, "name": "BIG" }],
          "isKeyFrame": true,
          "mid": "2020110316040863-98540",
          "score": null,
          "type": "rectangle"
        }
    }
  }
}
"jsonResponse": {
  "0": { //Frame number
    "job_0": {
      "annotations": [
        {
          "boundingPoly": [
            {
              "normalizedVertices": [
                { "x": 0.24283568900708732, "y": 0.5538364851214209 },
                { "x": 0.24283568900708732, "y": 0.3020356974943339 },
                { "x": 0.3729654281518853, "y": 0.3020356974943339 },
                { "x": 0.3729654281518853, "y": 0.5538364851214209 }
              ]
            }
          ],
          "categories": [{ "confidence": 100, "name": "BIG" }],
          "isKeyFrame": true,
          "mid": "2020110316040863-98540",
          "score": null,
          "type": "rectangle"
        }
    }
  }
}

Named entity recognition (NER)

Check the detailed JSON property descriptions
  • annotationsList of annotations
    • beginOffset: Position of the entity mention
    • categories: Category of the object
      • name: Name of the category
      • confidence: Confidence (100 by default when done by human)
    • content: Content of the mention
    • mid: A unique identifier of the object

Examples:

'json_response': {
    "JOB": {
        "annotations": [
            {"categories": [{ "name": "NAME", "confidence": 100 }],
            "beginOffset": 0,
            "content": "Chicago Bulls",
            "mid": "chicago"},
            {"categories": [{ "name": "NAME", "confidence": 100 }],
            "beginOffset": 30,
            "content": "Jerry Krause",
            "mid": "krause"},
            {"categories": [{ "name": "NAME", "confidence": 100 }],
            "beginOffset": 63,
            "content": "Gatorade",
            "mid": "gatorade"},
            {"categories": [{ "name": "NOUN", "confidence": 100 }],
            "beginOffset": 22,
            "content": "Manager",
            "mid": "manager"},
            {"categories": [{ "name": "NOUN", "confidence": 100 }],
            "beginOffset": 14,
            "content": "General",
            "mid": "general"},
            {"categories": [{ "name": "NOUN", "confidence": 100 }],
            "beginOffset": 84,
            "content": "medicine",
            "mid": "medicine"},
            {"categories": [{ "name": "NOUN", "confidence": 100 }],
            "beginOffset": 104,
            "content": "players",
            "mid": "players"},
            {"categories": [{ "name": "NOUN", "confidence": 100 }],
            "beginOffset": 116,
            "content": "coaches",
            "mid": "coaches"},
            {"categories": [{ "name": "VERB", "confidence": 100 }],
            "beginOffset": 124,
            "content": "milled",
            "mid": "milled"},
            {"categories": [{ "name": "VERB", "confidence": 100 }],
            "beginOffset": 43,
            "content": "was standing",
            "mid": "standing"},
            {"categories": [{ "name": "NOUN", "confidence": 100 }],
            "beginOffset": 96,
            "content": "hand",
            "mid": "hand"},
            {"categories": [{ "name": "NOUN", "confidence": 100 }],
            "beginOffset": 72,
            "content": "cooler",
            "mid": "cooler"}
        ]
    }
}
'json_response': {
    "JOB": {
        "annotations": [
            {
                "beginId": 'PART1',
                "beginOffset": 14,
                "categories": [{ "name": "NOMINAL_GROUP", "confidence": 100 }],
                "content": "Declaration of the thirteen United States of America",
                "endId": 'PART2',
                "endOffset": 41,
                "mid": "declaration",
            },
        ]
    }
}
'json_response': {
    "NAMED_ENTITIES_RECOGNITION_JOB": {
        "annotations": [
            {"categories": [{ "name": "SUBJECT", "confidence": 100 }],
            "beginOffset": 0,
            "content": "Jordan",
            "mid": "Jordan"},
            {"categories": [{ "name": "VERB", "confidence": 100 }],
            "beginOffset": 84,
            "content": "was wearing",
            "mid": "wearing"},
            {"categories": [{ "name": "VERB", "confidence": 100 }],
            "beginOffset": 111,
            "content": "tucked",
            "mid": "tucked verb"},
            {"categories": [{ "name": "COMPLEMENT", "confidence": 100 }],
            "beginOffset": 96,
            "content": "a blue sweater tucked into high-rise pants",
            "mid": "blue sweater complement"},
            {"categories": [{ "name": "VERB", "confidence": 100 }],
            "beginOffset": 47,
            "content": "peered down",
            "mid": "peered"},
            {"categories": [{ "name": "COMPLEMENT", "confidence": 100 }],
            "beginOffset": 62,
            "content": "the hefty Krause",
            "mid": "Krause complement"},
            {"categories": [{ "name": "SUBJECT", "confidence": 100 }],
            "beginOffset": 62,
            "content": "the hefty Krause",
            "mid": "Krause subject"}
        ]
    },
    "NAMED_ENTITIES_RELATION_JOB": {
        "annotations": [
            {"categories": [{ "name": "VERB_AND_SUBJECT_S", "confidence": 100 }],
            "startEntities": [{ "mid": "peered" }],
            "endEntities": [{ "mid": "Jordan" }]},
            {"categories": [{ "name": "VERB_AND_COMPLEMENT_S", "confidence": 100 }],
            "startEntities": [{ "mid": "peered" }],
            "endEntities": [{ "mid": "Krause complement" }]},
            {"categories": [{ "name": "VERB_AND_SUBJECT_S", "confidence": 100 }],
            "startEntities": [{ "mid": "wearing" }],
            "endEntities": [{ "mid": "Krause subject" }]},
            {"categories": [{ "name": "VERB_AND_COMPLEMENT_S", "confidence": 100 }],
            "startEntities": [{ "mid": "wearing" }],
            "endEntities": [{ "mid": "blue sweater complement" }]}
        ]
    }
}

PDF

📘

Annotation structure for NER in PDF is different. Instead of beginOffset, the annotations work with the coordinates of the polygon that the data belongs to and the page number.

Check the detailed JSON property descriptions
  • annotations: List of annotations
    • annotations: List of positions of the annotation (for NER, when an annotation spans multiple lines, there will be multiple polys and a single boundingPoly)
      • boundingPoly: Polygon of the object contour
        • normalizedVertices: List of vertices of the polygon. In the case of a bounding box, 4 vertices
          • x: Abscissa of the vertex position from top left and expressed as a percentage of the total image size
          • y: Ordinate of the vertex position from top left and expressed as a percentage of the total image size
      • pageNumberArray: The pages where the annotation appears
      • polys: Coordinates from the different rectangles in the annotation. An annotation can have several rectangles (for example if the annotation covers more than one line)
    • categories: Category of the object
      • name: Name of the category
      • confidence: Confidence (100 by default when done by human)
    • mid: A unique identifier of the object

Example:

'json_response': {
            'NAMED_ENTITIES_RECOGNITION_JOB': {
                'annotations': [{
                    'annotations': [{
                        'boundingPoly': [{
                            'normalizedVertices': [[
                                {'x': 0.28, 'y': 0.12},
                                {'x': 0.28, 'y': 0.15},
                                {'x': 0.72, 'y': 0.12},
                                {'x': 0.72, 'y': 0.15}
                            ]]
                        }],
                        'polys': [{
                            'normalizedVertices': [[
                                {'x': 0.28, 'y': 0.12},
                                {'x': 0.28, 'y': 0.15},
                                {'x': 0.72, 'y': 0.12},
                                {'x': 0.72, 'y': 0.15}
                            ]]
                        }],
                        'pageNumber': 1
                    }],
                    'categories': [{'name': 'TITLE', 'confidence': 100}],
                    'content': 'Learning Active Learning from Data',
                    'mid': 'article-title'
                }],
            }
        }

Pose Estimation

Check the detailed JSON property descriptions
  • annotationsList of annotations
  • categories: Category of the object
    • name: Name of the category
    • confidence: Confidence (100 by default when done by human)
  • kind: Job kind. In pose estimation jobs, this is always "POSE_ESTIMATION"
  • mid: A unique identifier of the object
  • points: List of the points composing the object
    • categories: Category of the object which point belongs to
    • code: Identifier (unique for each point in an object)
    • jobName: The job which annotated point belongs to
    • mid: Id of the point
    • name: Name of the point
    • point: Coordinates of the point
      • x: Point abscissa
      • y: Point ordinate
  • type: Tool used to annotate the point. In pose estimation jobs, it's `marker`

Examples:

"jsonResponse": {
                "JOB_0": {
                    "annotations": [
                        {
                            "categories": [
                                {
                                    "confidence": 100,
                                    "name": "POSE_A"
                                }
                            ],
                            "children": {},
                            "jobName": "JOB_0",
                            "kind": "POSE_ESTIMATION",
                            "mid": "20220511111820SS-508",
                            "points": [
                                {
                                    "categories": [
                                        {
                                            "confidence": 100,
                                            "name": "POSE_A"
                                        }
                                    ],
                                    "children": {},
                                    "code": "POINT_A1",
                                    "jobName": "JOB_0",
                                    "mid": "20220511111820SS-508",
                                    "name": "Point A1",
                                    "point": {
                                        "x": 0.3817781479042206,
                                        "y": 0.10908308099784103
                                    }
                                },
                                {
                                    "categories": [
                                        {
                                            "confidence": 100,
                                            "name": "POSE_A"
                                        }
                                    ],
                                    "children": {},
                                    "code": "POINT_A2",
                                    "jobName": "JOB_0",
                                    "mid": "20220511111820SS-508",
                                    "name": "Point A2",
                                    "point": {
                                        "x": 0.31799659574838424,
                                        "y": 0.3245229905019995
                                    }
                                },
                                {
                                    "categories": [
                                        {
                                            "confidence": 100,
                                            "name": "POSE_A"
                                        }
                                    ],
                                    "children": {},
                                    "code": "POINT_A3",
                                    "jobName": "JOB_0",
                                    "mid": "20220511111820SS-508",
                                    "name": "Point A3",
                                    "point": {
                                        "x": 0.5129859123390841,
                                        "y": 0.3569199693748055
                                    }
                                },
                                {
                                    "categories": [
                                        {
                                            "confidence": 100,
                                            "name": "POSE_A"
                                        }
                                    ],
                                    "children": {},
                                    "code": "POINT_A4",
                                    "jobName": "JOB_0",
                                    "mid": "20220511111820SS-508",
                                    "name": "Point A4",
                                    "point": {
                                        "x": 0.5439655233862045,
                                        "y": 0.12366172149060362
                                    }
                                }
                            ],
                            "type": "marker"
                        }
                    ]
                }
            }
"jsonResponse": {
                "JOB_0": {
                    "annotations": [
                        {
                            "categories": [
                                {
                                    "confidence": 100,
                                    "name": "POSE_A"
                                }
                            ],
                            "children": {},
                            "jobName": "JOB_0",
                            "kind": "POSE_ESTIMATION",
                            "mid": "20220511112452SS-21114",
                            "points": [
                                {
                                    "categories": [
                                        {
                                            "confidence": 100,
                                            "name": "POSE_A"
                                        }
                                    ],
                                    "children": {
                                        "CLASSIFICATION_JOB": {
                                            "categories": [
                                                {
                                                    "confidence": 100,
                                                    "name": "CATEGORY_1"
                                                }
                                            ]
                                        }
                                    },
                                    "code": "POINT_A1",
                                    "jobName": "JOB_0",
                                    "mid": "20220511112452SS-21114",
                                    "name": "Point A1",
                                    "point": {
                                        "x": 0.4853515625,
                                        "y": 0.203125
                                    }
                                },
                                {
                                    "categories": [
                                        {
                                            "confidence": 100,
                                            "name": "POSE_A"
                                        }
                                    ],
                                    "children": {
                                        "CLASSIFICATION_JOB_0": {
                                            "categories": [
                                                {
                                                    "confidence": 100,
                                                    "name": "CATEGORY_2"
                                                }
                                            ]
                                        }
                                    },
                                    "code": "POINT_A2",
                                    "jobName": "JOB_0",
                                    "mid": "20220511112452SS-21114",
                                    "name": "Point A2",
                                    "point": {
                                        "x": 0.400390625,
                                        "y": 0.43229166666666685
                                    }
                                },
                                {
                                    "categories": [
                                        {
                                            "confidence": 100,
                                            "name": "POSE_A"
                                        }
                                    ],
                                    "children": {
                                        "CLASSIFICATION_JOB_1": {
                                            "categories": [
                                                {
                                                    "confidence": 100,
                                                    "name": "CATEGORY_3"
                                                }
                                            ]
                                        }
                                    },
                                    "code": "POINT_A3",
                                    "jobName": "JOB_0",
                                    "mid": "20220511112452SS-21114",
                                    "name": "Point A3",
                                    "point": {
                                        "x": 0.625,
                                        "y": 0.4878472222222222
                                    }
                                },
                                {
                                    "categories": [
                                        {
                                            "confidence": 100,
                                            "name": "POSE_A"
                                        }
                                    ],
                                    "children": {
                                        "CLASSIFICATION_JOB_2": {
                                            "categories": [
                                                {
                                                    "confidence": 100,
                                                    "name": "CATEGORY_4"
                                                }
                                            ]
                                        }
                                    },
                                    "code": "POINT_A4",
                                    "jobName": "JOB_0",
                                    "mid": "20220511112452SS-21114",
                                    "name": "Point A4",
                                    "point": {
                                        "x": 0.685546875,
                                        "y": 0.22048611111111116
                                    }
                                }
                            ],
                            "type": "marker"
                        }
                    ]
                }
            }

Transcription

Check the detailed JSON property descriptions
  • annotationsList of annotations
    • boundingPoly: Polygon of the object contour
      • normalizedVertices: List of vertices of the polygon. In the case of a bounding box, 4 vertices
        • x: Abscissa of the vertex position from top left and expressed as a percentage of the total image size
        • y: Ordinate of the vertex position from top left and expressed as a percentage of the total image size
    • children: Job containing the text recognized by the OCR
    • categories: Category of the object
      • name: Name of the category
      • confidence: Confidence (100 by default when done by human)
    • mid: A unique identifier of the object
    • score: When a pre-annotation model is used, the score is the confidence of the object detection

Examples:

'json_response': {
            'JOB_0': {
                'annotations': [{
                    'boundingPoly': [{
                        'normalizedVertices': [
                            {'x': 0.47, 'y': 0.53},
                            {'x': 0.47, 'y': 0.48},
                            {'x': 0.65, 'y': 0.48},
                            {'x': 0.65, 'y': 0.53}
                        ]
                    }],
                    'categories': [{'name': 'NATIONALITY', 'confidence': 100}],
                    'mid': 'nationality',
                    'type': 'rectangle',
                    'children': {'JOB_1': {'text': 'Nederlandse'}}
                },
                {
                    'boundingPoly': [{
                        'normalizedVertices': [
                            {'x': 0.36, 'y': 0.37},
                            {'x': 0.36, 'y': 0.32},
                            {'x': 0.51, 'y': 0.32},
                            {'x': 0.51, 'y': 0.37}
                        ]
                    }],
                    'categories': [{'name': 'NAME', 'confidence': 100}],
                    'mid': 'name',
                    'type': 'rectangle',
                    'children': {'JOB_2': {'text': 'De Bruijn'}}
                }]
            }
        }
'json_response': {
            'JOB_0': {
                'annotations': [{
                    'boundingPoly': [{
                        'normalizedVertices': [
                            {'x': 0.47, 'y': 0.53},
                            {'x': 0.47, 'y': 0.48},
                            {'x': 0.65, 'y': 0.48},
                            {'x': 0.65, 'y': 0.53}
                        ]
                    }],
                    'categories': [{'name': 'NATIONALITY', 'confidence': 100}],
                    'type': 'rectangle',
                    'mid': 'netherlands',
                    'children': {'TRANSCRIPTION_JOB_1': {'text': 'Nederlandse'}}
                },
                {
                    'boundingPoly': [{
                        'normalizedVertices': [
                            {'x': 0.36, 'y': 0.37},
                            {'x': 0.36, 'y': 0.32},
                            {'x': 0.51, 'y': 0.32},
                            {'x': 0.51, 'y': 0.37}
                        ]
                    }],
                    'categories': [{'name': 'NAME', 'confidence': 100}],
                    'type': 'rectangle',
                    'mid': 'bruijn',
                    'children': {'TRANSCRIPTION_JOB_2': {'text': 'De Bruijn'}}
                },
                {
                    'boundingPoly': [{
                        'normalizedVertices': [
                            {'x': 0.36, 'y': 0.61},
                            {'x': 0.36, 'y': 0.56},
                            {'x': 0.63, 'y': 0.56},
                            {'x': 0.63, 'y': 0.61}
                        ]
                    }],
                    'categories': [{'name': 'BIRTH_DATE', 'confidence': 100}],
                    'type': 'rectangle',
                    'mid': 'date',
                    'children': {'TRANSCRIPTION_JOB_3': {'text': '10 MAA/MAR 1965'}}
                }]
            },
            "OCR_RELATION_JOB": {
                "annotations": [
                    {"categories": [{ "name": "NAME_AND_NATIONALITY", "confidence": 100 }],
                    "startObjects": [{ "mid": "bruijn" }],
                    "endObjects": [{ "mid": "netherlands" }]},
                    {"categories": [{ "name": "NAME_AND_BIRTH_DATE", "confidence": 100 }],
                    "startObjects": [{ "mid": "bruijn" }],
                    "endObjects": [{ "mid": "date" }]}
                ]
            }
        }

For examples of how you can use the jsonResponse string, refer to these recipes:


Did this page help you?