import os
from dataframer import Dataframer
client = Dataframer(
api_key=os.environ.get("DATAFRAMER_API_KEY"), # This is the default and can be omitted
)
evaluation = client.dataframer.judges.evaluations.create(
judge_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e",
messages=[{
"role": "system",
"content": "You are a support assistant.",
}, {
"role": "user",
"content": "Where is my refund?",
}],
output="Your refund was issued yesterday and should arrive within 3 business days.",
)
print(evaluation.id)curl https://df-api.dataframer.ai/api/dataframer/judges/$JUDGE_ID/evaluations/ \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $DATAFRAMER_API_KEY" \
-d '{
"messages": [
{
"role": "system",
"content": "You are a support assistant."
},
{
"role": "user",
"content": "Where is my refund?"
}
],
"output": "Your refund was issued yesterday and should arrive within 3 business days."
}'{
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"judge_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"judge_version": 123,
"source": "api",
"status": "succeeded",
"model": "<string>",
"verdict": {
"overall_score": 123,
"confidence": 123,
"reasoning": "<string>",
"dimensions": [
{
"dimension_id": "<string>",
"name": "<string>",
"scoring_model": "pass_fail",
"label": "<string>",
"labels": [
"<string>"
],
"score": 123,
"passed": true,
"notes": "<string>"
}
]
},
"error": "<string>",
"latency_ms": 123,
"created_at": "2023-11-07T05:31:56Z",
"content_location": {
"data_plane": true,
"s3_key": "<string>",
"storage_uri": "<string>",
"sha256": "<string>",
"bytes_size": 123,
"version": 123,
"fields": {}
},
"content_unavailable_reason": "result_content_in_data_plane"
}{
"error": "<string>"
}{
"error": "<string>"
}{
"error": "<string>"
}{
"error": "<string>"
}{
"error": "<string>"
}{
"error": "<string>",
"code": "not_supported_hybrid"
}{
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"judge_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"judge_version": 123,
"source": "api",
"status": "succeeded",
"model": "<string>",
"verdict": {
"overall_score": 123,
"confidence": 123,
"reasoning": "<string>",
"dimensions": [
{
"dimension_id": "<string>",
"name": "<string>",
"scoring_model": "pass_fail",
"label": "<string>",
"labels": [
"<string>"
],
"score": 123,
"passed": true,
"notes": "<string>"
}
]
},
"error": "<string>",
"latency_ms": 123,
"created_at": "2023-11-07T05:31:56Z",
"content_location": {
"data_plane": true,
"s3_key": "<string>",
"storage_uri": "<string>",
"sha256": "<string>",
"bytes_size": 123,
"version": 123,
"fields": {}
},
"content_unavailable_reason": "result_content_in_data_plane"
}Evaluate content with a judge
Grade content against a judge’s rubric and get the verdict inline
import os
from dataframer import Dataframer
client = Dataframer(
api_key=os.environ.get("DATAFRAMER_API_KEY"), # This is the default and can be omitted
)
evaluation = client.dataframer.judges.evaluations.create(
judge_id="182bd5e5-6e1a-4fe4-a799-aa6d9a6ab26e",
messages=[{
"role": "system",
"content": "You are a support assistant.",
}, {
"role": "user",
"content": "Where is my refund?",
}],
output="Your refund was issued yesterday and should arrive within 3 business days.",
)
print(evaluation.id)curl https://df-api.dataframer.ai/api/dataframer/judges/$JUDGE_ID/evaluations/ \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $DATAFRAMER_API_KEY" \
-d '{
"messages": [
{
"role": "system",
"content": "You are a support assistant."
},
{
"role": "user",
"content": "Where is my refund?"
}
],
"output": "Your refund was issued yesterday and should arrive within 3 business days."
}'{
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"judge_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"judge_version": 123,
"source": "api",
"status": "succeeded",
"model": "<string>",
"verdict": {
"overall_score": 123,
"confidence": 123,
"reasoning": "<string>",
"dimensions": [
{
"dimension_id": "<string>",
"name": "<string>",
"scoring_model": "pass_fail",
"label": "<string>",
"labels": [
"<string>"
],
"score": 123,
"passed": true,
"notes": "<string>"
}
]
},
"error": "<string>",
"latency_ms": 123,
"created_at": "2023-11-07T05:31:56Z",
"content_location": {
"data_plane": true,
"s3_key": "<string>",
"storage_uri": "<string>",
"sha256": "<string>",
"bytes_size": 123,
"version": 123,
"fields": {}
},
"content_unavailable_reason": "result_content_in_data_plane"
}{
"error": "<string>"
}{
"error": "<string>"
}{
"error": "<string>"
}{
"error": "<string>"
}{
"error": "<string>"
}{
"error": "<string>",
"code": "not_supported_hybrid"
}{
"id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"judge_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"judge_version": 123,
"source": "api",
"status": "succeeded",
"model": "<string>",
"verdict": {
"overall_score": 123,
"confidence": 123,
"reasoning": "<string>",
"dimensions": [
{
"dimension_id": "<string>",
"name": "<string>",
"scoring_model": "pass_fail",
"label": "<string>",
"labels": [
"<string>"
],
"score": 123,
"passed": true,
"notes": "<string>"
}
]
},
"error": "<string>",
"latency_ms": 123,
"created_at": "2023-11-07T05:31:56Z",
"content_location": {
"data_plane": true,
"s3_key": "<string>",
"storage_uri": "<string>",
"sha256": "<string>",
"bytes_size": 123,
"version": 123,
"fields": {}
},
"content_unavailable_reason": "result_content_in_data_plane"
}Authorizations
API Key authentication. Format: "Bearer YOUR_API_KEY"
Path Parameters
Body
The content to grade. Provide either messages (with the candidate reply in output) or a plain input/output pair — messages and input are mutually exclusive, and at least one of messages/input/output is required.
Chat transcript in the standard role/content shape (the messages sent to your model). Content may be a string or a list of content blocks; inline files ride as base64 data: URIs in file blocks.
Plain input (string or JSON) — alternative to messages.
The candidate output to grade (string or message object).
Name for this interaction (shown to the judge as context).
The model that produced the content (context for the judge — not the judge model).
Free-form context shown to the judge (rendered as the interaction's trace metadata).
Normally omitted — the judge already pins its grading model, and every evaluation uses it by default. Set this only to deliberately grade with a different judge model for this one call (public judge-model names; append -thinking for extended reasoning).
Response
Verdict produced
Version number of the judge version that graded this content.
api, monitor, ui succeeded, failed The judge model used.
Null when status is failed.
Show child attributes
Show child attributes
Why the evaluation failed (null when it succeeded). For an evaluation that ran in a customer-hosted deployment a direct API response carries a short label only; the full text is stored in your own cloud, and the Python SDK returns it
Where the judge's reasoning and notes are stored in your own cloud (null unless the evaluation ran in a customer-hosted deployment)
Show child attributes
Show child attributes
Set when verdict.reasoning and each dimension's notes are empty because they are stored in your own cloud, not because the judge wrote none (null otherwise). The scores and labels are still complete. The Python SDK reads the text from your data plane, fills the fields and clears this
result_content_in_data_plane