evaluations

Creates, updates, deletes, gets or lists an evaluations resource.

Overview

Name	evaluations
Type	Resource
Id	azure.ai_evaluation.evaluations

Fields

The following fields are returned by SELECT queries:

get

Name	Datatype	Description
id	string	Identifier of the evaluation. Required.
data	object	Data for evaluation. Required.
description	string	Description of the evaluation. It can be used to store additional information about the evaluation and is mutable.
displayName	string	Display Name for evaluation. It helps to find the evaluation easily in AI Foundry. It does not need to be unique.
evaluators	object	Evaluators to be used for the evaluation. Required.
properties	object	Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.
status	string	Status of the evaluation. It is set by service and is read-only.
tags	object	Evaluation's tags. Unlike properties, tags are fully mutable.
target	object	Specifies the type and configuration of the entity used for this evaluation.

list

Name	Datatype	Description
id	string	Identifier of the evaluation. Required.
data	object	Data for evaluation. Required.
description	string	Description of the evaluation. It can be used to store additional information about the evaluation and is mutable.
displayName	string	Display Name for evaluation. It helps to find the evaluation easily in AI Foundry. It does not need to be unique.
evaluators	object	Evaluators to be used for the evaluation. Required.
properties	object	Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.
status	string	Status of the evaluation. It is set by service and is read-only.
tags	object	Evaluation's tags. Unlike properties, tags are fully mutable.
target	object	Specifies the type and configuration of the entity used for this evaluation.

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get	select	name, endpoint		Get an evaluation run by name.
list	select	endpoint		List evaluation runs.
create	insert	endpoint, data, evaluators		Creates an evaluation run.
delete	delete	name, endpoint		Delete an evaluation run by name.
create_agent_evaluation	exec	endpoint, runId, evaluators, appInsightsConnectionString		Creates an agent evaluation run.
cancel	exec	name, endpoint		Cancel an evaluation run by name.
check_annotation	exec	endpoint		Check annotation supported by the service.
submit_annotation	exec	endpoint, AnnotationTask, ContentType, UserTextList, Contents, MetricList, PromptVersion		Submit the annotation.
operation_results	exec	operation_id, endpoint		Poll for the operation results.
upload_run	exec	endpoint		Upload the result to an evaluation run.
upload_update_run	exec	name, endpoint		Update the uploaded the result to an evaluation run.

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
endpoint	string	The service endpoint host (no scheme), e.g. myaccount.table.cosmos.azure.com:443 - value of the client endpoint parameter. (default: )
name	string	Name of the evaluation run to update. Required.
operation_id	string	Operation ID for the polling operation. Required.

SELECT examples

get

Get an evaluation run by name.

SELECT
id,
data,
description,
displayName,
evaluators,
properties,
status,
tags,
target
FROM azure.ai_evaluation.evaluations
WHERE name = '{{ name }}' -- required
AND endpoint = '{{ endpoint }}' -- required
;

list

List evaluation runs.

SELECT
id,
data,
description,
displayName,
evaluators,
properties,
status,
tags,
target
FROM azure.ai_evaluation.evaluations
WHERE endpoint = '{{ endpoint }}' -- required
;

INSERT examples

create

Creates an evaluation run.

INSERT INTO azure.ai_evaluation.evaluations (
data,
displayName,
description,
tags,
properties,
evaluators,
target,
endpoint
)
SELECT 
'{{ data }}' /* required */,
'{{ displayName }}',
'{{ description }}',
'{{ tags }}',
'{{ properties }}',
'{{ evaluators }}' /* required */,
'{{ target }}',
'{{ endpoint }}'
RETURNING
id,
data,
description,
displayName,
evaluators,
properties,
status,
tags,
target
;

Manifest

# Description fields are for documentation purposes
- name: evaluations
props:
  - name: endpoint
    value: "{{ endpoint }}"
    description: Required parameter for the evaluations resource.
  - name: data
    description: |
      Data for evaluation. Required.
    value:
      type: "{{ type }}"
  - name: displayName
    value: "{{ displayName }}"
    description: |
      Display Name for evaluation. It helps to find the evaluation easily in AI Foundry. It does not need to be unique.
  - name: description
    value: "{{ description }}"
    description: |
      Description of the evaluation. It can be used to store additional information about the evaluation and is mutable.
  - name: tags
    value: "{{ tags }}"
    description: |
      Evaluation's tags. Unlike properties, tags are fully mutable.
  - name: properties
    value: "{{ properties }}"
    description: |
      Evaluation's properties. Unlike tags, properties are add-only. Once added, a property cannot be removed.
  - name: evaluators
    value: "{{ evaluators }}"
    description: |
      Evaluators to be used for the evaluation. Required.
  - name: target
    description: |
      Specifies the type and configuration of the entity used for this evaluation.
    value:
      type: "{{ type }}"

DELETE examples

delete

Delete an evaluation run by name.

DELETE FROM azure.ai_evaluation.evaluations
WHERE name = '{{ name }}' --required
AND endpoint = '{{ endpoint }}' --required
;

Lifecycle Methods

create_agent_evaluation

Creates an agent evaluation run.

EXEC azure.ai_evaluation.evaluations.create_agent_evaluation 
@endpoint='{{ endpoint }}' --required 
@@json=
'{
"runId": "{{ runId }}", 
"threadId": "{{ threadId }}", 
"evaluators": "{{ evaluators }}", 
"samplingConfiguration": "{{ samplingConfiguration }}", 
"redactionConfiguration": "{{ redactionConfiguration }}", 
"appInsightsConnectionString": "{{ appInsightsConnectionString }}"
}'
;

cancel

Cancel an evaluation run by name.

EXEC azure.ai_evaluation.evaluations.cancel 
@name='{{ name }}' --required, 
@endpoint='{{ endpoint }}' --required
;

check_annotation

Check annotation supported by the service.

EXEC azure.ai_evaluation.evaluations.check_annotation 
@endpoint='{{ endpoint }}' --required
;

submit_annotation

Submit the annotation.

EXEC azure.ai_evaluation.evaluations.submit_annotation 
@endpoint='{{ endpoint }}' --required 
@@json=
'{
"AnnotationTask": "{{ AnnotationTask }}", 
"ContentType": "{{ ContentType }}", 
"UserTextList": "{{ UserTextList }}", 
"Contents": "{{ Contents }}", 
"MetricList": "{{ MetricList }}", 
"PromptVersion": "{{ PromptVersion }}"
}'
;

operation_results

Poll for the operation results.

EXEC azure.ai_evaluation.evaluations.operation_results 
@operation_id='{{ operation_id }}' --required, 
@endpoint='{{ endpoint }}' --required
;

upload_run

Upload the result to an evaluation run.

EXEC azure.ai_evaluation.evaluations.upload_run 
@endpoint='{{ endpoint }}' --required 
@@json=
'{
"data": "{{ data }}", 
"target": "{{ target }}", 
"displayName": "{{ displayName }}", 
"description": "{{ description }}", 
"status": "{{ status }}", 
"tags": "{{ tags }}", 
"properties": "{{ properties }}", 
"evaluators": "{{ evaluators }}", 
"outputs": "{{ outputs }}"
}'
;

upload_update_run

Update the uploaded the result to an evaluation run.

EXEC azure.ai_evaluation.evaluations.upload_update_run 
@name='{{ name }}' --required, 
@endpoint='{{ endpoint }}' --required 
@@json=
'{
"data": "{{ data }}", 
"target": "{{ target }}", 
"displayName": "{{ displayName }}", 
"description": "{{ description }}", 
"status": "{{ status }}", 
"tags": "{{ tags }}", 
"properties": "{{ properties }}", 
"evaluators": "{{ evaluators }}", 
"outputs": "{{ outputs }}"
}'
;