# To use the Python SDK, install the plugin:
# pip install --upgrade pinecone pinecone-plugin-assistant
# pip install requests
import requests
from pinecone_plugins.assistant.models.chat import Message
qa_data = {
"question": "What are the capital cities of France, England and Spain?",
"ground_truth_answer": "Paris is the capital city of France, London of England and Madrid of Spain"
}
for qa in qa_data:
chat_context = [Message(role="user", content=qa["question"])]
response = assistant.chat(messages=chat_context)
answer = response.message.content # The answer from the Assistant - see https://docs.pinecone.io/guides/assistant/chat-with-assistant
eval_data = {
"question": qa["question"],
"answer": answer,
"ground_truth_answer": qa["ground_truth_answer"]
}
response = requests.post(
"https://prod-1-data.ke.pinecone.io/assistant/evaluation/metrics/alignment",
headers={
"Api-Key": os.environ["PINECONE_API_KEY"],
"Content-Type": "application/json"
},
json=eval_data
)
print(response.text)
{
"metrics": {
"correctness": 1.0,
"completeness": 1.0,
"alignment": 1.0
},
"reasoning": {
"evaluated_facts": [
{
"fact": {
"content": "<string>"
},
"entailment": "entailed"
}
]
},
"usage": {
"prompt_tokens": 123,
"completion_tokens": 123,
"total_tokens": 123
}
}
Evaluate the correctness and completeness of a response from an assistant or a RAG system. The correctness and completeness are evaluated based on the precision and recall of the generated answer with respect to the ground truth answer facts. Alignment is the harmonic mean of correctness and completeness.
For guidance and examples, see Evaluate answers.
# To use the Python SDK, install the plugin:
# pip install --upgrade pinecone pinecone-plugin-assistant
# pip install requests
import requests
from pinecone_plugins.assistant.models.chat import Message
qa_data = {
"question": "What are the capital cities of France, England and Spain?",
"ground_truth_answer": "Paris is the capital city of France, London of England and Madrid of Spain"
}
for qa in qa_data:
chat_context = [Message(role="user", content=qa["question"])]
response = assistant.chat(messages=chat_context)
answer = response.message.content # The answer from the Assistant - see https://docs.pinecone.io/guides/assistant/chat-with-assistant
eval_data = {
"question": qa["question"],
"answer": answer,
"ground_truth_answer": qa["ground_truth_answer"]
}
response = requests.post(
"https://prod-1-data.ke.pinecone.io/assistant/evaluation/metrics/alignment",
headers={
"Api-Key": os.environ["PINECONE_API_KEY"],
"Content-Type": "application/json"
},
json=eval_data
)
print(response.text)
{
"metrics": {
"correctness": 1.0,
"completeness": 1.0,
"alignment": 1.0
},
"reasoning": {
"evaluated_facts": [
{
"fact": {
"content": "<string>"
},
"entailment": "entailed"
}
]
},
"usage": {
"prompt_tokens": 123,
"completion_tokens": 123,
"total_tokens": 123
}
}
# To use the Python SDK, install the plugin:
# pip install --upgrade pinecone pinecone-plugin-assistant
# pip install requests
import requests
from pinecone_plugins.assistant.models.chat import Message
qa_data = {
"question": "What are the capital cities of France, England and Spain?",
"ground_truth_answer": "Paris is the capital city of France, London of England and Madrid of Spain"
}
for qa in qa_data:
chat_context = [Message(role="user", content=qa["question"])]
response = assistant.chat(messages=chat_context)
answer = response.message.content # The answer from the Assistant - see https://docs.pinecone.io/guides/assistant/chat-with-assistant
eval_data = {
"question": qa["question"],
"answer": answer,
"ground_truth_answer": qa["ground_truth_answer"]
}
response = requests.post(
"https://prod-1-data.ke.pinecone.io/assistant/evaluation/metrics/alignment",
headers={
"Api-Key": os.environ["PINECONE_API_KEY"],
"Content-Type": "application/json"
},
json=eval_data
)
print(response.text)
{
"metrics": {
"correctness": 1.0,
"completeness": 1.0,
"alignment": 1.0
},
"reasoning": {
"evaluated_facts": [
{
"fact": {
"content": "<string>"
},
"entailment": "entailed"
}
]
},
"usage": {
"prompt_tokens": 123,
"completion_tokens": 123,
"total_tokens": 123
}
}
The request body for the alignment evaluation.
The request for the alignment evaluation.
The evaluation metrics and reasoning for the generated answer.
The response for the alignment evaluation.
Was this page helpful?