Evaluate answers - Pinecone Docs

Evaluate responses from an assistant or other RAG system to measure their correctness and completeness against a ground truth answer.

Evaluate answers with the API or an SDK

To evaluate a response from an assistant, call evaluate_alignment (evaluate in Node.js). The question is what you asked, the answer is the response to evaluate, and ground_truth_answer (groundTruth in Node.js) is the response you expect.

from pinecone import Pinecone

pc = Pinecone(api_key="YOUR_API_KEY")

result = pc.assistants.evaluate_alignment(
    question="What are the capital cities of France, England and Spain?",
    answer="Paris is the capital city of France and Barcelona of Spain",
    ground_truth_answer="Paris is the capital city of France, London of England and Madrid of Spain.",
)

print(result)

import { Pinecone } from '@pinecone-database/pinecone';

const pc = new Pinecone({ apiKey: 'YOUR_API_KEY' });

const result = await pc.assistants.evaluate({
  question: 'What are the capital cities of France, England and Spain?',
  answer: 'Paris is the capital city of France and Barcelona of Spain',
  groundTruth: 'Paris is the capital city of France, London of England and Madrid of Spain.',
});

console.log(result);

PINECONE_API_KEY="YOUR_API_KEY"

curl "https://prod-1-data.ke.pinecone.io/assistant/evaluation/metrics/alignment" \
  -H "Api-Key: $PINECONE_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
  "question": "What are the capital cities of France, England and Spain?",
  "answer": "Paris is the capital city of France and Barcelona of Spain",
  "ground_truth_answer": "Paris is the capital city of France, London of England and Madrid of Spain"
}'

Response

{
  "metrics": {
    "correctness": 0.5,
    "completeness": 0.3333,
    "alignment": 0.4
  },
  "reasoning": {
    "evaluated_facts": [
      {
        "fact": {
          "content": "Paris is the capital city of France."
        },
        "entailment": "entailed"
      },
      {
        "fact": {
          "content": "London is the capital city of England."
        },
        "entailment": "neutral"
      },
      {
        "fact": {
          "content": "Madrid is the capital city of Spain."
        },
        "entailment": "contradicted"
      }
    ]
  },
  "usage": {
    "prompt_tokens": 1223,
    "completion_tokens": 51,
    "total_tokens": 1274
  }
}