Docs
|
GitHub
|
Community
Answer Relevancy and Context Relevancy Evaluations
👉 See Llama-Index notebook for more info 👈
```shell
pip install -Uqqq "arize-phoenix[llama-index]>=4.6" nest_asyncio
```
# Enter OpenAI API Key
```python
import os
from getpass import getpass
if not os.getenv("OPENAI_API_KEY"):
os.environ["OPENAI_API_KEY"] = getpass("🔑 Enter your OpenAI API key: ")
```
# Import Modules
```python
import json
import tempfile
from textwrap import shorten
from time import time_ns
from typing import Tuple
import nest_asyncio
import phoenix as px
from llama_index.core import VectorStoreIndex
from llama_index.core.evaluation import AnswerRelevancyEvaluator, ContextRelevancyEvaluator
from llama_index.core.llama_dataset import download_llama_dataset
from llama_index.llms.openai import OpenAI
from openinference.instrumentation.llama_index import LlamaIndexInstrumentor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from phoenix.experiments import evaluate_experiment, run_experiment
from phoenix.experiments.types import Explanation, Score
nest_asyncio.apply()
```
# Launch Phoenix
```python
px.launch_app()
```
# Instrument Llama-Index
```python
endpoint = "http://127.0.0.1:4317"
(tracer_provider := TracerProvider()).add_span_processor(
SimpleSpanProcessor(OTLPSpanExporter(endpoint))
)
LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider)
```
# Upload Dataset to Phoenix
```python
sample_size = 7
dataset_name = "EvaluatingLlmSurveyPaperDataset"
with tempfile.TemporaryDirectory() as dir_name:
rag_dataset, documents = download_llama_dataset(dataset_name, dir_name)
dataset = px.Client().upload_dataset(
dataset_name=f"{dataset_name}_{time_ns()}",
dataframe=rag_dataset.to_pandas().sample(sample_size, random_state=42),
)
```
# Dataset Can be Viewed as Dataframe
```python
dataset.as_dataframe()
```
# Take a Look at the Data Structure of an Example
```python
dataset[0]
```
# Define Task Function on Examples
Task function can be either sync or async.
```python
index = VectorStoreIndex.from_documents(documents=documents)
query_engine = index.as_query_engine()
async def task(input):
ans = await query_engine.aquery(input["query"])
return {
"contexts": [node.text for node in ans.source_nodes],
"response": ans.response,
}
```
# Check that Task Can Run Successfully
```python
example = dataset[0]
task_output = await task(example.input)
print(shorten(json.dumps(task_output), width=80))
```
# Dry-Run Experiment
On 3 randomly selected examples
```python
experiment = run_experiment(dataset, task, dry_run=3)
```
# Experiment Results Can be Viewed as Dataframe
```python
experiment.as_dataframe()
```
# Take a Look at the Data Structure of an Experiment Run
```python
experiment[0]
```
# Define Evaluators For Each Experiment Run
Evaluators can be sync or async.
Function arguments `output` and `input` refers to the attributes of the same name in the `ExperimentRun` data structure shown above.
```python
async def answer_relevancy(output, input) -> Tuple[Score, Explanation]:
ans = await AnswerRelevancyEvaluator(
llm=OpenAI(temperature=0, model="gpt-4o"),
).aevaluate(input["query"], response=output["response"])
return ans.score, ans.feedback
async def context_relevancy(output, input) -> Tuple[Score, Explanation]:
ans = await ContextRelevancyEvaluator(
llm=OpenAI(temperature=0, model="gpt-4o"),
).aevaluate(input["query"], contexts=output["contexts"])
return ans.score, ans.feedback
evaluators = [answer_relevancy, context_relevancy]
```
# Check that Evals Can Run Successfully
```python
run = experiment[0]
example = dataset.examples[run.dataset_example_id]
for fn in (answer_relevancy, context_relevancy):
_ = await fn(run.output, example.input)
print(fn.__qualname__)
print(shorten(json.dumps(_), width=80))
```
# Run Evaluations
```python
experiment = evaluate_experiment(experiment, evaluators)
```
# Evaluation Results Can be Viewed as Dataframe
```python
experiment.get_evaluations()
```
# Run Task and Evals Together
```python
_ = run_experiment(dataset, task, evaluators)
```
```python
```