Docs
|
GitHub
|
Community
Guideline Eval
👉 See Llama-Index notebook for more info 👈
```shell
pip install -Uqqq "arize-phoenix[llama-index]>=4.6" datasets nest_asyncio
```
# Enter OpenAI API Key
```python
import os
from getpass import getpass
if not os.getenv("OPENAI_API_KEY"):
os.environ["OPENAI_API_KEY"] = getpass("🔑 Enter your OpenAI API key: ")
```
# Import Modules
```python
import json
from functools import partial
from textwrap import shorten
from time import time_ns
from typing import Tuple
import nest_asyncio
import phoenix as px
from datasets import load_dataset
from llama_index.core.evaluation import GuidelineEvaluator
from llama_index.llms.openai import OpenAI
from openinference.instrumentation.llama_index import LlamaIndexInstrumentor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from phoenix.experiments import evaluate_experiment, run_experiment
from phoenix.experiments.types import Explanation, Score
nest_asyncio.apply()
```
# Launch Phoenix
```python
px.launch_app()
```
# Instrument Llama-Index
```python
endpoint = "http://127.0.0.1:4317"
(tracer_provider := TracerProvider()).add_span_processor(
SimpleSpanProcessor(OTLPSpanExporter(endpoint))
)
LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider)
```
# Upload Dataset to Phoenix
```python
sample_size = 7
path = "nvidia/ChatQA-Training-Data"
name = "synthetic_convqa"
df = load_dataset(path, name, split="train").to_pandas()
df = df.loc[:, ["messages", "document"]]
dataset = px.Client().upload_dataset(
dataset_name=f"{name}_{time_ns()}",
dataframe=df.sample(sample_size, random_state=42),
)
```
# Dataset Can be Viewed as Dataframe
```python
dataset.as_dataframe()
```
# Take a Look at the Data Structure of an Example
```python
dataset[0]
```
# Define Task Function on Examples
Task function can be either sync or async.
```python
llm = OpenAI(model="gpt-3.5-turbo")
def task(input):
return llm.complete(input["document"] + "\n\n" + input["messages"][-1]["content"]).text
```
# Check that Task Can Run Successfully
```python
example = dataset[0]
task_output = task(example.input)
print(shorten(json.dumps(task_output), width=80))
```
# Dry-Run Experiment
On 3 randomly selected examples
```python
experiment = run_experiment(dataset, task, dry_run=3)
```
# Experiment Results Can be Viewed as Dataframe
```python
experiment.as_dataframe()
```
# Take a Look at the Data Structure of an Experiment Run
```python
experiment[0]
```
# Define Evaluators For Each Experiment Run
Evaluators can be sync or async.
Function arguments `output` and `input` refer to the attributes of the same name in the `ExperimentRun` data structure shown above.
```python
llm = OpenAI(temperature=0, model="gpt-4o")
guidelines = {
"answer_fully": "The response should fully answer the query.",
"unambiguous": "The response should avoid being vague or ambiguous.",
"use_numbers": "The response should be specific and use statistics or numbers when possible.",
}
async def adapt(fn, output, input) -> Tuple[Score, Explanation]:
ans = await fn(
query=input["messages"][0]["content"],
response=output,
contexts=[input["document"]],
)
return ans.passing, ans.feedback
evaluators = {
name: partial(adapt, GuidelineEvaluator(llm=llm, guidelines=guideline).aevaluate)
for name, guideline in guidelines.items()
}
```
# Check that Evals Can Run Successfully
```python
run = experiment[0]
example = dataset.examples[run.dataset_example_id]
for name, fn in evaluators.items():
_ = await fn(run.output, example.input)
print(name)
print(shorten(json.dumps(_), width=80))
```
# Run Evaluations
```python
experiment = evaluate_experiment(experiment, evaluators)
```
# Evaluation Results Can be Viewed as Dataframe
```python
experiment.get_evaluations()
```
# Run Task and Evals Together
```python
_ = run_experiment(dataset, task, evaluators)
```