{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<center>\n",
    "    <p style=\"text-align:center\">\n",
    "        <img alt=\"phoenix logo\" src=\"https://raw.githubusercontent.com/Arize-ai/phoenix-assets/9e6101d95936f4bd4d390efc9ce646dc6937fb2d/images/socal/github-large-banner-phoenix.jpg\" width=\"1000\"/>\n",
    "        <br>\n",
    "        <br>\n",
    "        <a href=\"https://docs.arize.com/phoenix/\">Docs</a>\n",
    "        |\n",
    "        <a href=\"https://github.com/Arize-ai/phoenix\">GitHub</a>\n",
    "        |\n",
    "        <a href=\"https://join.slack.com/t/arize-ai/shared_invite/zt-1px8dcmlf-fmThhDFD_V_48oU7ALan4Q\">Community</a>\n",
    "    </p>\n",
    "</center>\n",
    "<h1 align=\"center\">Guideline Eval</h1>\n",
    "<h5 align=\"center\">👉 See Llama-Index <a href=\"https://github.com/run-llama/llama_index/blob/80cee5a511360eedd7837f20d283bf0a9bd05603/docs/docs/examples/evaluation/guideline_eval.ipynb\">notebook</a> for more info 👈</h5>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/arize-ai/phoenix/blob/main/tutorials/experiments/llama-index/guideline_eval.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install -Uqqq \"arize-phoenix[llama-index]>=4.6\" datasets nest_asyncio"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Enter OpenAI API Key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from getpass import getpass\n",
    "\n",
    "if not os.getenv(\"OPENAI_API_KEY\"):\n",
    "    os.environ[\"OPENAI_API_KEY\"] = getpass(\"🔑 Enter your OpenAI API key: \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from functools import partial\n",
    "from textwrap import shorten\n",
    "from time import time_ns\n",
    "from typing import Tuple\n",
    "\n",
    "import nest_asyncio\n",
    "from datasets import load_dataset\n",
    "from llama_index.core.evaluation import GuidelineEvaluator\n",
    "from llama_index.llms.openai import OpenAI\n",
    "from openinference.instrumentation.llama_index import LlamaIndexInstrumentor\n",
    "\n",
    "import phoenix as px\n",
    "from phoenix.experiments import evaluate_experiment, run_experiment\n",
    "from phoenix.experiments.types import Explanation, Score\n",
    "from phoenix.otel import register\n",
    "\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Launch Phoenix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "px.launch_app().view()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Instrument Llama-Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tracer_provider = register(endpoint=\"http://127.0.0.1:4317\")\n",
    "LlamaIndexInstrumentor().instrument(skip_dep_check=True, tracer_provider=tracer_provider)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upload Dataset to Phoenix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_size = 7\n",
    "path = \"nvidia/ChatQA-Training-Data\"\n",
    "name = \"synthetic_convqa\"\n",
    "df = load_dataset(path, name, split=\"train\").to_pandas()\n",
    "df = df.loc[:, [\"messages\", \"document\"]]\n",
    "dataset = px.Client().upload_dataset(\n",
    "    dataset_name=f\"{name}_{time_ns()}\",\n",
    "    dataframe=df.sample(sample_size, random_state=42),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset Can be Viewed as Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Take a Look at the Data Structure of an Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Task Function on Examples\n",
    "\n",
    "Task function can be either sync or async."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = OpenAI(model=\"gpt-3.5-turbo\")\n",
    "\n",
    "\n",
    "def task(input):\n",
    "    return llm.complete(input[\"document\"] + \"\\n\\n\" + input[\"messages\"][-1][\"content\"]).text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check that Task Can Run Successfully"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example = dataset[0]\n",
    "task_output = task(example.input)\n",
    "print(shorten(json.dumps(task_output), width=80))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dry-Run Experiment\n",
    "\n",
    "On 3 randomly selected examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment = run_experiment(dataset, task, dry_run=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment Results Can be Viewed as Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment.as_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Take a Look at the Data Structure of an Experiment Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Evaluators For Each Experiment Run\n",
    "\n",
    "Evaluators can be sync or async.\n",
    "\n",
    "Function arguments `output` and `input` refer to the attributes of the same name in the `ExperimentRun` data structure shown above."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = OpenAI(temperature=0, model=\"gpt-4o\")\n",
    "guidelines = {\n",
    "    \"answer_fully\": \"The response should fully answer the query.\",\n",
    "    \"unambiguous\": \"The response should avoid being vague or ambiguous.\",\n",
    "    \"use_numbers\": \"The response should be specific and use statistics or numbers when possible.\",\n",
    "}\n",
    "\n",
    "\n",
    "async def adapt(fn, output, input) -> Tuple[Score, Explanation]:\n",
    "    ans = await fn(\n",
    "        query=input[\"messages\"][0][\"content\"],\n",
    "        response=output,\n",
    "        contexts=[input[\"document\"]],\n",
    "    )\n",
    "    return ans.passing, ans.feedback\n",
    "\n",
    "\n",
    "evaluators = {\n",
    "    name: partial(adapt, GuidelineEvaluator(llm=llm, guidelines=guideline).aevaluate)\n",
    "    for name, guideline in guidelines.items()\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Check that Evals Can Run Successfully"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run = experiment[0]\n",
    "example = dataset.examples[run.dataset_example_id]\n",
    "for name, fn in evaluators.items():\n",
    "    _ = await fn(run.output, example.input)\n",
    "    print(name)\n",
    "    print(shorten(json.dumps(_), width=80))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run Evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment = evaluate_experiment(experiment, evaluators)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation Results Can be Viewed as Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment.get_evaluations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run Task and Evals Together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = run_experiment(dataset, task, evaluators)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}