Skip to content

Commit

Permalink
Fixed moderation cookbook
Browse files Browse the repository at this point in the history
  • Loading branch information
@jverre
jverre committed Dec 21, 2024
1 parent f07111a commit 19ff7ea
Showing 1 changed file with 14 additions and 16 deletions.
30 changes: 14 additions & 16 deletions apps/opik-documentation/documentation/docs/cookbook/evaluate_moderation_metric.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,6 @@
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.3.1\u001b[0m\n",
"\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
"Note: you may need to restart the kernel to use updated packages.\n"
]
}
Expand All @@ -49,7 +46,8 @@
"name": "stderr",
"output_type": "stream",
"text": [
"OPIK: Opik is already configured. You can check the settings by viewing the config file at /Users/jacquesverre/.opik.config\n"
"OPIK: Your Opik API key is available in your account settings, can be found at https://www.comet.com/api/my/settings/ for Opik cloud\n",
"OPIK: Configuration saved to file: /Users/jacquesverre/.opik.config\n"
]
}
],
Expand Down Expand Up @@ -151,37 +149,39 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluation: 0%| | 0/50 [00:00<?, ?it/s]OPIK: Started logging traces to the \"Default Project\" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=Default%20Project.\n",
"Evaluation: 100%|██████████| 50/50 [00:08<00:00, 6.11it/s]\n"
"OPIK: Started logging traces to the \"Default Project\" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=Default%20Project.\n",
"OPIK: Started logging traces to the \"Default Project\" project at https://www.comet.com/opik/jacques-comet/redirect/projects?name=Default%20Project.\n",
"Evaluation: 100%|██████████| 50/50 [00:05<00:00, 8.86it/s]\n"
]
},
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">╭─ OpenAIModerationDataset (50 samples) ─╮\n",
"│ │\n",
"│ <span style=\"font-weight: bold\">Total time: </span> 00:00:08 │\n",
"│ <span style=\"font-weight: bold\">Total time: </span> 00:00:06 │\n",
"│ <span style=\"font-weight: bold\">Number of samples:</span> 50 │\n",
"│ │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Correct moderation score: 0.8800 (avg)</span> │\n",
"│ <span style=\"color: #008000; text-decoration-color: #008000; font-weight: bold\">Correct moderation score: 0.8600 (avg)</span> │\n",
"│ │\n",
"╰────────────────────────────────────────╯\n",
"</pre>\n"
],
"text/plain": [
"╭─ OpenAIModerationDataset (50 samples) ─╮\n",
"│ │\n",
"│ \u001b[1mTotal time: \u001b[0m 00:00:08 │\n",
"│ \u001b[1mTotal time: \u001b[0m 00:00:06 │\n",
"│ \u001b[1mNumber of samples:\u001b[0m 50 │\n",
"│ │\n",
"│ \u001b[1;32mCorrect moderation score: 0.8800 (avg)\u001b[0m │\n",
"│ \u001b[1;32mCorrect moderation score: 0.8600 (avg)\u001b[0m │\n",
"│ │\n",
"╰────────────────────────────────────────╯\n"
]
Expand All @@ -205,11 +205,11 @@
{
"data": {
"text/html": [
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">View the results <a href=\"https://www.comet.com/opik/jacques-comet/experiments/01939721-3a29-7f9a-ac8b-737923e30e31/compare?experiments=%5B%2206751b32-5e8a-7276-8000-2da2be3b1cd9%22%5D\" target=\"_blank\">in your Opik dashboard</a>.\n",
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">View the results <a href=\"https://www.comet.com/opik/jacques-comet/experiments/01939721-3a29-7f9a-ac8b-737923e30e31/compare?experiments=%5B%2206766a9d-2e22-7b5a-8000-afa08cbccbb4%22%5D\" target=\"_blank\">in your Opik dashboard</a>.\n",
"</pre>\n"
],
"text/plain": [
"View the results \u001b]8;id=595231;https://www.comet.com/opik/jacques-comet/experiments/01939721-3a29-7f9a-ac8b-737923e30e31/compare?experiments=%5B%2206751b32-5e8a-7276-8000-2da2be3b1cd9%22%5D\u001b\\in your Opik dashboard\u001b]8;;\u001b\\.\n"
"View the results \u001b]8;id=373788;https://www.comet.com/opik/jacques-comet/experiments/01939721-3a29-7f9a-ac8b-737923e30e31/compare?experiments=%5B%2206766a9d-2e22-7b5a-8000-afa08cbccbb4%22%5D\u001b\\in your Opik dashboard\u001b]8;;\u001b\\.\n"
]
},
"metadata": {},
Expand All @@ -229,15 +229,13 @@
" metric = Moderation()\n",
" try:\n",
" metric_score = metric.score(output=x[\"output\"])\n",
" moderation_score = metric_score.value\n",
" moderation_score = \"moderated\" if metric_score.value > 0.5 else \"not_moderated\"\n",
" moderation_reason = metric_score.reason\n",
" except Exception as e:\n",
" print(e)\n",
" moderation_score = None\n",
" moderation_reason = str(e)\n",
"\n",
" moderation_score = \"moderated\" if metric_score.value > 0.5 else \"not_moderated\"\n",
"\n",
" return {\n",
" \"moderation_score\": moderation_score,\n",
" \"moderation_reason\": moderation_reason,\n",
Expand Down Expand Up @@ -282,7 +280,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": "py312_llm_eval",
"language": "python",
"name": "python3"
},
Expand Down

0 comments on commit 19ff7ea

Please sign in to comment.