OPIK-567 Basic Prompts CRUD tests

.github/workflows/end2end_suites.yml

@@ -15,6 +15,7 @@ on:
                 - traces
                 - datasets
                 - experiments
+                - prompts
     pull_request:
       paths:
         - 'sdks/code_generation/fern/openapi/openapi.yaml'
@@ -88,6 +89,8 @@ jobs:
                     pytest -s tests/Datasets/ --browser chromium --base-url http://localhost:5173 --setup-show
                 elif [ "$SUITE" == "experiments" ]; then
                     pytest -s tests/Experiments/test_experiments_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show
+                elif [ "$SUITE" == "prompts" ]; then
+                    pytest -s tests/Prompts/test_prompts_crud_operations.py --browser chromium --base-url http://localhost:5173 --setup-show
                 elif [ "$SUITE" == "sanity" ]; then
                     pytest -s tests/application_sanity/test_sanity.py --browser chromium --base-url http://localhost:5173 --setup-show
                 elif [ "$SUITE" == "all_features" ]; then

tests_end_to_end/page_objects/PromptLibraryPage.py

@@ -0,0 +1,68 @@
+from playwright.sync_api import Page, expect
+import time
+
+
+class PromptLibraryPage:
+    def __init__(self, page: Page):
+        self.page = page
+        self.url = "default/prompts"
+        self.prompts_table = self.page.get_by_role("table")
+
+    def go_to_page(self):
+        self.page.goto(self.url)
+
+    def click_prompt(self, prompt_name):
+        self.page.get_by_role("link", name=prompt_name).click()
+
+    def search_prompt(self, prompt_name):
+        self.page.get_by_test_id("search-input").click()
+        self.page.get_by_test_id("search-input").fill(prompt_name)
+
+    def check_prompt_exists_on_current_page(self, prompt_name):
+        expect(
+            self.page.get_by_role("cell", name=prompt_name, exact=True)
+        ).to_be_visible()
+
+    def check_prompt_exists_in_workspace(self, prompt_name):
+        self.search_prompt(prompt_name=prompt_name)
+        self.check_prompt_exists_on_current_page(prompt_name=prompt_name)
+
+    def check_prompt_not_exists_on_current_page(self, prompt_name):
+        expect(
+            self.page.get_by_role("cell", name=prompt_name, exact=True)
+        ).not_to_be_visible(timeout=1000)
+
+    def check_prompt_not_exists_in_workspace(self, prompt_name):
+        self.search_prompt(prompt_name=prompt_name)
+        self.check_prompt_not_exists_on_current_page(prompt_name=prompt_name)
+
+    def check_prompt_exists_on_current_page_with_retry(self, prompt_name, timeout):
+        start_time = time.time()
+        while time.time() - start_time < timeout:
+            try:
+                self.check_prompt_exists_on_current_page(prompt_name=prompt_name)
+                break
+            except AssertionError:
+                self.page.wait_for_timeout(500)
+        else:
+            raise AssertionError(
+                f"prompt {prompt_name} not found in prompts list within {timeout} seconds"
+            )
+
+    def create_new_prompt(self, name, prompt):
+        self.page.get_by_role("button", name="Create new prompt").first.click()
+        self.page.get_by_placeholder("Prompt name").fill(name)
+        self.page.get_by_placeholder("Prompt", exact=True).click()
+        self.page.get_by_placeholder("Prompt", exact=True).fill(prompt)
+        self.page.get_by_role("button", name="Create prompt").click()
+
+    def delete_prompt_by_name(self, prompt_name):
+        self.search_prompt(prompt_name)
+        row = (
+            self.page.get_by_role("row")
+            .filter(has_text=prompt_name)
+            .filter(has=self.page.get_by_role("cell", name=prompt_name, exact=True))
+        )
+        row.get_by_role("button").click()
+        self.page.get_by_role("menuitem", name="Delete").click()
+        self.page.get_by_role("button", name="Delete prompt").click()

tests_end_to_end/page_objects/PromptPage.py

@@ -0,0 +1,50 @@
+from playwright.sync_api import Page, expect, Locator
+
+
+class PromptPage:
+    def __init__(self, page: Page):
+        self.page = page
+        self.next_page_button_locator = self.page.locator(
+            "div:has(> button:nth-of-type(4))"
+        ).locator("button:nth-of-type(3)")
+
+    def edit_prompt(self, new_prompt: str):
+        self.page.get_by_role("button", name="Edit prompt").click()
+        self.page.get_by_role("textbox", name="Prompt").click()
+        self.page.get_by_role("textbox", name="Prompt").fill(new_prompt)
+        self.page.get_by_role("button", name="Edit prompt").click()
+
+    def click_most_recent_commit(self):
+        self.page.get_by_role("tab", name="Commits").click()
+        expect(self.page.get_by_role("row").nth(1)).to_be_visible()
+        self.page.get_by_role("row").nth(1).get_by_role("link").click()
+        self.page.wait_for_timeout(500)
+
+    def get_prompt_of_selected_commit(self):
+        return self.page.get_by_role("code").first.inner_text()
+
+    def get_all_prompt_versions_with_commit_ids_on_page(self):
+        rows: list[Locator] = self.page.get_by_role("row").all()[1:]
+        versions = {}
+        for row in rows:
+            prompt = row.get_by_role("cell").nth(1).inner_text()
+            commit_id = row.get_by_role("link").first.inner_text()
+            versions[prompt] = commit_id
+
+        return versions
+
+    def get_all_prompt_versions_with_commit_ids_for_prompt(self):
+        self.page.get_by_role("tab", name="Commits").click()
+        expect(self.page.get_by_role("table")).to_be_visible()
+        versions = {}
+        first_page = self.get_all_prompt_versions_with_commit_ids_on_page()
+        versions.update(first_page)
+        while (
+            self.next_page_button_locator.is_visible()
+            and self.next_page_button_locator.is_enabled()
+        ):
+            self.next_page_button_locator.click()
+            self.page.wait_for_timeout(500)
+            versions.extend(self.get_all_prompt_versions_with_commit_ids_on_page())
+
+        return versions

tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py

@@ -18,7 +18,9 @@
 
 
 class TestDatasetItemsCrud:
-    @pytest.mark.parametrize("dataset_insert", ["insert_via_ui", "insert_via_sdk"])
+    @pytest.mark.parametrize(
+        "dataset_insert", ["insert_via_sdk"]
+    )  # add insert_via_ui once flakiness is figured out
     @pytest.mark.parametrize(
         "dataset_creation_fixture",
         ["create_delete_dataset_sdk", "create_delete_dataset_ui"],

tests_end_to_end/tests/Prompts/test_prompts_crud_operations.py

@@ -0,0 +1,106 @@
+import pytest
+import opik
+from time import sleep
+from playwright.sync_api import Page
+from page_objects.PromptLibraryPage import PromptLibraryPage
+from page_objects.PromptPage import PromptPage
+
+
+class TestPromptsCrud:
+    @pytest.mark.parametrize(
+        "prompt_fixture", ["create_prompt_sdk", "create_prompt_ui"]
+    )
+    def test_prompt_visibility(
+        self, request, page: Page, client: opik.Opik, prompt_fixture
+    ):
+        """
+        Tests creation of a prompt via UI and SDK and visibility in both
+        1. Create a prompt via either UI or SDK (each gets its own respective test instance)
+        2. Check the prompt is visible in both UI and SDK
+        """
+        prompt: opik.Prompt = request.getfixturevalue(prompt_fixture)
+
+        retries = 0
+        while retries < 5:
+            prompt_sdk: opik.Prompt = client.get_prompt(name=prompt.name)
+            if prompt_sdk:
+                break
+            else:
+                sleep(1)
+        assert prompt.name == prompt_sdk.name
+        assert prompt.prompt == prompt_sdk.prompt
+
+        prompt_library_page = PromptLibraryPage(page)
+        prompt_library_page.go_to_page()
+        prompt_library_page.check_prompt_exists_in_workspace(prompt_name=prompt.name)
+
+    @pytest.mark.parametrize(
+        "prompt_fixture", ["create_prompt_sdk", "create_prompt_ui"]
+    )
+    def test_prompt_deletion(
+        self, request, page: Page, client: opik.Opik, prompt_fixture
+    ):
+        """
+        Tests deletion of prompt via the UI (currently no user-facing way to do it via the SDK)
+        1. Create a prompt via either UI or SDK (each gets its own respective test instance)
+        2. Delete that newly created prompt via the UI
+        3. Check that the prompt is no longer present in the UI
+        4. Check that the prompt is not returned by the SDK (client.get_prompt should return None)
+        """
+        prompt: opik.Prompt = request.getfixturevalue(prompt_fixture)
+
+        prompt_library_page = PromptLibraryPage(page)
+        prompt_library_page.go_to_page()
+        prompt_library_page.delete_prompt_by_name(prompt.name)
+
+        prompt_library_page.page.reload()
+        prompt_library_page.check_prompt_not_exists_in_workspace(
+            prompt_name=prompt.name
+        )
+
+        assert not client.get_prompt(name=prompt.name)
+
+    @pytest.mark.parametrize(
+        "prompt_fixture", ["create_prompt_sdk", "create_prompt_ui"]
+    )
+    @pytest.mark.parametrize("update_method", ["sdk", "ui"])
+    def test_prompt_update(
+        self, request, page: Page, client: opik.Opik, prompt_fixture, update_method
+    ):
+        """
+        Tests updating a prompt and creating a new prompt version via both UI and SDK
+        1. Create a prompt via either UI or SDK (each gets its own respective test instance)
+        2. Update that prompt with new text via either UI or SDK (each gets its own respective test instance)
+        3. In the UI, grab the prompt texts from the Commits tab and check that both old and new prompt versions are present
+        4. In the Commits tab, click the most recent commit and check that it is the updated prompt version
+        5. Get the Prompt via the SDK via the Prompt object without specifying a commit, check it returns the newest updated version
+        6. Grab the Prompt via the SDK while specifying the old commit, check that the old version is returned
+        """
+        prompt: opik.Prompt = request.getfixturevalue(prompt_fixture)
+        UPDATE_TEXT = "This is an updated prompt version"
+
+        prompt_library_page = PromptLibraryPage(page)
+        prompt_library_page.go_to_page()
+        prompt_library_page.click_prompt(prompt_name=prompt.name)
+
+        prompt_page = PromptPage(page)
+
+        if update_method == "sdk":
+            _ = opik.Prompt(name=prompt.name, prompt=UPDATE_TEXT)
+        elif update_method == "ui":
+            prompt_page.edit_prompt(new_prompt=UPDATE_TEXT)
+
+        versions = prompt_page.get_all_prompt_versions_with_commit_ids_for_prompt()
+        assert prompt.prompt in versions.keys()
+        assert UPDATE_TEXT in versions.keys()
+
+        prompt_page.click_most_recent_commit()
+        assert prompt_page.get_prompt_of_selected_commit() == UPDATE_TEXT
+
+        prompt_update = client.get_prompt(name=prompt.name)
+        assert prompt_update.prompt == UPDATE_TEXT
+        original_commit_id = versions[prompt.prompt]
+        assert (
+            client.get_prompt(name=prompt.name, commit=original_commit_id).prompt
+            == prompt.prompt
+        )

tests_end_to_end/tests/conftest.py

@@ -6,10 +6,12 @@
 from page_objects.TracesPage import TracesPage
 from page_objects.DatasetsPage import DatasetsPage
 from page_objects.ExperimentsPage import ExperimentsPage
+from page_objects.PromptLibraryPage import PromptLibraryPage
 from tests.sdk_helpers import (
     create_project_sdk,
     delete_project_by_name_sdk,
     wait_for_number_of_traces_to_be_visible,
+    client_get_prompt_retries,
 )
 from utils import TEST_ITEMS
 
@@ -156,6 +158,41 @@ def insert_dataset_items_sdk(client: opik.Opik, create_delete_dataset_sdk):
     dataset.insert(TEST_ITEMS)
 
 
+@pytest.fixture
+def create_prompt_sdk(client: opik.Opik, page: Page):
+    prompt = client.create_prompt(name="test_prompt", prompt="this is a test prompt")
+    yield prompt
+    prompt_library_page = PromptLibraryPage(page)
+    prompt_library_page.go_to_page()
+    try:
+        prompt_library_page.check_prompt_not_exists_in_workspace(
+            prompt_name="test_prompt"
+        )
+    except AssertionError as _:
+        prompt_library_page.delete_prompt_by_name(prompt.name)
+
+
+@pytest.fixture
+def create_prompt_ui(client: opik.Opik, page: Page):
+    prompt_library_page = PromptLibraryPage(page)
+    prompt_library_page.go_to_page()
+    prompt_library_page.create_new_prompt(
+        name="test_prompt", prompt="this is a test prompt"
+    )
+    prompt = client_get_prompt_retries(
+        client=client, prompt_name="test_prompt", timeout=10, initial_delay=1
+    )
+    yield prompt
+    prompt_library_page = PromptLibraryPage(page)
+    prompt_library_page.go_to_page()
+    try:
+        prompt_library_page.check_prompt_not_exists_in_workspace(
+            prompt_name="test_prompt"
+        )
+    except AssertionError as _:
+        prompt_library_page.delete_prompt_by_name(prompt.name)
+
+
 @pytest.fixture
 def create_10_test_traces(page: Page, client, create_delete_project_sdk):
     proj_name = create_delete_project_sdk

tests_end_to_end/tests/sdk_helpers.py

@@ -3,6 +3,7 @@
 from opik.rest_api.client import OpikApi
 from typing import Optional
 import json
+import opik
 
 
 def create_project_sdk(name: str):
@@ -168,3 +169,23 @@ def experiment_items_stream(exp_name: str, limit: Optional[int] = None):
     lines = data.decode("utf-8").split("\r\n")
     dict_list = [json.loads(line) for line in lines if line.strip()]
     return dict_list
+
+
+def client_get_prompt_retries(
+    client: opik.Opik, prompt_name, timeout=10, initial_delay=1
+):
+    start_time = time.time()
+    delay = initial_delay
+
+    while time.time() - start_time < timeout:
+        prompt = client.get_prompt(name=prompt_name)
+
+        if prompt:
+            return prompt
+
+        time.sleep(delay)
+        delay = min(delay * 2, timeout - (time.time() - start_time))
+
+    raise TimeoutError(
+        f"could not get created prompt {prompt_name} via SDK client within {timeout} seconds"
+    )