diff --git a/.github/workflows/tests_end_to_end_linter.yml b/.github/workflows/tests_end_to_end_linter.yml new file mode 100644 index 0000000000..d93d722a6f --- /dev/null +++ b/.github/workflows/tests_end_to_end_linter.yml @@ -0,0 +1,24 @@ +--- + name: E2E Tests Linter + run-name: "E2E Tests Linter ${{ github.ref_name }} by @${{ github.actor }}" + on: + pull_request: + paths: + - 'tests_end_to_end/**' + push: + branches: + - 'main' + paths: + - 'tests_end_to_end/**' + jobs: + lint: + runs-on: ubuntu-latest + defaults: + run: + working-directory: tests_end_to_end + steps: + - uses: actions/checkout@v4 + - name: install pre-commit + run: pip install pre-commit + - name: linting + run: pre-commit run --all-files diff --git a/tests_end_to_end/.pre-commit-config.yaml b/tests_end_to_end/.pre-commit-config.yaml new file mode 100644 index 0000000000..e6251a1180 --- /dev/null +++ b/tests_end_to_end/.pre-commit-config.yaml @@ -0,0 +1,25 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + files: ^tests_end_to_end + - id: end-of-file-fixer + files: ^tests_end_to_end + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.9 + hooks: + - id: ruff + args: [ --fix, --show-fixes] + files: ^tests_end_to_end + - id: ruff-format + files: ^tests_end_to_end + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.10.0 + hooks: + - id: mypy + files: ^tests_end_to_end + args: [--config-file, tests_end_to_end/pyproject.toml] + additional_dependencies: ['types-PyYAML'] diff --git a/tests_end_to_end/installer_utils/check_backend.sh b/tests_end_to_end/installer_utils/check_backend.sh index 3bd96fdc09..535943f435 100644 --- a/tests_end_to_end/installer_utils/check_backend.sh +++ b/tests_end_to_end/installer_utils/check_backend.sh @@ -20,4 +20,4 @@ do done echo "Error: Backend did not respond with 200 OK after $((max_retries * wait_interval)) seconds." -exit 1 \ No newline at end of file +exit 1 diff --git a/tests_end_to_end/installer_utils/check_docker_compose_pods.sh b/tests_end_to_end/installer_utils/check_docker_compose_pods.sh index 5ae51dff06..363369174a 100644 --- a/tests_end_to_end/installer_utils/check_docker_compose_pods.sh +++ b/tests_end_to_end/installer_utils/check_docker_compose_pods.sh @@ -23,4 +23,4 @@ if [[ $retries -eq $max_retries ]]; then echo "Containers failed to start" docker compose ps exit 1 -fi \ No newline at end of file +fi diff --git a/tests_end_to_end/installer_utils/test_app_status.py b/tests_end_to_end/installer_utils/test_app_status.py index d93e940997..9145a4139e 100644 --- a/tests_end_to_end/installer_utils/test_app_status.py +++ b/tests_end_to_end/installer_utils/test_app_status.py @@ -1,6 +1,6 @@ from playwright.sync_api import Page, expect + def test_app_loads(page: Page): page.goto("http://localhost:5173/default/projects") expect(page.get_by_role("heading", name="Projects")).to_be_visible() - diff --git a/tests_end_to_end/page_objects/DatasetItemsPage.py b/tests_end_to_end/page_objects/DatasetItemsPage.py index e1df04f682..04c2243418 100644 --- a/tests_end_to_end/page_objects/DatasetItemsPage.py +++ b/tests_end_to_end/page_objects/DatasetItemsPage.py @@ -1,93 +1,99 @@ -from playwright.sync_api import Page, expect, Locator +from playwright.sync_api import Page + class DatasetItemsPage: def __init__(self, page: Page): self.page = page - self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)') + self.next_page_button_locator = self.page.locator( + "div:has(> button:nth-of-type(4))" + ).locator("button:nth-of-type(3)") def remove_default_columns(self): - self.page.get_by_role('button', name='Columns').click() - created_toggle = self.page.get_by_role('button', name='Created', exact=True).get_by_role('checkbox') + self.page.get_by_role("button", name="Columns").click() + created_toggle = self.page.get_by_role( + "button", name="Created", exact=True + ).get_by_role("checkbox") if created_toggle.is_checked(): created_toggle.click() - - last_updated_toggle = self.page.get_by_role('button', name='Last updated').get_by_role('checkbox') + + last_updated_toggle = self.page.get_by_role( + "button", name="Last updated" + ).get_by_role("checkbox") if last_updated_toggle.is_checked(): last_updated_toggle.click() - created_by_toggle = self.page.get_by_role('button', name='Created by', exact=True).get_by_role('checkbox') + created_by_toggle = self.page.get_by_role( + "button", name="Created by", exact=True + ).get_by_role("checkbox") if created_by_toggle.is_checked(): created_by_toggle.click() - - self.page.keyboard.press('Escape') + self.page.keyboard.press("Escape") def delete_first_item_on_page_and_return_content(self): self.remove_default_columns() - keys: list[str] = self.page.locator('th').all_inner_texts()[1:-1] - item={} + keys: list[str] = self.page.locator("th").all_inner_texts()[1:-1] + item = {} - row = self.page.locator('tr').nth(1) - cells = row.locator('td').all() + row = self.page.locator("tr").nth(1) + cells = row.locator("td").all() for cell_index, cell in enumerate(cells[1:-1]): - content = '' + content = "" if cell_index == 0: - cell.get_by_role('button').hover() - row.get_by_role('button').nth(1).click() - content = self.page.evaluate('navigator.clipboard.readText()') + cell.get_by_role("button").hover() + row.get_by_role("button").nth(1).click() + content = self.page.evaluate("navigator.clipboard.readText()") else: content = cell.text_content() item[keys[cell_index]] = content - - row.get_by_role('button', name='Actions menu').click() - self.page.get_by_role('menuitem', name='Delete').click() - self.page.get_by_role('button', name='Delete dataset item').click() - return item + row.get_by_role("button", name="Actions menu").click() + self.page.get_by_role("menuitem", name="Delete").click() + self.page.get_by_role("button", name="Delete dataset item").click() + return item def insert_dataset_item(self, item: str): - self.page.get_by_role('button', name='Create dataset item').click() - textbox = self.page.get_by_role('textbox') + self.page.get_by_role("button", name="Create dataset item").click() + textbox = self.page.get_by_role("textbox") textbox.focus() - self.page.keyboard.press('Meta+A') - self.page.keyboard.press('Backspace') + self.page.keyboard.press("Meta+A") + self.page.keyboard.press("Backspace") textbox.fill(item) - self.page.get_by_role('button', name='Create dataset item').click() + self.page.get_by_role("button", name="Create dataset item").click() - def get_all_dataset_items_on_current_page(self): self.remove_default_columns() - keys: list[str] = self.page.locator('th').all_inner_texts()[1:-1] + keys: list[str] = self.page.locator("th").all_inner_texts()[1:-1] items = [] - rows = self.page.locator('tr').all() + rows = self.page.locator("tr").all() for row_index, row in enumerate(rows[1:]): item = {} - cells = row.locator('td').all() + cells = row.locator("td").all() for cell_index, cell in enumerate(cells[1:-1]): - content = '' + content = "" if cell_index == 0: - cell.get_by_role('button').hover() - row.get_by_role('button').nth(1).click() - content = self.page.evaluate('navigator.clipboard.readText()') + cell.get_by_role("button").hover() + row.get_by_role("button").nth(1).click() + content = self.page.evaluate("navigator.clipboard.readText()") else: content = cell.text_content() item[keys[cell_index]] = content items.append(item) return items - def get_all_items_in_dataset(self): items = [] items.extend(self.get_all_dataset_items_on_current_page()) - while self.next_page_button_locator.is_visible() and self.next_page_button_locator.is_enabled(): + while ( + self.next_page_button_locator.is_visible() + and self.next_page_button_locator.is_enabled() + ): self.next_page_button_locator.click() self.page.wait_for_timeout(500) items.extend(self.get_all_dataset_items_on_current_page()) return items - - diff --git a/tests_end_to_end/page_objects/DatasetsPage.py b/tests_end_to_end/page_objects/DatasetsPage.py index d3e2c518d4..325f72cb8e 100644 --- a/tests_end_to_end/page_objects/DatasetsPage.py +++ b/tests_end_to_end/page_objects/DatasetsPage.py @@ -1,17 +1,18 @@ from playwright.sync_api import Page, expect + class DatasetsPage: def __init__(self, page: Page): self.page = page - self.url = '/default/datasets' + self.url = "/default/datasets" def go_to_page(self): self.page.goto(self.url) def create_dataset_by_name(self, dataset_name: str): - self.page.get_by_role('button', name='Create new dataset').first.click() - self.page.get_by_placeholder('Dataset name').fill(dataset_name) - self.page.get_by_role('button', name='Create dataset').click() + self.page.get_by_role("button", name="Create new dataset").first.click() + self.page.get_by_placeholder("Dataset name").fill(dataset_name) + self.page.get_by_role("button", name="Create dataset").click() def select_database_by_name(self, name): self.page.get_by_text(name, exact=True).first.click() @@ -22,15 +23,17 @@ def search_dataset(self, dataset_name): def check_dataset_exists_on_page_by_name(self, dataset_name): expect(self.page.get_by_text(dataset_name).first).to_be_visible() - + def check_dataset_not_exists_on_page_by_name(self, dataset_name): expect(self.page.get_by_text(dataset_name).first).not_to_be_visible() - + def delete_dataset_by_name(self, dataset_name): self.search_dataset(dataset_name) - row = self.page.get_by_role('row').filter(has_text=dataset_name).filter(has=self.page.get_by_role('cell', name=dataset_name, exact=True)) + row = ( + self.page.get_by_role("row") + .filter(has_text=dataset_name) + .filter(has=self.page.get_by_role("cell", name=dataset_name, exact=True)) + ) row.get_by_role("button").click() self.page.get_by_role("menuitem", name="Delete").click() self.page.get_by_role("button", name="Delete dataset").click() - - \ No newline at end of file diff --git a/tests_end_to_end/page_objects/ExperimentItemsPage.py b/tests_end_to_end/page_objects/ExperimentItemsPage.py index da6d17bfdd..904ae2ab7b 100644 --- a/tests_end_to_end/page_objects/ExperimentItemsPage.py +++ b/tests_end_to_end/page_objects/ExperimentItemsPage.py @@ -1,53 +1,54 @@ -from playwright.sync_api import Page, expect, Locator +from playwright.sync_api import Page, Locator import re -class ExperimentItemsPage: +class ExperimentItemsPage: def __init__(self, page: Page): self.page = page - self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)') + self.next_page_button_locator = self.page.locator( + "div:has(> button:nth-of-type(4))" + ).locator("button:nth-of-type(3)") def get_pagination_button(self) -> Locator: - return self.page.get_by_role('button', name='Showing') + return self.page.get_by_role("button", name="Showing") def get_total_number_of_items_in_experiment(self): pagination_button_text = self.get_pagination_button().inner_text() - match = re.search(r'of (\d+)', pagination_button_text) + match = re.search(r"of (\d+)", pagination_button_text) if match: return int(match.group(1)) else: return 0 - + def get_id_of_nth_experiment_item(self, n: int): - row = self.page.locator('tr').nth(n+1) - cell = row.locator('td').first + row = self.page.locator("tr").nth(n + 1) + cell = row.locator("td").first cell.hover() - cell.get_by_role('button').nth(1).click() - id = self.page.evaluate('navigator.clipboard.readText()') + cell.get_by_role("button").nth(1).click() + id = self.page.evaluate("navigator.clipboard.readText()") return id - - + def get_all_item_ids_on_current_page(self): ids = [] - rows = self.page.locator('tr').all() + rows = self.page.locator("tr").all() for row_index, row in enumerate(rows[2:]): - item = {} - cells = row.locator('td').all() - cell = row.locator('td').first + cell = row.locator("td").first cell.hover() - cell.get_by_role('button').nth(1).click() - id = self.page.evaluate('navigator.clipboard.readText()') + cell.get_by_role("button").nth(1).click() + id = self.page.evaluate("navigator.clipboard.readText()") ids.append(id) return ids - def get_all_item_ids_in_experiment(self): ids = [] ids.extend(self.get_all_item_ids_on_current_page()) - while self.next_page_button_locator.is_visible() and self.next_page_button_locator.is_enabled(): + while ( + self.next_page_button_locator.is_visible() + and self.next_page_button_locator.is_enabled() + ): self.next_page_button_locator.click() self.page.wait_for_timeout(500) ids.extend(self.get_all_dataset_items_on_current_page()) - return ids \ No newline at end of file + return ids diff --git a/tests_end_to_end/page_objects/ExperimentsPage.py b/tests_end_to_end/page_objects/ExperimentsPage.py index 64666263ee..5081b236f2 100644 --- a/tests_end_to_end/page_objects/ExperimentsPage.py +++ b/tests_end_to_end/page_objects/ExperimentsPage.py @@ -1,10 +1,11 @@ from playwright.sync_api import Page, expect + class ExperimentsPage: def __init__(self, page: Page): self.page = page - self.url = '/default/experiments' - self.search_bar = self.page.get_by_test_id('search-input') + self.url = "/default/experiments" + self.search_bar = self.page.get_by_test_id("search-input") def go_to_page(self): self.page.goto(self.url) @@ -15,18 +16,20 @@ def search_experiment_by_name(self, exp_name: str): def click_first_experiment_that_matches_name(self, exp_name: str): self.search_experiment_by_name(exp_name=exp_name) - self.page.get_by_role('link', name=exp_name).first.click() - + self.page.get_by_role("link", name=exp_name).first.click() + def check_experiment_exists_by_name(self, exp_name: str): self.search_experiment_by_name(exp_name) expect(self.page.get_by_text(exp_name).first).to_be_visible() - + def check_experiment_not_exists_by_name(self, exp_name: str): self.search_experiment_by_name(exp_name) expect(self.page.get_by_text(exp_name)).not_to_be_visible() def delete_experiment_by_name(self, exp_name: str): self.search_experiment_by_name(exp_name) - self.page.get_by_role('row', name=exp_name).first.get_by_role('button', name='Actions menu').click() - self.page.get_by_role('menuitem', name='Delete').click() - self.page.get_by_role('button', name='Delete experiment').click() \ No newline at end of file + self.page.get_by_role("row", name=exp_name).first.get_by_role( + "button", name="Actions menu" + ).click() + self.page.get_by_role("menuitem", name="Delete").click() + self.page.get_by_role("button", name="Delete experiment").click() diff --git a/tests_end_to_end/page_objects/IndividualDatasetPage.py b/tests_end_to_end/page_objects/IndividualDatasetPage.py index 42574315b4..a85ebef391 100644 --- a/tests_end_to_end/page_objects/IndividualDatasetPage.py +++ b/tests_end_to_end/page_objects/IndividualDatasetPage.py @@ -1,9 +1,10 @@ from playwright.sync_api import Page, expect + class IndividualDatasetPage: def __init__(self, page: Page): self.page = page - self.traces_table = page.get_by_role('table') + self.traces_table = page.get_by_role("table") def check_cell_exists_by_text(self, text): - expect(self.traces_table.get_by_text(text, exact=True)).to_be_visible() \ No newline at end of file + expect(self.traces_table.get_by_text(text, exact=True)).to_be_visible() diff --git a/tests_end_to_end/page_objects/ProjectsPage.py b/tests_end_to_end/page_objects/ProjectsPage.py index e65d777fd5..8852915806 100644 --- a/tests_end_to_end/page_objects/ProjectsPage.py +++ b/tests_end_to_end/page_objects/ProjectsPage.py @@ -1,27 +1,32 @@ from playwright.sync_api import Page, expect import time + class ProjectsPage: def __init__(self, page: Page): self.page = page - self.url = '/projects' - self.projects_table = self.page.get_by_role('table') + self.url = "/projects" + self.projects_table = self.page.get_by_role("table") def go_to_page(self): self.page.goto(self.url) def click_project(self, project_name): - self.page.get_by_role('link', name=project_name).click() + self.page.get_by_role("link", name=project_name).click() def search_project(self, project_name): self.page.get_by_test_id("search-input").click() self.page.get_by_test_id("search-input").fill(project_name) def check_project_exists_on_current_page(self, project_name): - expect(self.page.get_by_role('cell', name=project_name, exact=True)).to_be_visible() + expect( + self.page.get_by_role("cell", name=project_name, exact=True) + ).to_be_visible() def check_project_not_exists_on_current_page(self, project_name): - expect(self.page.get_by_role('cell', name=project_name, exact=True)).not_to_be_visible() + expect( + self.page.get_by_role("cell", name=project_name, exact=True) + ).not_to_be_visible() def check_project_exists_on_current_page_with_retry(self, project_name, timeout): start_time = time.time() @@ -32,7 +37,9 @@ def check_project_exists_on_current_page_with_retry(self, project_name, timeout) except AssertionError: self.page.wait_for_timeout(500) else: - raise AssertionError(f'project {project_name} not found in projects list within {timeout} seconds') + raise AssertionError( + f"project {project_name} not found in projects list within {timeout} seconds" + ) def create_new_project(self, project_name): self.page.get_by_role("button", name="Create new project").click() @@ -45,7 +52,11 @@ def create_new_project(self, project_name): def delete_project_by_name(self, project_name): self.search_project(project_name) - row = self.page.get_by_role('row').filter(has_text=project_name).filter(has=self.page.get_by_role('cell', name=project_name, exact=True)) + row = ( + self.page.get_by_role("row") + .filter(has_text=project_name) + .filter(has=self.page.get_by_role("cell", name=project_name, exact=True)) + ) row.get_by_role("button").click() self.page.get_by_role("menuitem", name="Delete").click() - self.page.get_by_role("button", name="Delete project").click() \ No newline at end of file + self.page.get_by_role("button", name="Delete project").click() diff --git a/tests_end_to_end/page_objects/TracesPage.py b/tests_end_to_end/page_objects/TracesPage.py index f9240c522e..aa60917bb2 100644 --- a/tests_end_to_end/page_objects/TracesPage.py +++ b/tests_end_to_end/page_objects/TracesPage.py @@ -1,103 +1,105 @@ from playwright.sync_api import Page, expect, Locator import re + class TracesPage: def __init__(self, page: Page): self.page = page - self.traces_table = self.page.get_by_role('table') - self.trace_names_selector = 'tr td:nth-child(3) div span' - self.trace_id_selector = 'tr:nth-child({}) > td:nth-child(2) > div'.format - self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)') - self.delete_button_locator = self.page.locator("div").filter(has_text=re.compile(r"^Add to dataset$")).get_by_role("button").nth(2) - - + self.traces_table = self.page.get_by_role("table") + self.trace_names_selector = "tr td:nth-child(3) div span" + self.trace_id_selector = "tr:nth-child({}) > td:nth-child(2) > div".format + self.next_page_button_locator = self.page.locator( + "div:has(> button:nth-of-type(4))" + ).locator("button:nth-of-type(3)") + self.delete_button_locator = ( + self.page.locator("div") + .filter(has_text=re.compile(r"^Add to dataset$")) + .get_by_role("button") + .nth(2) + ) def get_all_trace_names_on_page(self): self.page.wait_for_selector(self.trace_names_selector) names = self.page.locator(self.trace_names_selector).all_inner_texts() return names - - - def click_first_trace_that_has_name(self, trace_name: str): - self.page.get_by_role('row').filter(has_text=trace_name).first.get_by_role('button').first.click() - - - def click_nth_trace_on_page(self, n: int): - self.trace_id_selector(n).click() + def click_first_trace_that_has_name(self, trace_name: str): + self.page.get_by_role("row").filter(has_text=trace_name).first.get_by_role( + "button" + ).first.click() def get_first_trace_name_on_page(self): self.page.wait_for_selector(self.trace_names_selector) name = self.page.locator(self.trace_names_selector).first.text_content() return name - def get_all_trace_names_in_project(self): names = [] names.extend(self.get_all_trace_names_on_page()) - while self.next_page_button_locator.is_visible() and self.next_page_button_locator.is_enabled(): + while ( + self.next_page_button_locator.is_visible() + and self.next_page_button_locator.is_enabled() + ): self.next_page_button_locator.click() self.page.wait_for_timeout(500) names.extend(self.get_all_trace_names_on_page()) return names - def get_pagination_button(self) -> Locator: - return self.page.get_by_role('button', name='Showing') - + return self.page.get_by_role("button", name="Showing") def get_number_of_traces_on_page(self): try: - expect(self.page.get_by_role('row').first).to_be_visible() - except Exception as e: + expect(self.page.get_by_role("row").first).to_be_visible() + except Exception: return 0 finally: - return self.page.get_by_role('row').count() - + return self.page.get_by_role("row").count() def get_total_number_of_traces_in_project(self): pagination_button_text = self.get_pagination_button().inner_text() - match = re.search(r'of (\d+)', pagination_button_text) + match = re.search(r"of (\d+)", pagination_button_text) if match: return int(match.group(1)) else: return 0 - def delete_single_trace_by_name(self, name: str): - trace = self.page.get_by_role('row').filter(has_text=name).first - trace.get_by_label('Select row').click() + trace = self.page.get_by_role("row").filter(has_text=name).first + trace.get_by_label("Select row").click() self.delete_button_locator.click() - self.page.get_by_role('button', name='Delete traces').click() - + self.page.get_by_role("button", name="Delete traces").click() def delete_all_traces_that_match_name_contains_filter(self, name: str): - #TODO compact this into smaller functions + # TODO compact this into smaller functions self.page.get_by_role("button", name="Filters").click() - filter_row = self.page.get_by_role('row').filter(has=self.page.get_by_role('cell', name='Where')) - filter_row.get_by_role('cell').filter(has_text=re.compile(r"^Column$")).click() + filter_row = self.page.get_by_role("row").filter( + has=self.page.get_by_role("cell", name="Where") + ) + filter_row.get_by_role("cell").filter(has_text=re.compile(r"^Column$")).click() self.page.get_by_label("Name").click() self.page.get_by_test_id("filter-string-input").click() self.page.get_by_test_id("filter-string-input").fill(name) - expect(self.page.get_by_role('button', name='Filters (1)')).to_be_visible() + expect(self.page.get_by_role("button", name="Filters (1)")).to_be_visible() self.page.keyboard.press(key="Escape") total_traces = self.get_total_number_of_traces_in_project() while total_traces > 0: - expect(self.page.get_by_label('Select all')).to_be_visible() + expect(self.page.get_by_label("Select all")).to_be_visible() self.page.get_by_label("Select all").click() self.delete_button_locator.click() self.page.get_by_role("button", name="Delete traces").click() - + pagination_button = self.get_pagination_button() - expect(pagination_button).not_to_have_text(f'Showing 1-10 of {total_traces}') + expect(pagination_button).not_to_have_text( + f"Showing 1-10 of {total_traces}" + ) total_traces = self.get_total_number_of_traces_in_project() - def add_all_traces_to_new_dataset(self, dataset_name: str): self.page.get_by_label("Select all").click() self.page.get_by_role("button", name="Add to dataset").click() self.page.get_by_role("button", name="Create new dataset").click() self.page.get_by_placeholder("Dataset name").fill(dataset_name) - self.page.get_by_role("button", name="Create dataset").click() \ No newline at end of file + self.page.get_by_role("button", name="Create dataset").click() diff --git a/tests_end_to_end/page_objects/TracesPageSpansMenu.py b/tests_end_to_end/page_objects/TracesPageSpansMenu.py index 853e663230..b5f8cc2b6b 100644 --- a/tests_end_to_end/page_objects/TracesPageSpansMenu.py +++ b/tests_end_to_end/page_objects/TracesPageSpansMenu.py @@ -1,29 +1,30 @@ from playwright.sync_api import Page, expect + class TracesPageSpansMenu: def __init__(self, page: Page): self.page = page - self.input_output_tab = 'Input/Output' - self.feedback_scores_tab = 'Feedback scores' - self.metadata_tab = 'Metadata' + self.input_output_tab = "Input/Output" + self.feedback_scores_tab = "Feedback scores" + self.metadata_tab = "Metadata" def get_first_trace_by_name(self, name): - return self.page.get_by_role('button', name=name).first - + return self.page.get_by_role("button", name=name).first + def get_first_span_by_name(self, name): - return self.page.get_by_role('button', name=name).first + return self.page.get_by_role("button", name=name).first def check_span_exists_by_name(self, name): - expect(self.page.get_by_role('button', name=name)).to_be_visible() + expect(self.page.get_by_role("button", name=name)).to_be_visible() def check_tag_exists_by_name(self, tag_name): expect(self.page.get_by_text(tag_name)).to_be_visible() def get_input_output_tab(self): - return self.page.get_by_role('tab', name=self.input_output_tab) + return self.page.get_by_role("tab", name=self.input_output_tab) def get_feedback_scores_tab(self): - return self.page.get_by_role('tab', name=self.feedback_scores_tab) - + return self.page.get_by_role("tab", name=self.feedback_scores_tab) + def get_metadata_tab(self): - return self.page.get_by_role('tab', name='Metadata') \ No newline at end of file + return self.page.get_by_role("tab", name="Metadata") diff --git a/tests_end_to_end/pyproject.toml b/tests_end_to_end/pyproject.toml new file mode 100644 index 0000000000..50bd747c83 --- /dev/null +++ b/tests_end_to_end/pyproject.toml @@ -0,0 +1,8 @@ +[tool.mypy] +follow_imports = "skip" +ignore_missing_imports = true +explicit_package_bases = true +disallow_untyped_calls = false +disallow_untyped_defs = false +check_untyped_defs = false +disallow_incomplete_defs = false diff --git a/tests_end_to_end/test_requirements.txt b/tests_end_to_end/test_requirements.txt index ffe90ac21b..9c5d36765a 100644 --- a/tests_end_to_end/test_requirements.txt +++ b/tests_end_to_end/test_requirements.txt @@ -1,3 +1,3 @@ pytest playwright -pytest-playwright \ No newline at end of file +pytest-playwright diff --git a/tests_end_to_end/tests/Datasets/__init__.py b/tests_end_to_end/tests/Datasets/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_end_to_end/tests/Datasets/conftest.py b/tests_end_to_end/tests/Datasets/conftest.py index 2c31f219df..280fe2ef3d 100644 --- a/tests_end_to_end/tests/Datasets/conftest.py +++ b/tests_end_to_end/tests/Datasets/conftest.py @@ -1,11 +1,6 @@ import pytest -import os from opik import Opik -from playwright.sync_api import Page -from page_objects.DatasetsPage import DatasetsPage -from page_objects.DatasetItemsPage import DatasetItemsPage -from datasets_utils import TEST_ITEMS -import json +from Datasets.datasets_utils import TEST_ITEMS @pytest.fixture diff --git a/tests_end_to_end/tests/Datasets/datasets_utils.py b/tests_end_to_end/tests/Datasets/datasets_utils.py index 08daea584e..9ba4ff93b7 100644 --- a/tests_end_to_end/tests/Datasets/datasets_utils.py +++ b/tests_end_to_end/tests/Datasets/datasets_utils.py @@ -7,30 +7,30 @@ import opik TEST_ITEMS = [ - {'input': 'input0', 'output': 'output0'}, - {'input': 'input1', 'output': 'output1'}, - {'input': 'input2', 'output': 'output2'}, - {'input': 'input3', 'output': 'output3'}, - {'input': 'input4', 'output': 'output4'}, - {'input': 'input5', 'output': 'output5'}, - {'input': 'input6', 'output': 'output6'}, - {'input': 'input7', 'output': 'output7'}, - {'input': 'input8', 'output': 'output8'}, - {'input': 'input9', 'output': 'output9'} + {"input": "input0", "output": "output0"}, + {"input": "input1", "output": "output1"}, + {"input": "input2", "output": "output2"}, + {"input": "input3", "output": "output3"}, + {"input": "input4", "output": "output4"}, + {"input": "input5", "output": "output5"}, + {"input": "input6", "output": "output6"}, + {"input": "input7", "output": "output7"}, + {"input": "input8", "output": "output8"}, + {"input": "input9", "output": "output9"}, ] TEST_ITEMS_UPDATE = [ - {'input': 'update-input0', 'output': 'update-output0'}, - {'input': 'update-input1', 'output': 'update-output1'}, - {'input': 'update-input2', 'output': 'update-output2'}, - {'input': 'update-input3', 'output': 'update-output3'}, - {'input': 'update-input4', 'output': 'update-output4'}, - {'input': 'update-input5', 'output': 'update-output5'}, - {'input': 'update-input6', 'output': 'update-output6'}, - {'input': 'update-input7', 'output': 'update-output7'}, - {'input': 'update-input8', 'output': 'update-output8'}, - {'input': 'update-input9', 'output': 'update-output9'} + {"input": "update-input0", "output": "update-output0"}, + {"input": "update-input1", "output": "update-output1"}, + {"input": "update-input2", "output": "update-output2"}, + {"input": "update-input3", "output": "update-output3"}, + {"input": "update-input4", "output": "update-output4"}, + {"input": "update-input5", "output": "update-output5"}, + {"input": "update-input6", "output": "update-output6"}, + {"input": "update-input7", "output": "update-output7"}, + {"input": "update-input8", "output": "update-output8"}, + {"input": "update-input9", "output": "update-output9"}, ] @@ -45,7 +45,7 @@ def insert_dataset_items_ui(page: Page, dataset_name, items_list): datasets_page.select_database_by_name(dataset_name) dataset_items_page = DatasetItemsPage(page) - + for item in items_list: dataset_items_page.insert_dataset_item(json.dumps(item)) time.sleep(0.2) @@ -54,7 +54,7 @@ def insert_dataset_items_ui(page: Page, dataset_name, items_list): def delete_one_dataset_item_sdk(client: opik.Opik, dataset_name): dataset = client.get_dataset(dataset_name) item = dataset.get_items()[0] - dataset.delete([item['id']]) + dataset.delete([item["id"]]) return item @@ -80,14 +80,16 @@ def wait_for_dataset_to_be_visible(client: Opik, dataset_name: str, timeout=10): if dataset: break time.sleep(0.5) - + if dataset: return dataset else: raise -def wait_for_number_of_items_in_dataset(expected_items_number: int, dataset, timeout=10): +def wait_for_number_of_items_in_dataset( + expected_items_number: int, dataset, timeout=10 +): expected_number_achieved = False start_time = time.time() items = [] @@ -99,20 +101,25 @@ def wait_for_number_of_items_in_dataset(expected_items_number: int, dataset, tim time.sleep(0.5) if not expected_number_achieved: - raise AssertionError(f'expected to see {expected_items_number} in dataset, instead found {len(items)} after {timeout} seconds of retries') + raise AssertionError( + f"expected to see {expected_items_number} in dataset, instead found {len(items)} after {timeout} seconds of retries" + ) def compare_item_lists(expected: list[dict], actual: list[dict]): set_expected = {frozenset(d.items()) for d in expected} - set_actual = {frozenset((key, val) for key, val in d.items() if 'id' not in key.lower()) for d in actual} + set_actual = { + frozenset((key, val) for key, val in d.items() if "id" not in key.lower()) + for d in actual + } return set_expected == set_actual def get_updated_items(current: list[dict], update: list[dict]): result = [ - {'id': item2['id'], 'input': item1['input'], 'output': item1['output']} + {"id": item2["id"], "input": item1["input"], "output": item1["output"]} for item1, item2 in zip(update, current) ] - return result \ No newline at end of file + return result diff --git a/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py b/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py index 50a3f0def7..710182c6e8 100644 --- a/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py +++ b/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py @@ -1,21 +1,37 @@ import pytest from playwright.sync_api import Page, expect from page_objects.DatasetsPage import DatasetsPage -from page_objects.ProjectsPage import ProjectsPage -from page_objects.TracesPage import TracesPage from page_objects.DatasetItemsPage import DatasetItemsPage -from sdk_helpers import delete_dataset_by_name_if_exists, update_dataset_name, get_dataset_by_name -from datasets_utils import TEST_ITEMS, TEST_ITEMS_UPDATE, compare_item_lists, wait_for_dataset_to_be_visible, get_updated_items, insert_dataset_items_sdk, insert_dataset_items_ui, delete_one_dataset_item_sdk, delete_one_dataset_item_ui, wait_for_number_of_items_in_dataset +from Datasets.datasets_utils import ( + TEST_ITEMS, + TEST_ITEMS_UPDATE, + compare_item_lists, + wait_for_dataset_to_be_visible, + get_updated_items, + insert_dataset_items_sdk, + insert_dataset_items_ui, + delete_one_dataset_item_sdk, + delete_one_dataset_item_ui, + wait_for_number_of_items_in_dataset, +) import opik -import time class TestDatasetItemsCrud: - - @pytest.mark.parametrize('dataset_insert', ['insert_via_ui', 'insert_via_sdk']) - @pytest.mark.parametrize('dataset_creation_fixture', ['create_delete_dataset_sdk', 'create_delete_dataset_ui']) - @pytest.mark.browser_context_args(permissions=['clipboard-read']) - def test_dataset_item_insertion(self, request, page: Page, client: opik.Opik, dataset_creation_fixture, dataset_insert): + @pytest.mark.parametrize("dataset_insert", ["insert_via_ui", "insert_via_sdk"]) + @pytest.mark.parametrize( + "dataset_creation_fixture", + ["create_delete_dataset_sdk", "create_delete_dataset_ui"], + ) + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) + def test_dataset_item_insertion( + self, + request, + page: Page, + client: opik.Opik, + dataset_creation_fixture, + dataset_insert, + ): """ Tests insertion into database in all possible ways of creating dataset and the items themselves (creates 4 test instances), and checks syncing between UI and SDK 1. Create a dataset via either the SDK or the UI @@ -28,18 +44,24 @@ def test_dataset_item_insertion(self, request, page: Page, client: opik.Opik, da - dataset created by UI - items inserted by SDK - check they all appear correctly in both UI and SDK """ - dataset = wait_for_dataset_to_be_visible(client=client, dataset_name=request.getfixturevalue(dataset_creation_fixture), timeout=10) - - if 'ui' in dataset_insert: + dataset = wait_for_dataset_to_be_visible( + client=client, + dataset_name=request.getfixturevalue(dataset_creation_fixture), + timeout=10, + ) + + if "ui" in dataset_insert: insert_dataset_items_ui(page, dataset.name, TEST_ITEMS) - elif 'sdk' in dataset_insert: + elif "sdk" in dataset_insert: insert_dataset_items_sdk(client, dataset.name, TEST_ITEMS) - wait_for_number_of_items_in_dataset(expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=10) + wait_for_number_of_items_in_dataset( + expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=10 + ) items_from_sdk = dataset.get_items() - # CHECK THAT THE ITEMS INSERTED ARE EXACTLY THE ITEMS RETURNED BY THE SDK + # CHECK THAT THE ITEMS INSERTED ARE EXACTLY THE ITEMS RETURNED BY THE SDK assert compare_item_lists(expected=TEST_ITEMS, actual=items_from_sdk) dataset_page = DatasetsPage(page) @@ -51,9 +73,15 @@ def test_dataset_item_insertion(self, request, page: Page, client: opik.Opik, da # CHECK THAT THE ITEMS INSERTED ARE EXACTLY THE ITEMS FOUND IN THE UI assert compare_item_lists(expected=TEST_ITEMS, actual=items_from_ui) - - @pytest.mark.browser_context_args(permissions=['clipboard-read']) - def test_dataset_item_update(self, request, page: Page, client: opik.Opik, create_delete_dataset_sdk, insert_dataset_items_sdk): + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) + def test_dataset_item_update( + self, + request, + page: Page, + client: opik.Opik, + create_delete_dataset_sdk, + insert_dataset_items_sdk, + ): """ Tests updating existing dataset items with new information and the change syncing to both the UI and the SDK 1. Create a dataset via the SDK @@ -61,17 +89,23 @@ def test_dataset_item_update(self, request, page: Page, client: opik.Opik, creat 3. Using dataset.update(), insert new data into the existing dataset items 4. Check that the new data is correct on both the UI and the SDK """ - dataset = wait_for_dataset_to_be_visible(client=client, dataset_name=create_delete_dataset_sdk, timeout=10) + dataset = wait_for_dataset_to_be_visible( + client=client, dataset_name=create_delete_dataset_sdk, timeout=10 + ) - wait_for_number_of_items_in_dataset(expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=15) + wait_for_number_of_items_in_dataset( + expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=15 + ) items_from_sdk = dataset.get_items() - updated_items = get_updated_items(current=items_from_sdk, update=TEST_ITEMS_UPDATE) + updated_items = get_updated_items( + current=items_from_sdk, update=TEST_ITEMS_UPDATE + ) dataset.update(updated_items) items_from_sdk = dataset.get_items() - # CHECK THAT THE ITEMS FROM THE SDK EXACTLY MATCH THE UPDATED DATASET ITEMS DATA + # CHECK THAT THE ITEMS FROM THE SDK EXACTLY MATCH THE UPDATED DATASET ITEMS DATA assert compare_item_lists(expected=TEST_ITEMS_UPDATE, actual=items_from_sdk) dataset_page = DatasetsPage(page) @@ -80,34 +114,46 @@ def test_dataset_item_update(self, request, page: Page, client: opik.Opik, creat dataset_items_page = DatasetItemsPage(page) items_from_ui = dataset_items_page.get_all_items_in_dataset() - # CHECK THAT THE ITEMS FOUND IN THE UI EXACTLY MATCH THE UPDATED DATASET ITEMS DATA + # CHECK THAT THE ITEMS FOUND IN THE UI EXACTLY MATCH THE UPDATED DATASET ITEMS DATA assert compare_item_lists(expected=TEST_ITEMS_UPDATE, actual=items_from_ui) - - @pytest.mark.browser_context_args(permissions=['clipboard-read']) - @pytest.mark.parametrize('item_deletion', ['delete_via_ui', 'delete_via_sdk']) - def test_dataset_item_deletion(self, request, page: Page, client: opik.Opik, create_delete_dataset_sdk, insert_dataset_items_sdk, item_deletion): + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) + @pytest.mark.parametrize("item_deletion", ["delete_via_ui", "delete_via_sdk"]) + def test_dataset_item_deletion( + self, + request, + page: Page, + client: opik.Opik, + create_delete_dataset_sdk, + insert_dataset_items_sdk, + item_deletion, + ): """ Tests deletion of an item via both the UI and the SDK (2 test instances created) and the change being visible in both the UI and the SDK 1. Create a dataset via the SDK 2. Insert 10 items into it via the SDK 3. Using either the UI or the SDK (2 tests), delete one item from the dataset - 4. Check that the item with that data no longer exists in both the SDK and the UI and that the length of the item list is updated + 4. Check that the item with that data no longer exists in both the SDK and the UI and that the length of the item list is updated """ - dataset = wait_for_dataset_to_be_visible(client=client, dataset_name=create_delete_dataset_sdk, timeout=10) - + dataset = wait_for_dataset_to_be_visible( + client=client, dataset_name=create_delete_dataset_sdk, timeout=10 + ) + item_deleted = {} - if 'ui' in item_deletion: + if "ui" in item_deletion: item_deleted = delete_one_dataset_item_ui(page, dataset.name) - elif 'sdk' in item_deletion: + elif "sdk" in item_deletion: item_deleted = delete_one_dataset_item_sdk(client, dataset.name) - wait_for_number_of_items_in_dataset(expected_items_number=len(TEST_ITEMS)-1, dataset=dataset, timeout=15) + wait_for_number_of_items_in_dataset( + expected_items_number=len(TEST_ITEMS) - 1, dataset=dataset, timeout=15 + ) items_from_sdk = dataset.get_items() deleted_item_not_in_sdk_list = not any( - item['input'] == item_deleted['input'] and item['output'] == item_deleted['output'] + item["input"] == item_deleted["input"] + and item["output"] == item_deleted["output"] for item in items_from_sdk ) @@ -123,16 +169,23 @@ def test_dataset_item_deletion(self, request, page: Page, client: opik.Opik, cre assert len(items_from_ui) == len(TEST_ITEMS) - 1 deleted_item_not_in_ui_list = not any( - item['input'] == item_deleted['input'] and item['output'] == item_deleted['output'] + item["input"] == item_deleted["input"] + and item["output"] == item_deleted["output"] for item in items_from_ui ) # CHECK DATA OF DELETED ITEM NO LONGER PRESENT IN DATASET WHEN GETTING ITEMS FROM UI assert deleted_item_not_in_ui_list - - @pytest.mark.browser_context_args(permissions=['clipboard-read']) - def test_dataset_clear(self, request, page: Page, client: opik.Opik, create_delete_dataset_sdk, insert_dataset_items_sdk): + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) + def test_dataset_clear( + self, + request, + page: Page, + client: opik.Opik, + create_delete_dataset_sdk, + insert_dataset_items_sdk, + ): """ Tests mass deletion from the dataset using dataset.clear() 1. Create a dataset via the SDK @@ -140,11 +193,15 @@ def test_dataset_clear(self, request, page: Page, client: opik.Opik, create_dele 3. Deleting every item from the dataset using dataset.clear() 4. Check that no items exist in the dataset when trying to get them via both the SDK and the UI """ - dataset = wait_for_dataset_to_be_visible(client=client, dataset_name=create_delete_dataset_sdk, timeout=10) + dataset = wait_for_dataset_to_be_visible( + client=client, dataset_name=create_delete_dataset_sdk, timeout=10 + ) dataset.clear() # CHECK NO ITEMS RETURNED FROM THE SDK - wait_for_number_of_items_in_dataset(expected_items_number=0, dataset=dataset, timeout=15) + wait_for_number_of_items_in_dataset( + expected_items_number=0, dataset=dataset, timeout=15 + ) dataset_page = DatasetsPage(page) dataset_page.go_to_page() @@ -152,4 +209,6 @@ def test_dataset_clear(self, request, page: Page, client: opik.Opik, create_dele dataset_items_page = DatasetItemsPage(page) # CHECK -DATASET EMPTY- MESSAGE APPEARS, SIGNIFYING AN EMPTY DATASET - expect(dataset_items_page.page.get_by_text('There are no dataset items yet')).to_be_visible() \ No newline at end of file + expect( + dataset_items_page.page.get_by_text("There are no dataset items yet") + ).to_be_visible() diff --git a/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py b/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py index 6fe1b8af27..ae0d99f4f9 100644 --- a/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py +++ b/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py @@ -1,15 +1,18 @@ import pytest -from playwright.sync_api import Page, expect +from playwright.sync_api import Page from page_objects.DatasetsPage import DatasetsPage from page_objects.ProjectsPage import ProjectsPage from page_objects.TracesPage import TracesPage -from sdk_helpers import delete_dataset_by_name_if_exists, update_dataset_name, get_dataset_by_name +from sdk_helpers import ( + delete_dataset_by_name_if_exists, + update_dataset_name, + get_dataset_by_name, +) import opik import time class TestDatasetsCrud: - def test_create_dataset_ui_datasets_page(self, page: Page): """ Basic test to check dataset creation via UI. Uses the UI after creation to check the dataset exists @@ -19,18 +22,21 @@ def test_create_dataset_ui_datasets_page(self, page: Page): """ datasets_page = DatasetsPage(page) datasets_page.go_to_page() - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" try: datasets_page.create_dataset_by_name(dataset_name=dataset_name) - datasets_page.check_dataset_exists_on_page_by_name(dataset_name=dataset_name) + datasets_page.check_dataset_exists_on_page_by_name( + dataset_name=dataset_name + ) except Exception as e: - print(f'error during dataset creation: {e}') + print(f"error during dataset creation: {e}") raise finally: delete_dataset_by_name_if_exists(dataset_name=dataset_name) - - def test_create_dataset_ui_add_traces_to_new_dataset(self, page: Page, create_delete_project_sdk, create_10_test_traces): + def test_create_dataset_ui_add_traces_to_new_dataset( + self, page: Page, create_delete_project_sdk, create_10_test_traces + ): """ Basic test to check dataset creation via "add to new dataset" functionality in the traces page. Uses the UI after creation to check the project exists 1. Create a project with some traces @@ -38,7 +44,7 @@ def test_create_dataset_ui_add_traces_to_new_dataset(self, page: Page, create_de 3. Switch to the datasets page, check the dataset exists in the dataset table 4. If no errors raised and dataset exists, test passes """ - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" proj_name = create_delete_project_sdk projects_page = ProjectsPage(page) projects_page.go_to_page() @@ -46,18 +52,19 @@ def test_create_dataset_ui_add_traces_to_new_dataset(self, page: Page, create_de traces_page = TracesPage(page) traces_page.add_all_traces_to_new_dataset(dataset_name=dataset_name) - + try: datasets_page = DatasetsPage(page) datasets_page.go_to_page() - datasets_page.check_dataset_exists_on_page_by_name(dataset_name=dataset_name) + datasets_page.check_dataset_exists_on_page_by_name( + dataset_name=dataset_name + ) except Exception as e: - print(f'error: dataset not created: {e}') + print(f"error: dataset not created: {e}") raise finally: delete_dataset_by_name_if_exists(dataset_name=dataset_name) - def test_create_dataset_sdk_client(self, client: opik.Opik): """ Basic test to check dataset creation via SDK. Uses the SDK to fetch the created dataset to check it exists @@ -65,20 +72,23 @@ def test_create_dataset_sdk_client(self, client: opik.Opik): 2. Get the project via SDK OpikAPI client 3. If dataset creation fails, client.get_dataset will throw an error and the test will fail. """ - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" try: client.create_dataset(name=dataset_name) time.sleep(0.2) assert client.get_dataset(name=dataset_name) is not None except Exception as e: - print(f'error during dataset creation: {e}') + print(f"error during dataset creation: {e}") raise finally: delete_dataset_by_name_if_exists(dataset_name=dataset_name) - - @pytest.mark.parametrize('dataset_fixture', ['create_delete_dataset_ui', 'create_delete_dataset_sdk']) - def test_dataset_visibility(self, request, page: Page, client: opik.Opik, dataset_fixture): + @pytest.mark.parametrize( + "dataset_fixture", ["create_delete_dataset_ui", "create_delete_dataset_sdk"] + ) + def test_dataset_visibility( + self, request, page: Page, client: opik.Opik, dataset_fixture + ): """ Checks a created dataset is visible via both the UI and SDK. Test split in 2: checks on datasets created on both UI and SDK 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) @@ -95,9 +105,13 @@ def test_dataset_visibility(self, request, page: Page, client: opik.Opik, datase dataset_sdk = client.get_dataset(dataset_name) assert dataset_sdk.name == dataset_name - - @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) - def test_dataset_name_update(self, request, page: Page, client: opik.Opik, dataset_fixture): + @pytest.mark.parametrize( + "dataset_fixture", + ["create_dataset_sdk_no_cleanup", "create_dataset_ui_no_cleanup"], + ) + def test_dataset_name_update( + self, request, page: Page, client: opik.Opik, dataset_fixture + ): """ Checks using the SDK update method on a dataset. Test split into 2: checks on dataset created on both UI and SDK 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) @@ -107,7 +121,7 @@ def test_dataset_name_update(self, request, page: Page, client: opik.Opik, datas """ dataset_name = request.getfixturevalue(dataset_fixture) time.sleep(0.5) - new_name = 'updated_test_dataset_name' + new_name = "updated_test_dataset_name" name_updated = False try: @@ -116,16 +130,18 @@ def test_dataset_name_update(self, request, page: Page, client: opik.Opik, datas dataset_new_name = get_dataset_by_name(dataset_name=new_name) - dataset_id_updated_name = dataset_new_name['id'] + dataset_id_updated_name = dataset_new_name["id"] assert dataset_id_updated_name == dataset_id datasets_page = DatasetsPage(page) datasets_page.go_to_page() datasets_page.check_dataset_exists_on_page_by_name(dataset_name=new_name) - datasets_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) + datasets_page.check_dataset_not_exists_on_page_by_name( + dataset_name=dataset_name + ) except Exception as e: - print(f'Error occured during update of project name: {e}') + print(f"Error occured during update of project name: {e}") raise finally: @@ -134,9 +150,13 @@ def test_dataset_name_update(self, request, page: Page, client: opik.Opik, datas else: delete_dataset_by_name_if_exists(dataset_name) - - @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) - def test_dataset_deletion_in_sdk(self, request, page: Page, client: opik.Opik, dataset_fixture): + @pytest.mark.parametrize( + "dataset_fixture", + ["create_dataset_sdk_no_cleanup", "create_dataset_ui_no_cleanup"], + ) + def test_dataset_deletion_in_sdk( + self, request, page: Page, client: opik.Opik, dataset_fixture + ): """ Checks proper deletion of a dataset via the SDK. Test split into 2: checks on datasets created on both UI and SDK 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) @@ -151,16 +171,20 @@ def test_dataset_deletion_in_sdk(self, request, page: Page, client: opik.Opik, d dataset_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) try: _ = client.get_dataset(dataset_name) - assert False, f'datasets {dataset_name} somehow still exists after deletion' + assert False, f"datasets {dataset_name} somehow still exists after deletion" except Exception as e: - if '404' in str(e) or 'not found' in str(e).lower(): + if "404" in str(e) or "not found" in str(e).lower(): pass else: raise - - @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) - def test_dataset_deletion_in_ui(self, request, page: Page, client: opik.Opik, dataset_fixture): + @pytest.mark.parametrize( + "dataset_fixture", + ["create_dataset_sdk_no_cleanup", "create_dataset_ui_no_cleanup"], + ) + def test_dataset_deletion_in_ui( + self, request, page: Page, client: opik.Opik, dataset_fixture + ): """ Checks proper deletion of a dataset via the SDK. Test split into 2: checks on datasets created on both UI and SDK 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) @@ -176,9 +200,9 @@ def test_dataset_deletion_in_ui(self, request, page: Page, client: opik.Opik, da try: _ = client.get_dataset(dataset_name) - assert False, f'datasets {dataset_name} somehow still exists after deletion' + assert False, f"datasets {dataset_name} somehow still exists after deletion" except Exception as e: - if '404' in str(e) or 'not found' in str(e).lower(): + if "404" in str(e) or "not found" in str(e).lower(): pass else: raise @@ -186,4 +210,3 @@ def test_dataset_deletion_in_ui(self, request, page: Page, client: opik.Opik, da dataset_page = DatasetsPage(page) dataset_page.go_to_page() dataset_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) - diff --git a/tests_end_to_end/tests/Experiments/__init__.py b/tests_end_to_end/tests/Experiments/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_end_to_end/tests/Experiments/conftest.py b/tests_end_to_end/tests/Experiments/conftest.py index eebed161c6..5356699b42 100644 --- a/tests_end_to_end/tests/Experiments/conftest.py +++ b/tests_end_to_end/tests/Experiments/conftest.py @@ -1,41 +1,31 @@ import pytest -import os from opik import Opik -from playwright.sync_api import Page -from page_objects.DatasetsPage import DatasetsPage -from page_objects.DatasetItemsPage import DatasetItemsPage -from page_objects.ExperimentsPage import ExperimentsPage from opik.evaluation.metrics import Contains from opik.evaluation import evaluate from sdk_helpers import delete_experiment_by_id -import json def eval_task(item: dict): - return { - 'input': item['input'], - 'output': item['output'], - 'reference': 'output' - } + return {"input": item["input"], "output": item["output"], "reference": "output"} + @pytest.fixture() def mock_experiment(client: Opik, create_delete_dataset_sdk, insert_dataset_items_sdk): dataset = client.get_dataset(create_delete_dataset_sdk) - experiment_name='test_experiment' + experiment_name = "test_experiment" eval = evaluate( experiment_name=experiment_name, dataset=dataset, task=eval_task, - scoring_metrics=[Contains()] + scoring_metrics=[Contains()], ) yield { - 'id': eval.experiment_id, - 'name': experiment_name, - 'size': len(dataset.get_items()) + "id": eval.experiment_id, + "name": experiment_name, + "size": len(dataset.get_items()), } try: delete_experiment_by_id(eval.experiment_id) except Exception as e: - print(f'Experiment cleanup error: {e}') + print(f"Experiment cleanup error: {e}") pass - \ No newline at end of file diff --git a/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py b/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py index 51eb2b12f3..d7c4d30c44 100644 --- a/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py +++ b/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py @@ -1,14 +1,10 @@ import pytest -from playwright.sync_api import Page, expect -from page_objects.DatasetsPage import DatasetsPage +from playwright.sync_api import Page from page_objects.ExperimentsPage import ExperimentsPage from sdk_helpers import get_experiment_by_id, delete_experiment_by_id -import opik -import time class TestExperimentsCrud: - def test_experiment_visibility(self, page: Page, mock_experiment): """ Tests experiment creation and visibility of experiment in both UI and SDK @@ -18,13 +14,12 @@ def test_experiment_visibility(self, page: Page, mock_experiment): experiments_page = ExperimentsPage(page) experiments_page.go_to_page() - experiments_page.check_experiment_exists_by_name(mock_experiment['name']) + experiments_page.check_experiment_exists_by_name(mock_experiment["name"]) - experiment_sdk = get_experiment_by_id(mock_experiment['id']) - assert experiment_sdk.name == mock_experiment['name'] + experiment_sdk = get_experiment_by_id(mock_experiment["id"]) + assert experiment_sdk.name == mock_experiment["name"] - - @pytest.mark.parametrize('deletion_method', ['ui', 'sdk']) + @pytest.mark.parametrize("deletion_method", ["ui", "sdk"]) def test_experiment_deletion(self, page: Page, mock_experiment, deletion_method): """ Tests deletion of experiment via both the UI and the SDK and checks experiment correctly no longer appears @@ -32,24 +27,22 @@ def test_experiment_deletion(self, page: Page, mock_experiment, deletion_method) 2. Delete the experiment via either the UI or the SDK (2 separate test entitites) 3. Check the experiment does not appear in the UI and that requesting it via the API correctly returns a 404 """ - if deletion_method == 'ui': + if deletion_method == "ui": experiments_page = ExperimentsPage(page) experiments_page.go_to_page() - experiments_page.delete_experiment_by_name(mock_experiment['name']) - elif deletion_method == 'sdk': - delete_experiment_by_id(mock_experiment['id']) + experiments_page.delete_experiment_by_name(mock_experiment["name"]) + elif deletion_method == "sdk": + delete_experiment_by_id(mock_experiment["id"]) experiments_page = ExperimentsPage(page) experiments_page.go_to_page() - experiments_page.check_experiment_not_exists_by_name(mock_experiment['name']) + experiments_page.check_experiment_not_exists_by_name(mock_experiment["name"]) try: - _ = get_experiment_by_id(mock_experiment['id']) - assert False, f'experiment {mock_experiment['name']} somehow still exists after deletion' + _ = get_experiment_by_id(mock_experiment["id"]) + assert False, f"experiment {mock_experiment['name']} somehow still exists after deletion" except Exception as e: - if '404' in str(e) or 'not found' in str(e).lower(): + if "404" in str(e) or "not found" in str(e).lower(): pass else: raise - - \ No newline at end of file diff --git a/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py index a1937f44c7..bce3c8b570 100644 --- a/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py +++ b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py @@ -1,17 +1,17 @@ import pytest -from playwright.sync_api import Page, expect -from page_objects.DatasetsPage import DatasetsPage +from playwright.sync_api import Page from page_objects.ExperimentsPage import ExperimentsPage from page_objects.ExperimentItemsPage import ExperimentItemsPage -from sdk_helpers import get_experiment_by_id, delete_experiment_by_id, delete_experiment_items_by_id, experiment_items_stream -import opik -import time +from sdk_helpers import ( + get_experiment_by_id, + delete_experiment_items_by_id, + experiment_items_stream, +) from collections import Counter class TestExperimentItemsCrud: - - @pytest.mark.browser_context_args(permissions=['clipboard-read']) + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) def test_all_experiment_items_created(self, page: Page, mock_experiment): """ Creates an experiment with 10 experiment items, then checks that all items are visible in both UI and backend @@ -24,22 +24,26 @@ def test_all_experiment_items_created(self, page: Page, mock_experiment): """ experiments_page = ExperimentsPage(page) experiments_page.go_to_page() - experiments_page.click_first_experiment_that_matches_name(exp_name=mock_experiment['name']) + experiments_page.click_first_experiment_that_matches_name( + exp_name=mock_experiment["name"] + ) experiment_items_page = ExperimentItemsPage(page) items_on_page = experiment_items_page.get_total_number_of_items_in_experiment() - assert items_on_page == mock_experiment['size'] + assert items_on_page == mock_experiment["size"] - experiment_backend = get_experiment_by_id(mock_experiment['id']) - assert experiment_backend.trace_count == mock_experiment['size'] + experiment_backend = get_experiment_by_id(mock_experiment["id"]) + assert experiment_backend.trace_count == mock_experiment["size"] - ids_on_backend = [item['dataset_item_id'] for item in experiment_items_stream(mock_experiment['name'])] + ids_on_backend = [ + item["dataset_item_id"] + for item in experiment_items_stream(mock_experiment["name"]) + ] ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() assert Counter(ids_on_backend) == Counter(ids_on_frontend) - - @pytest.mark.browser_context_args(permissions=['clipboard-read']) + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) def test_delete_experiment_items(self, page: Page, mock_experiment): """ Deletes a single experiment item and checks that everything gets updated on both the UI and the backend @@ -54,25 +58,27 @@ def test_delete_experiment_items(self, page: Page, mock_experiment): """ experiments_page = ExperimentsPage(page) experiments_page.go_to_page() - experiments_page.click_first_experiment_that_matches_name(exp_name=mock_experiment['name']) + experiments_page.click_first_experiment_that_matches_name( + exp_name=mock_experiment["name"] + ) - id_to_delete = experiment_items_stream(exp_name=mock_experiment['name'], limit=1)[0]['id'] + id_to_delete = experiment_items_stream( + exp_name=mock_experiment["name"], limit=1 + )[0]["id"] delete_experiment_items_by_id(ids=[id_to_delete]) experiment_items_page = ExperimentItemsPage(page) experiment_items_page.page.reload() items_on_page = experiment_items_page.get_total_number_of_items_in_experiment() - assert items_on_page == mock_experiment['size'] - 1 + assert items_on_page == mock_experiment["size"] - 1 - experiment_sdk = get_experiment_by_id(mock_experiment['id']) - assert experiment_sdk.trace_count == mock_experiment['size'] - 1 + experiment_sdk = get_experiment_by_id(mock_experiment["id"]) + assert experiment_sdk.trace_count == mock_experiment["size"] - 1 - ids_on_backend = [item['dataset_item_id'] for item in experiment_items_stream(mock_experiment['name'])] + ids_on_backend = [ + item["dataset_item_id"] + for item in experiment_items_stream(mock_experiment["name"]) + ] ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() assert Counter(ids_on_backend) == Counter(ids_on_frontend) - - - - - diff --git a/tests_end_to_end/tests/Projects/__init__.py b/tests_end_to_end/tests/Projects/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_end_to_end/tests/Projects/test_projects_crud_operations.py b/tests_end_to_end/tests/Projects/test_projects_crud_operations.py index 379f19b22e..e1f47bcd62 100644 --- a/tests_end_to_end/tests/Projects/test_projects_crud_operations.py +++ b/tests_end_to_end/tests/Projects/test_projects_crud_operations.py @@ -1,6 +1,12 @@ import pytest from playwright.sync_api import Page -from sdk_helpers import create_project_sdk, find_project_by_name_sdk, delete_project_by_name_sdk, wait_for_project_to_be_visible, update_project_by_name_sdk +from sdk_helpers import ( + create_project_sdk, + find_project_by_name_sdk, + delete_project_by_name_sdk, + wait_for_project_to_be_visible, + update_project_by_name_sdk, +) from page_objects.ProjectsPage import ProjectsPage @@ -13,23 +19,22 @@ def test_project_creation_via_sdk(self): 3. If no errors raised and name of fetched project matches, test passes """ - project_name = 'test_project_creation_via_sdk' - + project_name = "test_project_creation_via_sdk" + try: create_project_sdk(name=project_name) wait_for_project_to_be_visible(project_name, timeout=10) - + check_project = find_project_by_name_sdk(project_name) - assert check_project[0]['name'] == project_name - + assert check_project[0]["name"] == project_name + except Exception as e: - print(f'error occured during creation test: {e}') + print(f"error occured during creation test: {e}") raise finally: delete_project_by_name_sdk(project_name) - def test_project_creation_via_ui(self, page: Page): """ Basic test to check project creation via UI. Uses the UI after creation to check the project exists @@ -37,30 +42,33 @@ def test_project_creation_via_ui(self, page: Page): 2. Check the project exists in the projects table 3. If no errors raised, test passes """ - project_name = 'test_project_creation_via_ui' + project_name = "test_project_creation_via_ui" projects_page = ProjectsPage(page) project_created = False try: projects_page.go_to_page() projects_page.create_new_project(project_name=project_name) - projects_page.check_project_exists_on_current_page(project_name=project_name) + projects_page.check_project_exists_on_current_page( + project_name=project_name + ) project_created = True assert project_created - + except Exception as e: - print(f'error during project creation or verification: {e}') + print(f"error during project creation or verification: {e}") raise finally: if project_created: - try: + try: projects_page.delete_project_by_name(project_name=project_name) except Exception as e: - print(f'error during cleanup: {e}') - + print(f"error during cleanup: {e}") - @pytest.mark.parametrize('project_fixture', ['create_delete_project_ui', 'create_delete_project_sdk']) + @pytest.mark.parametrize( + "project_fixture", ["create_delete_project_ui", "create_delete_project_sdk"] + ) def test_project_visibility(self, request, page: Page, project_fixture): """ Checks a created project is visible via both the UI and SDK. Checks on projects created on both UI and SDK @@ -74,14 +82,18 @@ def test_project_visibility(self, request, page: Page, project_fixture): projects_match = find_project_by_name_sdk(project_name) assert len(projects_match) > 0 - assert projects_match[0]['name'] == project_name + assert projects_match[0]["name"] == project_name projects_page = ProjectsPage(page) projects_page.go_to_page() - projects_page.check_project_exists_on_current_page_with_retry(project_name=project_name, timeout=5) - - - @pytest.mark.parametrize('project_fixture', ['create_project_sdk_no_cleanup', 'create_project_ui_no_cleanup']) + projects_page.check_project_exists_on_current_page_with_retry( + project_name=project_name, timeout=5 + ) + + @pytest.mark.parametrize( + "project_fixture", + ["create_project_sdk_no_cleanup", "create_project_ui_no_cleanup"], + ) def test_project_name_update(self, request, page: Page, project_fixture): """ Checks using the SDK update method on a project. Checks on projects created on both UI and SDK @@ -90,28 +102,34 @@ def test_project_name_update(self, request, page: Page, project_fixture): 3. Check on both the SDK and the UI that the project has been renamed (on SDK: check project ID matches. on UI: check project with new name appears and no project with old name appears) """ - + project_name = request.getfixturevalue(project_fixture) - new_name = 'updated_test_project_name' + new_name = "updated_test_project_name" name_updated = False try: - project_id = update_project_by_name_sdk(name=project_name, new_name=new_name) + project_id = update_project_by_name_sdk( + name=project_name, new_name=new_name + ) name_updated = True wait_for_project_to_be_visible(new_name, timeout=10) projects_match = find_project_by_name_sdk(new_name) - project_id_updated_name = projects_match[0]['id'] + project_id_updated_name = projects_match[0]["id"] assert project_id_updated_name == project_id projects_page = ProjectsPage(page) projects_page.go_to_page() - projects_page.check_project_exists_on_current_page_with_retry(project_name=new_name, timeout=5) - projects_page.check_project_not_exists_on_current_page(project_name=project_name) + projects_page.check_project_exists_on_current_page_with_retry( + project_name=new_name, timeout=5 + ) + projects_page.check_project_not_exists_on_current_page( + project_name=project_name + ) except Exception as e: - print(f'Error occured during update of project name: {e}') + print(f"Error occured during update of project name: {e}") raise finally: @@ -120,8 +138,10 @@ def test_project_name_update(self, request, page: Page, project_fixture): else: delete_project_by_name_sdk(project_name) - - @pytest.mark.parametrize('project_fixture', ['create_project_sdk_no_cleanup', 'create_project_ui_no_cleanup']) + @pytest.mark.parametrize( + "project_fixture", + ["create_project_sdk_no_cleanup", "create_project_ui_no_cleanup"], + ) def test_project_deletion_in_sdk(self, request, page: Page, project_fixture): """ Checks proper deletion of a project via the SDK. Checks on projects created on both UI and SDK @@ -134,13 +154,17 @@ def test_project_deletion_in_sdk(self, request, page: Page, project_fixture): projects_page = ProjectsPage(page) projects_page.go_to_page() - projects_page.check_project_not_exists_on_current_page(project_name=project_name) + projects_page.check_project_not_exists_on_current_page( + project_name=project_name + ) projects_found = find_project_by_name_sdk(project_name) assert len(projects_found) == 0 - - @pytest.mark.parametrize('project_fixture', ['create_project_sdk_no_cleanup', 'create_project_ui_no_cleanup']) + @pytest.mark.parametrize( + "project_fixture", + ["create_project_sdk_no_cleanup", "create_project_ui_no_cleanup"], + ) def test_project_deletion_in_ui(self, request, page: Page, project_fixture): """ Checks proper deletion of a project via the UI. Checks on projects created on both UI and SDK @@ -155,7 +179,9 @@ def test_project_deletion_in_ui(self, request, page: Page, project_fixture): projects_page = ProjectsPage(page) projects_page.go_to_page() - projects_page.check_project_not_exists_on_current_page(project_name=project_name) + projects_page.check_project_not_exists_on_current_page( + project_name=project_name + ) projects_found = find_project_by_name_sdk(project_name) - assert len(projects_found) == 0 \ No newline at end of file + assert len(projects_found) == 0 diff --git a/tests_end_to_end/tests/Traces/__init__.py b/tests_end_to_end/tests/Traces/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_end_to_end/tests/Traces/conftest.py b/tests_end_to_end/tests/Traces/conftest.py index bc9834e5ca..fb4d9845ea 100644 --- a/tests_end_to_end/tests/Traces/conftest.py +++ b/tests_end_to_end/tests/Traces/conftest.py @@ -1,46 +1,45 @@ import pytest -import os from opik import opik_context, track -from traces_config import PREFIX, PROJECT_NAME -from sdk_helpers import create_traces_sdk, get_traces_of_project_sdk, delete_list_of_traces_sdk, wait_for_traces_to_be_visible, wait_for_number_of_traces_to_be_visible +from Traces.traces_config import PREFIX, PROJECT_NAME +from sdk_helpers import wait_for_number_of_traces_to_be_visible -@pytest.fixture(scope='function') -def log_x_traces_with_one_span_via_decorator(traces_number): +@pytest.fixture(scope="function") +def log_x_traces_with_one_span_via_decorator(traces_number): @track def f2(input: str): - return 'test output' - + return "test output" + @track def f1(input: str): - opik_context.update_current_trace(name=PREFIX+str(i)) + opik_context.update_current_trace(name=PREFIX + str(i)) return f2(input) for i in range(traces_number): - f1('test input') + f1("test input") yield -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def log_x_traces_with_one_span_via_client(client, traces_number): for i in range(traces_number): client_trace = client.trace( - name=PREFIX+str(i), + name=PREFIX + str(i), project_name=PROJECT_NAME, - input={'input': 'test input'}, - output={'output': 'test output'}, + input={"input": "test input"}, + output={"output": "test output"}, ) - client_span = client_trace.span( - name='span', - input={'input': 'test input'}, - output={'output': 'test output'} + _ = client_trace.span( + name="span", input={"input": "test input"}, output={"output": "test output"} ) - wait_for_number_of_traces_to_be_visible(project_name=PROJECT_NAME, number_of_traces=traces_number) + wait_for_number_of_traces_to_be_visible( + project_name=PROJECT_NAME, number_of_traces=traces_number + ) yield - -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def create_traces(request, traces_number): - create = request.getfixturevalue(request.param) + _ = request.getfixturevalue(request.param) yield 0 diff --git a/tests_end_to_end/tests/Traces/test_traces_crud_operations.py b/tests_end_to_end/tests/Traces/test_traces_crud_operations.py index bf4e6e6576..415cb32e57 100644 --- a/tests_end_to_end/tests/Traces/test_traces_crud_operations.py +++ b/tests_end_to_end/tests/Traces/test_traces_crud_operations.py @@ -2,22 +2,34 @@ from playwright.sync_api import Page, expect from page_objects.ProjectsPage import ProjectsPage from page_objects.TracesPage import TracesPage -from traces_config import PREFIX +from Traces.traces_config import PREFIX from collections import Counter -from sdk_helpers import get_traces_of_project_sdk, delete_list_of_traces_sdk, wait_for_traces_to_be_visible +from sdk_helpers import ( + get_traces_of_project_sdk, + delete_list_of_traces_sdk, + wait_for_traces_to_be_visible, +) class TestTracesCrud: - - @pytest.mark.parametrize('traces_number', [1, 15]) - @pytest.mark.parametrize('create_traces', ['log_x_traces_with_one_span_via_decorator', 'log_x_traces_with_one_span_via_client'], indirect=True) - def test_trace_creation(self, page: Page, traces_number, create_delete_project_sdk, create_traces): + @pytest.mark.parametrize("traces_number", [1, 15]) + @pytest.mark.parametrize( + "create_traces", + [ + "log_x_traces_with_one_span_via_decorator", + "log_x_traces_with_one_span_via_client", + ], + indirect=True, + ) + def test_trace_creation( + self, page: Page, traces_number, create_delete_project_sdk, create_traces + ): """Testing basic creation of traces via both decorator and low-level client. Test case is split into 4, creating 1 and then 15 traces using both the decorator and the client respectively 1. Create a new project 2. Create the traces using one of the creation methods, following the naming convention of "test-trace-X", where X is from 1 to 25 (so all have unique names) - no errors should occur - 3. In the UI, check that the presented number of traces in the project matches the number of traces created in the test case + 3. In the UI, check that the presented number of traces in the project matches the number of traces created in the test case """ project_name = create_delete_project_sdk projects_page = ProjectsPage(page) @@ -29,14 +41,22 @@ def test_trace_creation(self, page: Page, traces_number, create_delete_project_s traces_created = traces_page.get_total_number_of_traces_in_project() assert traces_created == traces_number - - @pytest.mark.parametrize('traces_number', [25]) - @pytest.mark.parametrize('create_traces', ['log_x_traces_with_one_span_via_decorator', 'log_x_traces_with_one_span_via_client'], indirect=True) - def test_traces_visibility(self, page: Page, traces_number, create_delete_project_sdk, create_traces): + @pytest.mark.parametrize("traces_number", [25]) + @pytest.mark.parametrize( + "create_traces", + [ + "log_x_traces_with_one_span_via_decorator", + "log_x_traces_with_one_span_via_client", + ], + indirect=True, + ) + def test_traces_visibility( + self, page: Page, traces_number, create_delete_project_sdk, create_traces + ): """ Testing visibility within the UI and SDK of traces created via both the decorator and the client Test case is split into 2, creating traces via decorator first, and then via the low level client - + 1. Create a new project 2. Create 25 traces via either the decorator or the client, following the naming convention of "test-trace-X", where X is from 1 to 25 (so all have unique names) 3. Check all the traces are visible in the UI: @@ -53,18 +73,28 @@ def test_traces_visibility(self, page: Page, traces_number, create_delete_projec traces_page = TracesPage(page) _ = create_traces - created_trace_names = Counter([PREFIX+str(i) for i in range(traces_number)]) + created_trace_names = Counter([PREFIX + str(i) for i in range(traces_number)]) traces_ui = traces_page.get_all_trace_names_in_project() assert Counter(traces_ui) == created_trace_names - traces_sdk = get_traces_of_project_sdk(project_name=project_name, size=traces_number) - traces_sdk_names = [trace['name'] for trace in traces_sdk] + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] assert Counter(traces_sdk_names) == created_trace_names - - @pytest.mark.parametrize('traces_number', [10]) - @pytest.mark.parametrize('create_traces', ['log_x_traces_with_one_span_via_decorator', 'log_x_traces_with_one_span_via_client'], indirect=True) - def test_delete_traces_sdk(self, page: Page, traces_number, create_delete_project_sdk, create_traces): + @pytest.mark.parametrize("traces_number", [10]) + @pytest.mark.parametrize( + "create_traces", + [ + "log_x_traces_with_one_span_via_decorator", + "log_x_traces_with_one_span_via_client", + ], + indirect=True, + ) + def test_delete_traces_sdk( + self, page: Page, traces_number, create_delete_project_sdk, create_traces + ): """ Testing trace deletion via the SDK API client (v1/private/traces/delete endpoint) Test case is split into 2, creating traces via the decorator first, then via the client @@ -79,28 +109,46 @@ def test_delete_traces_sdk(self, page: Page, traces_number, create_delete_projec _ = create_traces wait_for_traces_to_be_visible(project_name=project_name, size=traces_number) - traces_sdk = get_traces_of_project_sdk(project_name=project_name, size=traces_number) - traces_sdk_names_ids = [{'id': trace['id'], 'name': trace['name']} for trace in traces_sdk] + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names_ids = [ + {"id": trace["id"], "name": trace["name"]} for trace in traces_sdk + ] traces_to_delete = traces_sdk_names_ids[0:2] - delete_list_of_traces_sdk(trace['id'] for trace in traces_to_delete) + delete_list_of_traces_sdk(trace["id"] for trace in traces_to_delete) projects_page = ProjectsPage(page) projects_page.go_to_page() projects_page.click_project(project_name) traces_page = TracesPage(page) - expect(traces_page.page.get_by_role('row', name=traces_to_delete[0]['name'])).not_to_be_visible() - expect(traces_page.page.get_by_role('row', name=traces_to_delete[1]['name'])).not_to_be_visible() - - traces_sdk = get_traces_of_project_sdk(project_name=project_name, size=traces_number) - traces_sdk_names = [trace['name'] for trace in traces_sdk] + expect( + traces_page.page.get_by_role("row", name=traces_to_delete[0]["name"]) + ).not_to_be_visible() + expect( + traces_page.page.get_by_role("row", name=traces_to_delete[1]["name"]) + ).not_to_be_visible() + + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] assert all(name not in traces_sdk_names for name in traces_to_delete) - - @pytest.mark.parametrize('traces_number', [10]) - @pytest.mark.parametrize('create_traces', ['log_x_traces_with_one_span_via_decorator', 'log_x_traces_with_one_span_via_client'], indirect=True) - def test_delete_traces_ui(self, page: Page, traces_number, create_delete_project_sdk, create_traces): + @pytest.mark.parametrize("traces_number", [10]) + @pytest.mark.parametrize( + "create_traces", + [ + "log_x_traces_with_one_span_via_decorator", + "log_x_traces_with_one_span_via_client", + ], + indirect=True, + ) + def test_delete_traces_ui( + self, page: Page, traces_number, create_delete_project_sdk, create_traces + ): """ Testing trace deletion via the UI Test case is split into 2, creating traces via the decorator first, then via the client @@ -118,19 +166,27 @@ def test_delete_traces_ui(self, page: Page, traces_number, create_delete_project traces_page = TracesPage(page) _ = create_traces - traces_sdk = get_traces_of_project_sdk(project_name=project_name, size=traces_number) - traces_sdk_names = [trace['name'] for trace in traces_sdk] + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] traces_to_delete = traces_sdk_names[0:2] traces_page.delete_single_trace_by_name(traces_to_delete[0]) traces_page.delete_single_trace_by_name(traces_to_delete[1]) traces_page.page.wait_for_timeout(200) - expect(traces_page.page.get_by_role('row', name=traces_to_delete[0])).not_to_be_visible() - expect(traces_page.page.get_by_role('row', name=traces_to_delete[1])).not_to_be_visible() + expect( + traces_page.page.get_by_role("row", name=traces_to_delete[0]) + ).not_to_be_visible() + expect( + traces_page.page.get_by_role("row", name=traces_to_delete[1]) + ).not_to_be_visible() wait_for_traces_to_be_visible(project_name=project_name, size=traces_number) - traces_sdk = get_traces_of_project_sdk(project_name=project_name, size=traces_number) - traces_sdk_names = [trace['name'] for trace in traces_sdk] + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] - assert all(name not in traces_sdk_names for name in traces_to_delete) \ No newline at end of file + assert all(name not in traces_sdk_names for name in traces_to_delete) diff --git a/tests_end_to_end/tests/Traces/traces_config.py b/tests_end_to_end/tests/Traces/traces_config.py index 972600ec3c..6ef082cbcf 100644 --- a/tests_end_to_end/tests/Traces/traces_config.py +++ b/tests_end_to_end/tests/Traces/traces_config.py @@ -1,2 +1,2 @@ -PREFIX='test-trace-' -PROJECT_NAME='automated_tests_project' \ No newline at end of file +PREFIX = "test-trace-" +PROJECT_NAME = "automated_tests_project" diff --git a/tests_end_to_end/tests/application_sanity/__init__.py b/tests_end_to_end/tests/application_sanity/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_end_to_end/tests/application_sanity/conftest.py b/tests_end_to_end/tests/application_sanity/conftest.py index 092a85aacc..392a76b866 100644 --- a/tests_end_to_end/tests/application_sanity/conftest.py +++ b/tests_end_to_end/tests/application_sanity/conftest.py @@ -3,36 +3,33 @@ import opik import yaml import json -from opik.configurator.configure import configure from opik.evaluation import evaluate from opik.evaluation.metrics import Contains, Equals from opik import opik_context, track -from playwright.sync_api import Page -from page_objects.ProjectsPage import ProjectsPage -from page_objects.TracesPage import TracesPage -from page_objects.DatasetsPage import DatasetsPage -from page_objects.ExperimentsPage import ExperimentsPage - -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def config(): curr_dir = os.path.dirname(__file__) - config_path = os.path.join(curr_dir, 'sanity_config.yaml') + config_path = os.path.join(curr_dir, "sanity_config.yaml") - with open(config_path, 'r') as f: + with open(config_path, "r") as f: conf = yaml.safe_load(f) - - os.environ['OPIK_PROJECT_NAME'] = conf['project']['name'] + + os.environ["OPIK_PROJECT_NAME"] = conf["project"]["name"] return conf -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def client_sanity(config): - return opik.Opik(project_name=config['project']['name'], workspace='default', host='http://localhost:5173/api') + return opik.Opik( + project_name=config["project"]["name"], + workspace="default", + host="http://localhost:5173/api", + ) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def log_traces_and_spans_low_level(client_sanity, config): """ Log 5 traces with spans and subspans using the low level Opik client @@ -40,43 +37,49 @@ def log_traces_and_spans_low_level(client_sanity, config): """ trace_config = { - 'count': config['traces']['count'], - 'prefix': config['traces']['client']['prefix'], - 'tags': config['traces']['client']['tags'], - 'metadata': config['traces']['client']['metadata'], - 'feedback_scores': [{'name': key, 'value': value} for key, value in config['traces']['client']['feedback-scores'].items()] + "count": config["traces"]["count"], + "prefix": config["traces"]["client"]["prefix"], + "tags": config["traces"]["client"]["tags"], + "metadata": config["traces"]["client"]["metadata"], + "feedback_scores": [ + {"name": key, "value": value} + for key, value in config["traces"]["client"]["feedback-scores"].items() + ], } span_config = { - 'count': config['spans']['count'], - 'prefix': config['spans']['client']['prefix'], - 'tags': config['spans']['client']['tags'], - 'metadata': config['spans']['client']['metadata'], - 'feedback_scores': [{'name': key, 'value': value} for key, value in config['spans']['client']['feedback-scores'].items()] + "count": config["spans"]["count"], + "prefix": config["spans"]["client"]["prefix"], + "tags": config["spans"]["client"]["tags"], + "metadata": config["spans"]["client"]["metadata"], + "feedback_scores": [ + {"name": key, "value": value} + for key, value in config["spans"]["client"]["feedback-scores"].items() + ], } - for trace_index in range(trace_config['count']): + for trace_index in range(trace_config["count"]): client_trace = client_sanity.trace( - name=trace_config['prefix'] + str(trace_index), - input={'input': f'input-{trace_index}'}, - output={'output': f'output-{trace_index}'}, - tags=trace_config['tags'], - metadata=trace_config['metadata'], - feedback_scores=trace_config['feedback_scores'] + name=trace_config["prefix"] + str(trace_index), + input={"input": f"input-{trace_index}"}, + output={"output": f"output-{trace_index}"}, + tags=trace_config["tags"], + metadata=trace_config["metadata"], + feedback_scores=trace_config["feedback_scores"], ) - for span_index in range(span_config['count']): + for span_index in range(span_config["count"]): client_span = client_trace.span( - name=span_config['prefix'] + str(span_index), - input={'input': f'input-{span_index}'}, - output={'output': f'output-{span_index}'}, - tags=span_config['tags'], - metadata=span_config['metadata'] + name=span_config["prefix"] + str(span_index), + input={"input": f"input-{span_index}"}, + output={"output": f"output-{span_index}"}, + tags=span_config["tags"], + metadata=span_config["metadata"], ) - for score in span_config['feedback_scores']: - client_span.log_feedback_score(name=score['name'], value=score['value']) + for score in span_config["feedback_scores"]: + client_span.log_feedback_score(name=score["name"], value=score["value"]) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def log_traces_and_spans_decorator(config): """ Log 5 traces with spans and subspans using the low level Opik client @@ -84,118 +87,118 @@ def log_traces_and_spans_decorator(config): """ trace_config = { - 'count': config['traces']['count'], - 'prefix': config['traces']['decorator']['prefix'], - 'tags': config['traces']['decorator']['tags'], - 'metadata': config['traces']['decorator']['metadata'], - 'feedback_scores': [{'name': key, 'value': value} for key, value in config['traces']['decorator']['feedback-scores'].items()] + "count": config["traces"]["count"], + "prefix": config["traces"]["decorator"]["prefix"], + "tags": config["traces"]["decorator"]["tags"], + "metadata": config["traces"]["decorator"]["metadata"], + "feedback_scores": [ + {"name": key, "value": value} + for key, value in config["traces"]["decorator"]["feedback-scores"].items() + ], } span_config = { - 'count': config['spans']['count'], - 'prefix': config['spans']['decorator']['prefix'], - 'tags': config['spans']['decorator']['tags'], - 'metadata': config['spans']['decorator']['metadata'], - 'feedback_scores': [{'name': key, 'value': value} for key, value in config['spans']['decorator']['feedback-scores'].items()] + "count": config["spans"]["count"], + "prefix": config["spans"]["decorator"]["prefix"], + "tags": config["spans"]["decorator"]["tags"], + "metadata": config["spans"]["decorator"]["metadata"], + "feedback_scores": [ + {"name": key, "value": value} + for key, value in config["spans"]["decorator"]["feedback-scores"].items() + ], } @track() def make_span(x): opik_context.update_current_span( - name=span_config['prefix'] + str(x), - input={'input': f'input-{x}'}, - metadata=span_config['metadata'], - tags=span_config['tags'], - feedback_scores=span_config['feedback_scores'] + name=span_config["prefix"] + str(x), + input={"input": f"input-{x}"}, + metadata=span_config["metadata"], + tags=span_config["tags"], + feedback_scores=span_config["feedback_scores"], ) - return {'output': f'output-{x}'} - + return {"output": f"output-{x}"} + @track() def make_trace(x): - for spans_no in range(span_config['count']): + for spans_no in range(span_config["count"]): make_span(spans_no) opik_context.update_current_trace( - name=trace_config['prefix'] + str(x), - input={'input': f'input-{x}'}, - metadata=trace_config['metadata'], - tags=trace_config['tags'], - feedback_scores=trace_config['feedback_scores'] + name=trace_config["prefix"] + str(x), + input={"input": f"input-{x}"}, + metadata=trace_config["metadata"], + tags=trace_config["tags"], + feedback_scores=trace_config["feedback_scores"], ) - return {'output': f'output-{x}'} - - for x in range(trace_config['count']): + return {"output": f"output-{x}"} + + for x in range(trace_config["count"]): make_trace(x) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def dataset(config, client_sanity): dataset_config = { - 'name': config['dataset']['name'], - 'filename': config['dataset']['filename'] + "name": config["dataset"]["name"], + "filename": config["dataset"]["filename"], } - dataset = client_sanity.get_or_create_dataset(dataset_config['name']) + dataset = client_sanity.get_or_create_dataset(dataset_config["name"]) curr_dir = os.path.dirname(__file__) - dataset_filepath = os.path.join(curr_dir, dataset_config['filename']) + dataset_filepath = os.path.join(curr_dir, dataset_config["filename"]) dataset.read_jsonl_from_file(dataset_filepath) return dataset -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def create_experiments(config, dataset): exp_config = { - 'prefix': config['experiments']['prefix'], - 'metrics': config['experiments']['metrics'], - 'dataset_name': config['experiments']['dataset-name'] + "prefix": config["experiments"]["prefix"], + "metrics": config["experiments"]["metrics"], + "dataset_name": config["experiments"]["dataset-name"], } def eval_contains(x): return { - 'input': x['input'], - 'output': x['expected_output'], - 'reference': 'hello' + "input": x["input"], + "output": x["expected_output"], + "reference": "hello", } def eval_equals(x): return { - 'input': x['input'], - 'output': x['expected_output'], - 'reference': 'goodbye' + "input": x["input"], + "output": x["expected_output"], + "reference": "goodbye", } - contains_metric = Contains( - name='Contains', - case_sensitive=False - ) - equals_metric = Equals( - name='Equals', - case_sensitive=False - ) + contains_metric = Contains(name="Contains", case_sensitive=False) + equals_metric = Equals(name="Equals", case_sensitive=False) evaluate( - experiment_name=exp_config['prefix'] + 'Contains', + experiment_name=exp_config["prefix"] + "Contains", dataset=dataset, task=eval_contains, - scoring_metrics=[contains_metric] + scoring_metrics=[contains_metric], ) evaluate( - experiment_name=exp_config['prefix'] + 'Equals', + experiment_name=exp_config["prefix"] + "Equals", dataset=dataset, task=eval_equals, - scoring_metrics=[equals_metric] + scoring_metrics=[equals_metric], ) - -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def dataset_content(config): curr_dir = os.path.dirname(__file__) - dataset_filepath = os.path.join(curr_dir, config['dataset']['filename']) + dataset_filepath = os.path.join(curr_dir, config["dataset"]["filename"]) data = [] - with open(dataset_filepath, 'r') as f: + with open(dataset_filepath, "r") as f: for line in f: data.append(json.loads(line)) - return data \ No newline at end of file + return data diff --git a/tests_end_to_end/tests/application_sanity/sanity_config.yaml b/tests_end_to_end/tests/application_sanity/sanity_config.yaml index 7efebfab0f..904c29fce7 100644 --- a/tests_end_to_end/tests/application_sanity/sanity_config.yaml +++ b/tests_end_to_end/tests/application_sanity/sanity_config.yaml @@ -12,7 +12,7 @@ traces: metadata: c-md1: "val1" c-md2: "val2" - + decorator: prefix: "decorator-trace-" tags: ["d-tag1", "d-tag2"] @@ -53,4 +53,4 @@ dataset: experiments: prefix: "test-experiment-" metrics: ["Equals", "Contains"] - dataset-name: *dataset-name \ No newline at end of file + dataset-name: *dataset-name diff --git a/tests_end_to_end/tests/application_sanity/test_sanity.py b/tests_end_to_end/tests/application_sanity/test_sanity.py index fbf09cb801..764e7363f8 100644 --- a/tests_end_to_end/tests/application_sanity/test_sanity.py +++ b/tests_end_to_end/tests/application_sanity/test_sanity.py @@ -1,136 +1,179 @@ -import pytest -import json - from playwright.sync_api import Page, expect from page_objects.TracesPageSpansMenu import TracesPageSpansMenu from page_objects.IndividualDatasetPage import IndividualDatasetPage -def test_project_name(projects_page_timeout, log_traces_and_spans_decorator, log_traces_and_spans_low_level): - ''' +def test_project_name( + projects_page_timeout, + log_traces_and_spans_decorator, + log_traces_and_spans_low_level, +): + """ Checks that the project created via the fixtures exists Does a timeout of 5 seconds to wait for the traces to show up in UI for later tests (TODO: figure out a better way to do this) 1. Open projects page 2. Check the created project exists - ''' - projects_page_timeout.check_project_exists_on_current_page('test-project') + """ + projects_page_timeout.check_project_exists_on_current_page("test-project") -def test_traces_created(traces_page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator): - ''' +def test_traces_created( + traces_page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator +): + """ Checks that every trace defined in the sanity_config file is present in the project 1. Open the traces page of the project 2. Grab all the names of the traces (should never set more than 15 in config so 1 page is safe) 3. Check that every possible name of the traces as defined in sanity_config.yaml is present in the names list - ''' + """ trace_names = traces_page.get_all_trace_names_on_page() - client_prefix = config['traces']['client']['prefix'] - decorator_prefix = config['traces']['decorator']['prefix'] + client_prefix = config["traces"]["client"]["prefix"] + decorator_prefix = config["traces"]["decorator"]["prefix"] - for count in range(config['traces']['count']): + for count in range(config["traces"]["count"]): for prefix in [client_prefix, decorator_prefix]: - assert prefix+str(count) in trace_names + assert prefix + str(count) in trace_names -def test_spans_of_traces(page, traces_page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator): - ''' +def test_spans_of_traces( + page, + traces_page, + config, + log_traces_and_spans_low_level, + log_traces_and_spans_decorator, +): + """ Checks that every trace has the correct number and names of spans defined in the sanity_config.yaml file 1. Open the traces page of the project 2. Go through each trace and click it 3. Check that the spans are present in each trace - ''' + """ trace_names = traces_page.get_all_trace_names_on_page() - traces_page.click_first_trace_that_has_name('decorator-trace-1') + traces_page.click_first_trace_that_has_name("decorator-trace-1") for trace in trace_names: traces_page.click_first_trace_that_has_name(trace) spans_menu = TracesPageSpansMenu(page) - trace_type = trace.split('-')[0] # 'client' or 'decorator' - for count in range(config['spans']['count']): - prefix = config['spans'][trace_type]['prefix'] - spans_menu.check_span_exists_by_name(f'{prefix}{count}') - - -def test_trace_and_span_details(page, traces_page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator): - ''' + trace_type = trace.split("-")[0] # 'client' or 'decorator' + for count in range(config["spans"]["count"]): + prefix = config["spans"][trace_type]["prefix"] + spans_menu.check_span_exists_by_name(f"{prefix}{count}") + + +def test_trace_and_span_details( + page, + traces_page, + config, + log_traces_and_spans_low_level, + log_traces_and_spans_decorator, +): + """ Checks that for each trace and spans, the attributes defined in sanity_config.yaml are present 1. Go through each trace of the project 2. Check the created tags are present 3. Check the created feedback scores are present 4. Check the defined metadata is present 5. Go through each span of the traces and repeat 2-4 - ''' + """ trace_names = traces_page.get_all_trace_names_on_page() for trace in trace_names: traces_page.click_first_trace_that_has_name(trace) spans_menu = TracesPageSpansMenu(page) - trace_type = trace.split('-')[0] - tag_names = config['traces'][trace_type]['tags'] + trace_type = trace.split("-")[0] + tag_names = config["traces"][trace_type]["tags"] for tag in tag_names: spans_menu.check_tag_exists_by_name(tag) spans_menu.get_feedback_scores_tab().click() - for score in config['traces'][trace_type]['feedback-scores']: - expect(page.get_by_role('cell', name=score, exact=True).first).to_be_visible() - expect(page.get_by_role('cell', name=str(config['traces'][trace_type]['feedback-scores'][score]), exact=True).first).to_be_visible() - + for score in config["traces"][trace_type]["feedback-scores"]: + expect( + page.get_by_role("cell", name=score, exact=True).first + ).to_be_visible() + expect( + page.get_by_role( + "cell", + name=str(config["traces"][trace_type]["feedback-scores"][score]), + exact=True, + ).first + ).to_be_visible() + spans_menu.get_metadata_tab().click() - for md_key in config['traces'][trace_type]['metadata']: - expect(page.get_by_text(f'{md_key}: {config['traces'][trace_type]['metadata'][md_key]}')).to_be_visible() - - for count in range(config['spans']['count']): - prefix = config['spans'][trace_type]['prefix'] - spans_menu.get_first_span_by_name(f'{prefix}{count}').click() + for md_key in config["traces"][trace_type]["metadata"]: + expect( + page.get_by_text( + f"{md_key}: {config['traces'][trace_type]['metadata'][md_key]}" + ) + ).to_be_visible() + + for count in range(config["spans"]["count"]): + prefix = config["spans"][trace_type]["prefix"] + spans_menu.get_first_span_by_name(f"{prefix}{count}").click() spans_menu.get_feedback_scores_tab().click() - for score in config['spans'][trace_type]['feedback-scores']: - expect(page.get_by_role('cell', name=score, exact=True)).to_be_visible() - expect(page.get_by_role('cell', name=str(config['spans'][trace_type]['feedback-scores'][score]), exact=True)).to_be_visible() - + for score in config["spans"][trace_type]["feedback-scores"]: + expect(page.get_by_role("cell", name=score, exact=True)).to_be_visible() + expect( + page.get_by_role( + "cell", + name=str(config["spans"][trace_type]["feedback-scores"][score]), + exact=True, + ) + ).to_be_visible() + spans_menu.get_metadata_tab().click() - for md_key in config['spans'][trace_type]['metadata']: - expect(page.get_by_text(f'{md_key}: {config['spans'][trace_type]['metadata'][md_key]}')).to_be_visible() + for md_key in config["spans"][trace_type]["metadata"]: + expect( + page.get_by_text( + f"{md_key}: {config['spans'][trace_type]['metadata'][md_key]}" + ) + ).to_be_visible() # provisional patchy solution, sometimes when clicking through spans very fast some of them show up as "no data" and the test fails page.wait_for_timeout(500) def test_dataset_name(datasets_page, config, dataset): - ''' + """ Checks that the dataset created via the fixture as defined in sanity_config.yaml is present on the datasets page - ''' - datasets_page.check_dataset_exists_on_page_by_name(config['dataset']['name']) + """ + datasets_page.check_dataset_exists_on_page_by_name(config["dataset"]["name"]) def test_dataset_items(page: Page, datasets_page, config, dataset_content): - ''' + """ Checks that the traces created via the fixture and defined in sanity_dataset.jsonl are present within the dataset - ''' - datasets_page.select_database_by_name(config['dataset']['name']) + """ + datasets_page.select_database_by_name(config["dataset"]["name"]) individual_dataset_page = IndividualDatasetPage(page) for item in dataset_content: - individual_dataset_page.check_cell_exists_by_text(item['input']) - individual_dataset_page.check_cell_exists_by_text(item['expected_output']) - + individual_dataset_page.check_cell_exists_by_text(item["input"]) + individual_dataset_page.check_cell_exists_by_text(item["expected_output"]) + def test_experiments_exist(experiments_page, config, create_experiments): - ''' + """ Checks that the experiments created via the fixture are present and have the correct values for the metrics (experiments defined in a way to always return the same results) - ''' - experiments_page.check_experiment_exists_by_name('test-experiment-Equals') - experiments_page.page.get_by_role('link', name='test-experiment-Equals').click() - assert 'Equals\n0' == experiments_page.page.get_by_test_id('feedback-score-tag').first.inner_text() - - experiments_page.page.get_by_role('link', name='Experiments').click() - - experiments_page.check_experiment_exists_by_name('test-experiment-Contains') - experiments_page.page.get_by_role('link', name='test-experiment-Contains').click() - assert 'Contains\n1' == experiments_page.page.get_by_test_id('feedback-score-tag').first.inner_text() \ No newline at end of file + """ + experiments_page.check_experiment_exists_by_name("test-experiment-Equals") + experiments_page.page.get_by_role("link", name="test-experiment-Equals").click() + assert ( + "Equals\n0" + == experiments_page.page.get_by_test_id("feedback-score-tag").first.inner_text() + ) + + experiments_page.page.get_by_role("link", name="Experiments").click() + + experiments_page.check_experiment_exists_by_name("test-experiment-Contains") + experiments_page.page.get_by_role("link", name="test-experiment-Contains").click() + assert ( + "Contains\n1" + == experiments_page.page.get_by_test_id("feedback-score-tag").first.inner_text() + ) diff --git a/tests_end_to_end/tests/conftest.py b/tests_end_to_end/tests/conftest.py index e2b130057a..d0819dfef2 100644 --- a/tests_end_to_end/tests/conftest.py +++ b/tests_end_to_end/tests/conftest.py @@ -1,26 +1,23 @@ import pytest import os import opik -import yaml -import json -from opik.configurator.configure import configure -from opik.evaluation import evaluate -from opik.evaluation.metrics import Contains, Equals -from opik import opik_context, track -from playwright.sync_api import Page, BrowserContext, Browser -import time +from playwright.sync_api import Page, Browser from page_objects.ProjectsPage import ProjectsPage from page_objects.TracesPage import TracesPage from page_objects.DatasetsPage import DatasetsPage from page_objects.ExperimentsPage import ExperimentsPage -from tests.sdk_helpers import create_project_sdk, delete_project_by_name_sdk, wait_for_number_of_traces_to_be_visible, delete_dataset_by_name_if_exists +from tests.sdk_helpers import ( + create_project_sdk, + delete_project_by_name_sdk, + wait_for_number_of_traces_to_be_visible, +) from utils import TEST_ITEMS @pytest.fixture def browser_clipboard_permissions(browser: Browser): context = browser.new_context() - context.grant_permissions(['clipboard-read', 'clipboard-write']) + context.grant_permissions(["clipboard-read", "clipboard-write"]) yield context context.close() @@ -32,26 +29,25 @@ def page_with_clipboard_perms(browser_clipboard_permissions): page.close() - -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def configure_local(): - os.environ['OPIK_URL_OVERRIDE'] = "http://localhost:5173/api" - os.environ['OPIK_WORKSPACE'] = 'default' + os.environ["OPIK_URL_OVERRIDE"] = "http://localhost:5173/api" + os.environ["OPIK_WORKSPACE"] = "default" -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def client() -> opik.Opik: - return opik.Opik(workspace='default', host='http://localhost:5173/api') + return opik.Opik(workspace="default", host="http://localhost:5173/api") -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def projects_page(page: Page): projects_page = ProjectsPage(page) projects_page.go_to_page() return projects_page - -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def projects_page_timeout(page: Page) -> ProjectsPage: projects_page = ProjectsPage(page) projects_page.go_to_page() @@ -59,39 +55,38 @@ def projects_page_timeout(page: Page) -> ProjectsPage: return projects_page -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def traces_page(page: Page, projects_page, config): - projects_page.click_project(config['project']['name']) + projects_page.click_project(config["project"]["name"]) traces_page = TracesPage(page) return traces_page -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def datasets_page(page: Page): datasets_page = DatasetsPage(page) datasets_page.go_to_page() return datasets_page -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def experiments_page(page: Page): experiments_page = ExperimentsPage(page) experiments_page.go_to_page() return experiments_page - -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_project_sdk_no_cleanup(): - proj_name = 'projects_crud_tests_sdk' + proj_name = "projects_crud_tests_sdk" create_project_sdk(name=proj_name) yield proj_name -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_project_ui_no_cleanup(page: Page): - proj_name = 'projects_crud_tests_ui' + proj_name = "projects_crud_tests_ui" projects_page = ProjectsPage(page) projects_page.go_to_page() projects_page.create_new_project(project_name=proj_name) @@ -99,19 +94,19 @@ def create_project_ui_no_cleanup(page: Page): yield proj_name -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_delete_project_sdk(): - proj_name = 'automated_tests_project' + proj_name = "automated_tests_project" create_project_sdk(name=proj_name) - os.environ['OPIK_PROJECT_NAME'] = proj_name + os.environ["OPIK_PROJECT_NAME"] = proj_name yield proj_name delete_project_by_name_sdk(name=proj_name) @pytest.fixture def create_delete_project_ui(page: Page): - proj_name = 'automated_tests_project' + proj_name = "automated_tests_project" projects_page = ProjectsPage(page) projects_page.go_to_page() projects_page.create_new_project(project_name=proj_name) @@ -120,35 +115,35 @@ def create_delete_project_ui(page: Page): delete_project_by_name_sdk(name=proj_name) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_delete_dataset_sdk(client: opik.Opik): - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" client.create_dataset(name=dataset_name) yield dataset_name client.delete_dataset(name=dataset_name) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_delete_dataset_ui(page: Page, client: opik.Opik): - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" datasets_page = DatasetsPage(page) datasets_page.go_to_page() datasets_page.create_dataset_by_name(dataset_name=dataset_name) - + yield dataset_name client.delete_dataset(name=dataset_name) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_dataset_sdk_no_cleanup(client: opik.Opik): - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" client.create_dataset(name=dataset_name) yield dataset_name -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_dataset_ui_no_cleanup(page: Page): - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" datasets_page = DatasetsPage(page) datasets_page.go_to_page() datasets_page.create_dataset_by_name(dataset_name=dataset_name) @@ -160,15 +155,16 @@ def insert_dataset_items_sdk(client: opik.Opik, create_delete_dataset_sdk): dataset = client.get_dataset(create_delete_dataset_sdk) dataset.insert(TEST_ITEMS) + @pytest.fixture def create_10_test_traces(page: Page, client, create_delete_project_sdk): proj_name = create_delete_project_sdk for i in range(10): - client_trace = client.trace( - name=f'trace{i}', + _ = client.trace( + name=f"trace{i}", project_name=proj_name, - input={'input': 'test input'}, - output={'output': 'test output'}, + input={"input": "test input"}, + output={"output": "test output"}, ) wait_for_number_of_traces_to_be_visible(project_name=proj_name, number_of_traces=10) - yield \ No newline at end of file + yield diff --git a/tests_end_to_end/tests/sdk_helpers.py b/tests_end_to_end/tests/sdk_helpers.py index 239f2c3c56..33357f45e8 100644 --- a/tests_end_to_end/tests/sdk_helpers.py +++ b/tests_end_to_end/tests/sdk_helpers.py @@ -1,13 +1,7 @@ -import pytest -import os import time import datetime as dt -import opik -from playwright.sync_api import Page -from page_objects.DatasetItemsPage import DatasetItemsPage -from page_objects.DatasetsPage import DatasetsPage from opik.rest_api.client import OpikApi -from opik import track +from typing import Optional import json @@ -19,34 +13,36 @@ def create_project_sdk(name: str): def find_project_by_name_sdk(name: str): client = OpikApi() proj_page = client.projects.find_projects(name=name, page=1, size=1) - return proj_page.dict()['content'] + return proj_page.dict()["content"] def delete_project_by_name_sdk(name: str): client = OpikApi() project = find_project_by_name_sdk(name=name) - client.projects.delete_project_by_id(project[0]['id']) + client.projects.delete_project_by_id(project[0]["id"]) def wait_for_project_to_be_visible(project_name, timeout=10, initial_delay=1): start_time = time.time() delay = initial_delay - + while time.time() - start_time < timeout: if find_project_by_name_sdk(project_name): return True - + time.sleep(delay) - delay = min(delay*2, timeout-(time.time() - start_time)) - - raise TimeoutError(f'could not get created project {project_name} via API within {timeout} seconds') + delay = min(delay * 2, timeout - (time.time() - start_time)) + + raise TimeoutError( + f"could not get created project {project_name} via API within {timeout} seconds" + ) def update_project_by_name_sdk(name: str, new_name: str): client = OpikApi() wait_for_project_to_be_visible(name, timeout=10) projects_match = find_project_by_name_sdk(name) - project_id = projects_match[0]['id'] + project_id = projects_match[0]["id"] client.projects.update_project(id=project_id, name=new_name) @@ -57,45 +53,53 @@ def create_traces_sdk(prefix: str, project_name: str, qty: int): client = OpikApi() for i in range(qty): client.traces.create_trace( - name=prefix+str(i), + name=prefix + str(i), project_name=project_name, - start_time=dt.datetime.now() + start_time=dt.datetime.now(), ) def wait_for_traces_to_be_visible(project_name, size, timeout=10, initial_delay=1): start_time = time.time() delay = initial_delay - + while time.time() - start_time < timeout: if get_traces_of_project_sdk(project_name=project_name, size=size): return True - + time.sleep(delay) - delay = min(delay*2, timeout-(time.time() - start_time)) - - raise TimeoutError(f'could not get traces of project {project_name} via API within {timeout} seconds') + delay = min(delay * 2, timeout - (time.time() - start_time)) + + raise TimeoutError( + f"could not get traces of project {project_name} via API within {timeout} seconds" + ) -def wait_for_number_of_traces_to_be_visible(project_name, number_of_traces, timeout=10, initial_delay=1): +def wait_for_number_of_traces_to_be_visible( + project_name, number_of_traces, timeout=10, initial_delay=1 +): start_time = time.time() delay = initial_delay - + while time.time() - start_time < timeout: - traces = get_traces_of_project_sdk(project_name=project_name, size=number_of_traces) + traces = get_traces_of_project_sdk( + project_name=project_name, size=number_of_traces + ) if len(traces) >= number_of_traces: return True - + time.sleep(delay) - delay = min(delay*2, timeout-(time.time() - start_time)) - - raise TimeoutError(f'could not get {number_of_traces} traces of project {project_name} via API within {timeout} seconds') + delay = min(delay * 2, timeout - (time.time() - start_time)) + + raise TimeoutError( + f"could not get {number_of_traces} traces of project {project_name} via API within {timeout} seconds" + ) def get_traces_of_project_sdk(project_name: str, size: int): client = OpikApi() traces = client.traces.get_traces_by_project(project_name=project_name, size=size) - return traces.dict()['content'] + return traces.dict()["content"] def delete_list_of_traces_sdk(ids: list[str]): @@ -105,7 +109,9 @@ def delete_list_of_traces_sdk(ids: list[str]): def update_trace_by_id(id: str): client = OpikApi() - client.traces.update_trace(id=id, ) + client.traces.update_trace( + id=id, + ) def get_dataset_by_name(dataset_name: str): @@ -117,7 +123,7 @@ def get_dataset_by_name(dataset_name: str): def update_dataset_name(name: str, new_name: str): client = OpikApi() dataset = get_dataset_by_name(dataset_name=name) - dataset_id = dataset['id'] + dataset_id = dataset["id"] dataset = client.datasets.update_dataset(id=dataset_id, name=new_name) @@ -129,8 +135,8 @@ def delete_dataset_by_name_if_exists(dataset_name: str): dataset = None try: dataset = get_dataset_by_name(dataset_name) - except Exception as e: - print(f'Trying to delete dataset {dataset_name}, but it does not exist') + except Exception as _: + print(f"Trying to delete dataset {dataset_name}, but it does not exist") finally: if dataset: client.datasets.delete_dataset_by_name(dataset_name=dataset_name) @@ -141,17 +147,24 @@ def get_experiment_by_id(exp_id: str): exp = client.experiments.get_experiment_by_id(exp_id) return exp + def delete_experiment_by_id(exp_id: str): client = OpikApi() client.experiments.delete_experiments_by_id(ids=[exp_id]) + def delete_experiment_items_by_id(ids: list[str]): client = OpikApi() client.experiments.delete_experiment_items(ids=ids) -def experiment_items_stream(exp_name: str, limit: int = None): + +def experiment_items_stream(exp_name: str, limit: Optional[int] = None): client = OpikApi() - data = b''.join(client.experiments.stream_experiment_items(experiment_name=exp_name, request_options={'chunk_size': 100})) - lines = data.decode('utf-8').split('\r\n') + data = b"".join( + client.experiments.stream_experiment_items( + experiment_name=exp_name, request_options={"chunk_size": 100} + ) + ) + lines = data.decode("utf-8").split("\r\n") dict_list = [json.loads(line) for line in lines if line.strip()] - return dict_list \ No newline at end of file + return dict_list diff --git a/tests_end_to_end/tests/utils.py b/tests_end_to_end/tests/utils.py index fc9cb0b560..55c2311d57 100644 --- a/tests_end_to_end/tests/utils.py +++ b/tests_end_to_end/tests/utils.py @@ -1,26 +1,26 @@ TEST_ITEMS = [ - {'input': 'input0', 'output': 'output0'}, - {'input': 'input1', 'output': 'output1'}, - {'input': 'input2', 'output': 'output2'}, - {'input': 'input3', 'output': 'output3'}, - {'input': 'input4', 'output': 'output4'}, - {'input': 'input5', 'output': 'output5'}, - {'input': 'input6', 'output': 'output6'}, - {'input': 'input7', 'output': 'output7'}, - {'input': 'input8', 'output': 'output8'}, - {'input': 'input9', 'output': 'output9'} + {"input": "input0", "output": "output0"}, + {"input": "input1", "output": "output1"}, + {"input": "input2", "output": "output2"}, + {"input": "input3", "output": "output3"}, + {"input": "input4", "output": "output4"}, + {"input": "input5", "output": "output5"}, + {"input": "input6", "output": "output6"}, + {"input": "input7", "output": "output7"}, + {"input": "input8", "output": "output8"}, + {"input": "input9", "output": "output9"}, ] TEST_ITEMS_UPDATE = [ - {'input': 'update-input0', 'output': 'update-output0'}, - {'input': 'update-input1', 'output': 'update-output1'}, - {'input': 'update-input2', 'output': 'update-output2'}, - {'input': 'update-input3', 'output': 'update-output3'}, - {'input': 'update-input4', 'output': 'update-output4'}, - {'input': 'update-input5', 'output': 'update-output5'}, - {'input': 'update-input6', 'output': 'update-output6'}, - {'input': 'update-input7', 'output': 'update-output7'}, - {'input': 'update-input8', 'output': 'update-output8'}, - {'input': 'update-input9', 'output': 'update-output9'} -] \ No newline at end of file + {"input": "update-input0", "output": "update-output0"}, + {"input": "update-input1", "output": "update-output1"}, + {"input": "update-input2", "output": "update-output2"}, + {"input": "update-input3", "output": "update-output3"}, + {"input": "update-input4", "output": "update-output4"}, + {"input": "update-input5", "output": "update-output5"}, + {"input": "update-input6", "output": "update-output6"}, + {"input": "update-input7", "output": "update-output7"}, + {"input": "update-input8", "output": "update-output8"}, + {"input": "update-input9", "output": "update-output9"}, +]