From 357fb308d9e432c68099ac0e66b9e5aa6841515f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Wed, 11 Dec 2024 16:48:03 +0200 Subject: [PATCH 01/17] basic experiment items tests --- .../page_objects/ExperimentItemsPage.py | 53 +++++++++++++ .../page_objects/ExperimentsPage.py | 4 + .../tests/Datasets/datasets_utils.py | 1 + .../tests/Experiments/conftest.py | 3 +- .../test_experiment_items_crud_operations.py | 78 +++++++++++++++++++ tests_end_to_end/tests/sdk_helpers.py | 13 +++- 6 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 tests_end_to_end/page_objects/ExperimentItemsPage.py create mode 100644 tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py diff --git a/tests_end_to_end/page_objects/ExperimentItemsPage.py b/tests_end_to_end/page_objects/ExperimentItemsPage.py new file mode 100644 index 0000000000..da6d17bfdd --- /dev/null +++ b/tests_end_to_end/page_objects/ExperimentItemsPage.py @@ -0,0 +1,53 @@ +from playwright.sync_api import Page, expect, Locator +import re + +class ExperimentItemsPage: + + def __init__(self, page: Page): + self.page = page + self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)') + + def get_pagination_button(self) -> Locator: + return self.page.get_by_role('button', name='Showing') + + def get_total_number_of_items_in_experiment(self): + pagination_button_text = self.get_pagination_button().inner_text() + match = re.search(r'of (\d+)', pagination_button_text) + if match: + return int(match.group(1)) + else: + return 0 + + def get_id_of_nth_experiment_item(self, n: int): + row = self.page.locator('tr').nth(n+1) + cell = row.locator('td').first + cell.hover() + cell.get_by_role('button').nth(1).click() + id = self.page.evaluate('navigator.clipboard.readText()') + return id + + + def get_all_item_ids_on_current_page(self): + ids = [] + rows = self.page.locator('tr').all() + for row_index, row in enumerate(rows[2:]): + item = {} + cells = row.locator('td').all() + cell = row.locator('td').first + cell.hover() + cell.get_by_role('button').nth(1).click() + id = self.page.evaluate('navigator.clipboard.readText()') + ids.append(id) + + return ids + + + def get_all_item_ids_in_experiment(self): + ids = [] + ids.extend(self.get_all_item_ids_on_current_page()) + while self.next_page_button_locator.is_visible() and self.next_page_button_locator.is_enabled(): + self.next_page_button_locator.click() + self.page.wait_for_timeout(500) + ids.extend(self.get_all_dataset_items_on_current_page()) + + return ids \ No newline at end of file diff --git a/tests_end_to_end/page_objects/ExperimentsPage.py b/tests_end_to_end/page_objects/ExperimentsPage.py index 1ac41f313c..64666263ee 100644 --- a/tests_end_to_end/page_objects/ExperimentsPage.py +++ b/tests_end_to_end/page_objects/ExperimentsPage.py @@ -12,6 +12,10 @@ def go_to_page(self): def search_experiment_by_name(self, exp_name: str): self.search_bar.click() self.search_bar.fill(exp_name) + + def click_first_experiment_that_matches_name(self, exp_name: str): + self.search_experiment_by_name(exp_name=exp_name) + self.page.get_by_role('link', name=exp_name).first.click() def check_experiment_exists_by_name(self, exp_name: str): self.search_experiment_by_name(exp_name) diff --git a/tests_end_to_end/tests/Datasets/datasets_utils.py b/tests_end_to_end/tests/Datasets/datasets_utils.py index 4d902dc659..08daea584e 100644 --- a/tests_end_to_end/tests/Datasets/datasets_utils.py +++ b/tests_end_to_end/tests/Datasets/datasets_utils.py @@ -48,6 +48,7 @@ def insert_dataset_items_ui(page: Page, dataset_name, items_list): for item in items_list: dataset_items_page.insert_dataset_item(json.dumps(item)) + time.sleep(0.2) def delete_one_dataset_item_sdk(client: opik.Opik, dataset_name): diff --git a/tests_end_to_end/tests/Experiments/conftest.py b/tests_end_to_end/tests/Experiments/conftest.py index c424b408c7..eebed161c6 100644 --- a/tests_end_to_end/tests/Experiments/conftest.py +++ b/tests_end_to_end/tests/Experiments/conftest.py @@ -30,7 +30,8 @@ def mock_experiment(client: Opik, create_delete_dataset_sdk, insert_dataset_item ) yield { 'id': eval.experiment_id, - 'name': experiment_name + 'name': experiment_name, + 'size': len(dataset.get_items()) } try: delete_experiment_by_id(eval.experiment_id) diff --git a/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py new file mode 100644 index 0000000000..a1937f44c7 --- /dev/null +++ b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py @@ -0,0 +1,78 @@ +import pytest +from playwright.sync_api import Page, expect +from page_objects.DatasetsPage import DatasetsPage +from page_objects.ExperimentsPage import ExperimentsPage +from page_objects.ExperimentItemsPage import ExperimentItemsPage +from sdk_helpers import get_experiment_by_id, delete_experiment_by_id, delete_experiment_items_by_id, experiment_items_stream +import opik +import time +from collections import Counter + + +class TestExperimentItemsCrud: + + @pytest.mark.browser_context_args(permissions=['clipboard-read']) + def test_all_experiment_items_created(self, page: Page, mock_experiment): + """ + Creates an experiment with 10 experiment items, then checks that all items are visible in both UI and backend + 1. Create an experiment on a dataset with 10 items (mock_experiment fixture) + 2. Check the item counter on the UI displays the correct total (10 items) + 3. Check the 'trace_count' parameter of the experiment as returned via the v1/private/experiments/{id} endpoint + matches the size of the dataset (10 items) + 4. Check the list of IDs displayed in the UI (currently dataset item IDs) perfectly matches the list of dataset item IDs + as returned from the v1/private/experiments/items/stream endpoint (easy change to grab the items via the SDK if we ever add this) + """ + experiments_page = ExperimentsPage(page) + experiments_page.go_to_page() + experiments_page.click_first_experiment_that_matches_name(exp_name=mock_experiment['name']) + + experiment_items_page = ExperimentItemsPage(page) + items_on_page = experiment_items_page.get_total_number_of_items_in_experiment() + assert items_on_page == mock_experiment['size'] + + experiment_backend = get_experiment_by_id(mock_experiment['id']) + assert experiment_backend.trace_count == mock_experiment['size'] + + ids_on_backend = [item['dataset_item_id'] for item in experiment_items_stream(mock_experiment['name'])] + ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() + + assert Counter(ids_on_backend) == Counter(ids_on_frontend) + + + @pytest.mark.browser_context_args(permissions=['clipboard-read']) + def test_delete_experiment_items(self, page: Page, mock_experiment): + """ + Deletes a single experiment item and checks that everything gets updated on both the UI and the backend + 1. Create an experiment on a dataset with 10 items (mock_experiment fixture) + 2. Grabbing an experiment ID from the v1/private/experiments/items/stream endpoint, send a delete request to delete + a single experiment item from the experiment + 3. Check the item counter in the UI is updated (to size(initial_experiment) - 1) + 4. Check the 'trace_count' parameter of the experiment as returned via the v1/private/experiments/{id} endpoint + is updated to the new size (as above) + 5. Check the list of IDs displayed in the UI (currently dataset item IDs) perfectly matches the list of dataset item IDs + as returned from the v1/private/experiments/items/stream endpoint (easy change to grab the items via the SDK if we ever add this) + """ + experiments_page = ExperimentsPage(page) + experiments_page.go_to_page() + experiments_page.click_first_experiment_that_matches_name(exp_name=mock_experiment['name']) + + id_to_delete = experiment_items_stream(exp_name=mock_experiment['name'], limit=1)[0]['id'] + delete_experiment_items_by_id(ids=[id_to_delete]) + + experiment_items_page = ExperimentItemsPage(page) + experiment_items_page.page.reload() + items_on_page = experiment_items_page.get_total_number_of_items_in_experiment() + assert items_on_page == mock_experiment['size'] - 1 + + experiment_sdk = get_experiment_by_id(mock_experiment['id']) + assert experiment_sdk.trace_count == mock_experiment['size'] - 1 + + ids_on_backend = [item['dataset_item_id'] for item in experiment_items_stream(mock_experiment['name'])] + ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() + + assert Counter(ids_on_backend) == Counter(ids_on_frontend) + + + + + diff --git a/tests_end_to_end/tests/sdk_helpers.py b/tests_end_to_end/tests/sdk_helpers.py index 7cd483d789..239f2c3c56 100644 --- a/tests_end_to_end/tests/sdk_helpers.py +++ b/tests_end_to_end/tests/sdk_helpers.py @@ -143,4 +143,15 @@ def get_experiment_by_id(exp_id: str): def delete_experiment_by_id(exp_id: str): client = OpikApi() - client.experiments.delete_experiments_by_id(ids=[exp_id]) \ No newline at end of file + client.experiments.delete_experiments_by_id(ids=[exp_id]) + +def delete_experiment_items_by_id(ids: list[str]): + client = OpikApi() + client.experiments.delete_experiment_items(ids=ids) + +def experiment_items_stream(exp_name: str, limit: int = None): + client = OpikApi() + data = b''.join(client.experiments.stream_experiment_items(experiment_name=exp_name, request_options={'chunk_size': 100})) + lines = data.decode('utf-8').split('\r\n') + dict_list = [json.loads(line) for line in lines if line.strip()] + return dict_list \ No newline at end of file From ffde2b11824d96c27e6d4899e846e622954b5c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 11:36:29 +0200 Subject: [PATCH 02/17] linting --- .github/workflows/tests_end_to_end_linter.yml | 24 +++++++++++++++++++ .../page_objects/ExperimentItemsPage.py | 2 +- .../test_experiment_items_crud_operations.py | 5 ---- 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/tests_end_to_end_linter.yml diff --git a/.github/workflows/tests_end_to_end_linter.yml b/.github/workflows/tests_end_to_end_linter.yml new file mode 100644 index 0000000000..d93d722a6f --- /dev/null +++ b/.github/workflows/tests_end_to_end_linter.yml @@ -0,0 +1,24 @@ +--- + name: E2E Tests Linter + run-name: "E2E Tests Linter ${{ github.ref_name }} by @${{ github.actor }}" + on: + pull_request: + paths: + - 'tests_end_to_end/**' + push: + branches: + - 'main' + paths: + - 'tests_end_to_end/**' + jobs: + lint: + runs-on: ubuntu-latest + defaults: + run: + working-directory: tests_end_to_end + steps: + - uses: actions/checkout@v4 + - name: install pre-commit + run: pip install pre-commit + - name: linting + run: pre-commit run --all-files diff --git a/tests_end_to_end/page_objects/ExperimentItemsPage.py b/tests_end_to_end/page_objects/ExperimentItemsPage.py index da6d17bfdd..430b09ee44 100644 --- a/tests_end_to_end/page_objects/ExperimentItemsPage.py +++ b/tests_end_to_end/page_objects/ExperimentItemsPage.py @@ -50,4 +50,4 @@ def get_all_item_ids_in_experiment(self): self.page.wait_for_timeout(500) ids.extend(self.get_all_dataset_items_on_current_page()) - return ids \ No newline at end of file + return ids diff --git a/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py index a1937f44c7..566b0e9a67 100644 --- a/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py +++ b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py @@ -71,8 +71,3 @@ def test_delete_experiment_items(self, page: Page, mock_experiment): ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() assert Counter(ids_on_backend) == Counter(ids_on_frontend) - - - - - From 9ef478c54f83c969ca7c4a19138a39ce6e3db7b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 11:46:16 +0200 Subject: [PATCH 03/17] forgot the rest of the files --- tests_end_to_end/.pre-commit-config.yaml | 24 ++++++++++++++++++++++++ tests_end_to_end/pyproject.toml | 4 ++++ 2 files changed, 28 insertions(+) create mode 100644 tests_end_to_end/.pre-commit-config.yaml create mode 100644 tests_end_to_end/pyproject.toml diff --git a/tests_end_to_end/.pre-commit-config.yaml b/tests_end_to_end/.pre-commit-config.yaml new file mode 100644 index 0000000000..c0ad456ea9 --- /dev/null +++ b/tests_end_to_end/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + files: ^tests_end_to_end + - id: end-of-file-fixer + files: ^tests_end_to_end + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.9 + hooks: + - id: ruff + args: [ --fix, --show-fixes] + files: ^tests_end_to_end + - id: ruff-format + files: ^tests_end_to_end + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.10.0 + hooks: + - id: mypy + files: ^tests_end_to_end + args: [--config-file, sdks/python/pyproject.toml] diff --git a/tests_end_to_end/pyproject.toml b/tests_end_to_end/pyproject.toml new file mode 100644 index 0000000000..777f965cc8 --- /dev/null +++ b/tests_end_to_end/pyproject.toml @@ -0,0 +1,4 @@ +[tool.mypy] +follow_imports = "skip" +ignore_missing_imports = true +check_untyped_defs = true From 678464d74a79aab2ec0494407f3035e0797a0d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 11:50:13 +0200 Subject: [PATCH 04/17] package bases --- tests_end_to_end/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests_end_to_end/pyproject.toml b/tests_end_to_end/pyproject.toml index 777f965cc8..79b45a067b 100644 --- a/tests_end_to_end/pyproject.toml +++ b/tests_end_to_end/pyproject.toml @@ -2,3 +2,4 @@ follow_imports = "skip" ignore_missing_imports = true check_untyped_defs = true +explicit_package_bases = true From b9fd1a028327f0cd2159f16107efc3d755d5be97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 11:54:40 +0200 Subject: [PATCH 05/17] fix check again --- tests_end_to_end/tests/Datasets/__init__.py | 0 tests_end_to_end/tests/Experiments/__init__.py | 0 tests_end_to_end/tests/Projects/__init__.py | 0 tests_end_to_end/tests/Traces/__init__.py | 0 tests_end_to_end/tests/application_sanity/__init__.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests_end_to_end/tests/Datasets/__init__.py create mode 100644 tests_end_to_end/tests/Experiments/__init__.py create mode 100644 tests_end_to_end/tests/Projects/__init__.py create mode 100644 tests_end_to_end/tests/Traces/__init__.py create mode 100644 tests_end_to_end/tests/application_sanity/__init__.py diff --git a/tests_end_to_end/tests/Datasets/__init__.py b/tests_end_to_end/tests/Datasets/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_end_to_end/tests/Experiments/__init__.py b/tests_end_to_end/tests/Experiments/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_end_to_end/tests/Projects/__init__.py b/tests_end_to_end/tests/Projects/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_end_to_end/tests/Traces/__init__.py b/tests_end_to_end/tests/Traces/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests_end_to_end/tests/application_sanity/__init__.py b/tests_end_to_end/tests/application_sanity/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From 100561c05f3261d83cb1644c0684a772ac273787 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 12:02:00 +0200 Subject: [PATCH 06/17] linting errors --- tests_end_to_end/page_objects/ExperimentItemsPage.py | 2 -- .../tests/Experiments/test_experiment_crud_operations.py | 2 +- tests_end_to_end/tests/Traces/conftest.py | 4 ++-- tests_end_to_end/tests/conftest.py | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tests_end_to_end/page_objects/ExperimentItemsPage.py b/tests_end_to_end/page_objects/ExperimentItemsPage.py index 430b09ee44..682d4e39a5 100644 --- a/tests_end_to_end/page_objects/ExperimentItemsPage.py +++ b/tests_end_to_end/page_objects/ExperimentItemsPage.py @@ -31,8 +31,6 @@ def get_all_item_ids_on_current_page(self): ids = [] rows = self.page.locator('tr').all() for row_index, row in enumerate(rows[2:]): - item = {} - cells = row.locator('td').all() cell = row.locator('td').first cell.hover() cell.get_by_role('button').nth(1).click() diff --git a/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py b/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py index 51eb2b12f3..18c333b6be 100644 --- a/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py +++ b/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py @@ -45,7 +45,7 @@ def test_experiment_deletion(self, page: Page, mock_experiment, deletion_method) try: _ = get_experiment_by_id(mock_experiment['id']) - assert False, f'experiment {mock_experiment['name']} somehow still exists after deletion' + assert False, f"experiment {mock_experiment['name']} somehow still exists after deletion" except Exception as e: if '404' in str(e) or 'not found' in str(e).lower(): pass diff --git a/tests_end_to_end/tests/Traces/conftest.py b/tests_end_to_end/tests/Traces/conftest.py index bc9834e5ca..0c4728820b 100644 --- a/tests_end_to_end/tests/Traces/conftest.py +++ b/tests_end_to_end/tests/Traces/conftest.py @@ -31,7 +31,7 @@ def log_x_traces_with_one_span_via_client(client, traces_number): input={'input': 'test input'}, output={'output': 'test output'}, ) - client_span = client_trace.span( + _ = client_trace.span( name='span', input={'input': 'test input'}, output={'output': 'test output'} @@ -42,5 +42,5 @@ def log_x_traces_with_one_span_via_client(client, traces_number): @pytest.fixture(scope='function') def create_traces(request, traces_number): - create = request.getfixturevalue(request.param) + _ = request.getfixturevalue(request.param) yield 0 diff --git a/tests_end_to_end/tests/conftest.py b/tests_end_to_end/tests/conftest.py index e2b130057a..a5be6a72d7 100644 --- a/tests_end_to_end/tests/conftest.py +++ b/tests_end_to_end/tests/conftest.py @@ -164,7 +164,7 @@ def insert_dataset_items_sdk(client: opik.Opik, create_delete_dataset_sdk): def create_10_test_traces(page: Page, client, create_delete_project_sdk): proj_name = create_delete_project_sdk for i in range(10): - client_trace = client.trace( + _ = client.trace( name=f'trace{i}', project_name=proj_name, input={'input': 'test input'}, From b187eba2be8e1e33b7ef1ebfaad71c344ddcac67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 12:05:52 +0200 Subject: [PATCH 07/17] one more f-string --- tests_end_to_end/tests/application_sanity/test_sanity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests_end_to_end/tests/application_sanity/test_sanity.py b/tests_end_to_end/tests/application_sanity/test_sanity.py index fbf09cb801..e29bddf6f1 100644 --- a/tests_end_to_end/tests/application_sanity/test_sanity.py +++ b/tests_end_to_end/tests/application_sanity/test_sanity.py @@ -83,7 +83,7 @@ def test_trace_and_span_details(page, traces_page, config, log_traces_and_spans_ spans_menu.get_metadata_tab().click() for md_key in config['traces'][trace_type]['metadata']: - expect(page.get_by_text(f'{md_key}: {config['traces'][trace_type]['metadata'][md_key]}')).to_be_visible() + expect(page.get_by_text(f"{md_key}: {config['traces'][trace_type]['metadata'][md_key]}")).to_be_visible() for count in range(config['spans']['count']): prefix = config['spans'][trace_type]['prefix'] @@ -96,7 +96,7 @@ def test_trace_and_span_details(page, traces_page, config, log_traces_and_spans_ spans_menu.get_metadata_tab().click() for md_key in config['spans'][trace_type]['metadata']: - expect(page.get_by_text(f'{md_key}: {config['spans'][trace_type]['metadata'][md_key]}')).to_be_visible() + expect(page.get_by_text(f"{md_key}: {config['spans'][trace_type]['metadata'][md_key]}")).to_be_visible() # provisional patchy solution, sometimes when clicking through spans very fast some of them show up as "no data" and the test fails page.wait_for_timeout(500) From 2de5e884d80be94e91365e3b8d14471dda5434fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 12:20:48 +0200 Subject: [PATCH 08/17] temporary leniency for untyped stuff --- tests_end_to_end/pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests_end_to_end/pyproject.toml b/tests_end_to_end/pyproject.toml index 79b45a067b..50bd747c83 100644 --- a/tests_end_to_end/pyproject.toml +++ b/tests_end_to_end/pyproject.toml @@ -1,5 +1,8 @@ [tool.mypy] follow_imports = "skip" ignore_missing_imports = true -check_untyped_defs = true explicit_package_bases = true +disallow_untyped_calls = false +disallow_untyped_defs = false +check_untyped_defs = false +disallow_incomplete_defs = false From f95b8fc073f28bd49099df9eb6c2d3b7239cd2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 12:24:50 +0200 Subject: [PATCH 09/17] oops --- tests_end_to_end/.pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_end_to_end/.pre-commit-config.yaml b/tests_end_to_end/.pre-commit-config.yaml index c0ad456ea9..07d96993fd 100644 --- a/tests_end_to_end/.pre-commit-config.yaml +++ b/tests_end_to_end/.pre-commit-config.yaml @@ -21,4 +21,4 @@ repos: hooks: - id: mypy files: ^tests_end_to_end - args: [--config-file, sdks/python/pyproject.toml] + args: [--config-file, tests_end_to_end/pyproject.toml] From 27991199b78aa8adafa82f3536b83020e0b0c55d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 12:33:10 +0200 Subject: [PATCH 10/17] more fixes --- .github/workflows/tests_end_to_end_linter.yml | 4 ++++ tests_end_to_end/page_objects/TracesPage.py | 4 ---- tests_end_to_end/tests/sdk_helpers.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests_end_to_end_linter.yml b/.github/workflows/tests_end_to_end_linter.yml index d93d722a6f..3558d70bea 100644 --- a/.github/workflows/tests_end_to_end_linter.yml +++ b/.github/workflows/tests_end_to_end_linter.yml @@ -18,6 +18,10 @@ working-directory: tests_end_to_end steps: - uses: actions/checkout@v4 + - name: Install missing type stubs + run: | + python3 -m pip install mypy + mypy --install-types - name: install pre-commit run: pip install pre-commit - name: linting diff --git a/tests_end_to_end/page_objects/TracesPage.py b/tests_end_to_end/page_objects/TracesPage.py index f9240c522e..f915d49eed 100644 --- a/tests_end_to_end/page_objects/TracesPage.py +++ b/tests_end_to_end/page_objects/TracesPage.py @@ -21,10 +21,6 @@ def get_all_trace_names_on_page(self): def click_first_trace_that_has_name(self, trace_name: str): self.page.get_by_role('row').filter(has_text=trace_name).first.get_by_role('button').first.click() - - def click_nth_trace_on_page(self, n: int): - self.trace_id_selector(n).click() - def get_first_trace_name_on_page(self): self.page.wait_for_selector(self.trace_names_selector) diff --git a/tests_end_to_end/tests/sdk_helpers.py b/tests_end_to_end/tests/sdk_helpers.py index 239f2c3c56..0e3eff441d 100644 --- a/tests_end_to_end/tests/sdk_helpers.py +++ b/tests_end_to_end/tests/sdk_helpers.py @@ -8,6 +8,7 @@ from page_objects.DatasetsPage import DatasetsPage from opik.rest_api.client import OpikApi from opik import track +from typing import Optional import json @@ -149,7 +150,7 @@ def delete_experiment_items_by_id(ids: list[str]): client = OpikApi() client.experiments.delete_experiment_items(ids=ids) -def experiment_items_stream(exp_name: str, limit: int = None): +def experiment_items_stream(exp_name: str, limit: Optional[int] = None): client = OpikApi() data = b''.join(client.experiments.stream_experiment_items(experiment_name=exp_name, request_options={'chunk_size': 100})) lines = data.decode('utf-8').split('\r\n') From f04fba09fb2d02a8b20f2b130cec252d4a704a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 12:35:48 +0200 Subject: [PATCH 11/17] more --- .github/workflows/tests_end_to_end_linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_end_to_end_linter.yml b/.github/workflows/tests_end_to_end_linter.yml index 3558d70bea..289473a213 100644 --- a/.github/workflows/tests_end_to_end_linter.yml +++ b/.github/workflows/tests_end_to_end_linter.yml @@ -21,7 +21,7 @@ - name: Install missing type stubs run: | python3 -m pip install mypy - mypy --install-types + mypy tests_end_to_end/ --install-types - name: install pre-commit run: pip install pre-commit - name: linting From c8e4c83bbf9dd7298d7a740171d4a9f0a7061b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 12:37:35 +0200 Subject: [PATCH 12/17] wrong path --- .github/workflows/tests_end_to_end_linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_end_to_end_linter.yml b/.github/workflows/tests_end_to_end_linter.yml index 289473a213..21cdac4240 100644 --- a/.github/workflows/tests_end_to_end_linter.yml +++ b/.github/workflows/tests_end_to_end_linter.yml @@ -21,7 +21,7 @@ - name: Install missing type stubs run: | python3 -m pip install mypy - mypy tests_end_to_end/ --install-types + mypy . --install-types - name: install pre-commit run: pip install pre-commit - name: linting From 1f24c6cb08be3019b48a6c187811a48664a6483c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 12:39:55 +0200 Subject: [PATCH 13/17] running into an interactive prompt --- .github/workflows/tests_end_to_end_linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_end_to_end_linter.yml b/.github/workflows/tests_end_to_end_linter.yml index 21cdac4240..24234df2f7 100644 --- a/.github/workflows/tests_end_to_end_linter.yml +++ b/.github/workflows/tests_end_to_end_linter.yml @@ -21,7 +21,7 @@ - name: Install missing type stubs run: | python3 -m pip install mypy - mypy . --install-types + mypy . --install-types --yes - name: install pre-commit run: pip install pre-commit - name: linting From fb81805ab06f572e1455fb6d47cbfee7d14a4352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 12:48:09 +0200 Subject: [PATCH 14/17] that wasnt working apparently --- .github/workflows/tests_end_to_end_linter.yml | 4 ---- tests_end_to_end/.pre-commit-config.yaml | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests_end_to_end_linter.yml b/.github/workflows/tests_end_to_end_linter.yml index 24234df2f7..d93d722a6f 100644 --- a/.github/workflows/tests_end_to_end_linter.yml +++ b/.github/workflows/tests_end_to_end_linter.yml @@ -18,10 +18,6 @@ working-directory: tests_end_to_end steps: - uses: actions/checkout@v4 - - name: Install missing type stubs - run: | - python3 -m pip install mypy - mypy . --install-types --yes - name: install pre-commit run: pip install pre-commit - name: linting diff --git a/tests_end_to_end/.pre-commit-config.yaml b/tests_end_to_end/.pre-commit-config.yaml index 07d96993fd..e6251a1180 100644 --- a/tests_end_to_end/.pre-commit-config.yaml +++ b/tests_end_to_end/.pre-commit-config.yaml @@ -22,3 +22,4 @@ repos: - id: mypy files: ^tests_end_to_end args: [--config-file, tests_end_to_end/pyproject.toml] + additional_dependencies: ['types-PyYAML'] From d45f4aafc25b43407a134055713a745047caa128 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 13:11:09 +0200 Subject: [PATCH 15/17] linter fixes --- .../installer_utils/check_backend.sh | 2 +- .../check_docker_compose_pods.sh | 2 +- .../installer_utils/test_app_status.py | 2 +- .../page_objects/DatasetItemsPage.py | 84 ++++---- tests_end_to_end/page_objects/DatasetsPage.py | 21 +- .../page_objects/ExperimentItemsPage.py | 39 ++-- .../page_objects/ExperimentsPage.py | 19 +- .../page_objects/IndividualDatasetPage.py | 5 +- tests_end_to_end/page_objects/ProjectsPage.py | 27 ++- tests_end_to_end/page_objects/TracesPage.py | 76 ++++--- .../page_objects/TracesPageSpansMenu.py | 23 +- tests_end_to_end/test_requirements.txt | 2 +- tests_end_to_end/tests/Datasets/conftest.py | 5 - .../tests/Datasets/datasets_utils.py | 63 +++--- .../test_dataset_items_crud_operations.py | 143 +++++++++---- .../Datasets/test_datasets_crud_operations.py | 95 +++++---- .../tests/Experiments/conftest.py | 26 +-- .../test_experiment_crud_operations.py | 31 ++- .../test_experiment_items_crud_operations.py | 51 +++-- .../Projects/test_projects_crud_operations.py | 96 ++++++--- tests_end_to_end/tests/Traces/conftest.py | 35 ++- .../Traces/test_traces_crud_operations.py | 130 +++++++---- .../tests/Traces/traces_config.py | 4 +- .../tests/application_sanity/conftest.py | 201 +++++++++--------- .../application_sanity/sanity_config.yaml | 4 +- .../tests/application_sanity/test_sanity.py | 173 +++++++++------ tests_end_to_end/tests/conftest.py | 86 ++++---- tests_end_to_end/tests/sdk_helpers.py | 86 ++++---- tests_end_to_end/tests/utils.py | 42 ++-- 29 files changed, 910 insertions(+), 663 deletions(-) diff --git a/tests_end_to_end/installer_utils/check_backend.sh b/tests_end_to_end/installer_utils/check_backend.sh index 3bd96fdc09..535943f435 100644 --- a/tests_end_to_end/installer_utils/check_backend.sh +++ b/tests_end_to_end/installer_utils/check_backend.sh @@ -20,4 +20,4 @@ do done echo "Error: Backend did not respond with 200 OK after $((max_retries * wait_interval)) seconds." -exit 1 \ No newline at end of file +exit 1 diff --git a/tests_end_to_end/installer_utils/check_docker_compose_pods.sh b/tests_end_to_end/installer_utils/check_docker_compose_pods.sh index 5ae51dff06..363369174a 100644 --- a/tests_end_to_end/installer_utils/check_docker_compose_pods.sh +++ b/tests_end_to_end/installer_utils/check_docker_compose_pods.sh @@ -23,4 +23,4 @@ if [[ $retries -eq $max_retries ]]; then echo "Containers failed to start" docker compose ps exit 1 -fi \ No newline at end of file +fi diff --git a/tests_end_to_end/installer_utils/test_app_status.py b/tests_end_to_end/installer_utils/test_app_status.py index d93e940997..9145a4139e 100644 --- a/tests_end_to_end/installer_utils/test_app_status.py +++ b/tests_end_to_end/installer_utils/test_app_status.py @@ -1,6 +1,6 @@ from playwright.sync_api import Page, expect + def test_app_loads(page: Page): page.goto("http://localhost:5173/default/projects") expect(page.get_by_role("heading", name="Projects")).to_be_visible() - diff --git a/tests_end_to_end/page_objects/DatasetItemsPage.py b/tests_end_to_end/page_objects/DatasetItemsPage.py index e1df04f682..04c2243418 100644 --- a/tests_end_to_end/page_objects/DatasetItemsPage.py +++ b/tests_end_to_end/page_objects/DatasetItemsPage.py @@ -1,93 +1,99 @@ -from playwright.sync_api import Page, expect, Locator +from playwright.sync_api import Page + class DatasetItemsPage: def __init__(self, page: Page): self.page = page - self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)') + self.next_page_button_locator = self.page.locator( + "div:has(> button:nth-of-type(4))" + ).locator("button:nth-of-type(3)") def remove_default_columns(self): - self.page.get_by_role('button', name='Columns').click() - created_toggle = self.page.get_by_role('button', name='Created', exact=True).get_by_role('checkbox') + self.page.get_by_role("button", name="Columns").click() + created_toggle = self.page.get_by_role( + "button", name="Created", exact=True + ).get_by_role("checkbox") if created_toggle.is_checked(): created_toggle.click() - - last_updated_toggle = self.page.get_by_role('button', name='Last updated').get_by_role('checkbox') + + last_updated_toggle = self.page.get_by_role( + "button", name="Last updated" + ).get_by_role("checkbox") if last_updated_toggle.is_checked(): last_updated_toggle.click() - created_by_toggle = self.page.get_by_role('button', name='Created by', exact=True).get_by_role('checkbox') + created_by_toggle = self.page.get_by_role( + "button", name="Created by", exact=True + ).get_by_role("checkbox") if created_by_toggle.is_checked(): created_by_toggle.click() - - self.page.keyboard.press('Escape') + self.page.keyboard.press("Escape") def delete_first_item_on_page_and_return_content(self): self.remove_default_columns() - keys: list[str] = self.page.locator('th').all_inner_texts()[1:-1] - item={} + keys: list[str] = self.page.locator("th").all_inner_texts()[1:-1] + item = {} - row = self.page.locator('tr').nth(1) - cells = row.locator('td').all() + row = self.page.locator("tr").nth(1) + cells = row.locator("td").all() for cell_index, cell in enumerate(cells[1:-1]): - content = '' + content = "" if cell_index == 0: - cell.get_by_role('button').hover() - row.get_by_role('button').nth(1).click() - content = self.page.evaluate('navigator.clipboard.readText()') + cell.get_by_role("button").hover() + row.get_by_role("button").nth(1).click() + content = self.page.evaluate("navigator.clipboard.readText()") else: content = cell.text_content() item[keys[cell_index]] = content - - row.get_by_role('button', name='Actions menu').click() - self.page.get_by_role('menuitem', name='Delete').click() - self.page.get_by_role('button', name='Delete dataset item').click() - return item + row.get_by_role("button", name="Actions menu").click() + self.page.get_by_role("menuitem", name="Delete").click() + self.page.get_by_role("button", name="Delete dataset item").click() + return item def insert_dataset_item(self, item: str): - self.page.get_by_role('button', name='Create dataset item').click() - textbox = self.page.get_by_role('textbox') + self.page.get_by_role("button", name="Create dataset item").click() + textbox = self.page.get_by_role("textbox") textbox.focus() - self.page.keyboard.press('Meta+A') - self.page.keyboard.press('Backspace') + self.page.keyboard.press("Meta+A") + self.page.keyboard.press("Backspace") textbox.fill(item) - self.page.get_by_role('button', name='Create dataset item').click() + self.page.get_by_role("button", name="Create dataset item").click() - def get_all_dataset_items_on_current_page(self): self.remove_default_columns() - keys: list[str] = self.page.locator('th').all_inner_texts()[1:-1] + keys: list[str] = self.page.locator("th").all_inner_texts()[1:-1] items = [] - rows = self.page.locator('tr').all() + rows = self.page.locator("tr").all() for row_index, row in enumerate(rows[1:]): item = {} - cells = row.locator('td').all() + cells = row.locator("td").all() for cell_index, cell in enumerate(cells[1:-1]): - content = '' + content = "" if cell_index == 0: - cell.get_by_role('button').hover() - row.get_by_role('button').nth(1).click() - content = self.page.evaluate('navigator.clipboard.readText()') + cell.get_by_role("button").hover() + row.get_by_role("button").nth(1).click() + content = self.page.evaluate("navigator.clipboard.readText()") else: content = cell.text_content() item[keys[cell_index]] = content items.append(item) return items - def get_all_items_in_dataset(self): items = [] items.extend(self.get_all_dataset_items_on_current_page()) - while self.next_page_button_locator.is_visible() and self.next_page_button_locator.is_enabled(): + while ( + self.next_page_button_locator.is_visible() + and self.next_page_button_locator.is_enabled() + ): self.next_page_button_locator.click() self.page.wait_for_timeout(500) items.extend(self.get_all_dataset_items_on_current_page()) return items - - diff --git a/tests_end_to_end/page_objects/DatasetsPage.py b/tests_end_to_end/page_objects/DatasetsPage.py index d3e2c518d4..325f72cb8e 100644 --- a/tests_end_to_end/page_objects/DatasetsPage.py +++ b/tests_end_to_end/page_objects/DatasetsPage.py @@ -1,17 +1,18 @@ from playwright.sync_api import Page, expect + class DatasetsPage: def __init__(self, page: Page): self.page = page - self.url = '/default/datasets' + self.url = "/default/datasets" def go_to_page(self): self.page.goto(self.url) def create_dataset_by_name(self, dataset_name: str): - self.page.get_by_role('button', name='Create new dataset').first.click() - self.page.get_by_placeholder('Dataset name').fill(dataset_name) - self.page.get_by_role('button', name='Create dataset').click() + self.page.get_by_role("button", name="Create new dataset").first.click() + self.page.get_by_placeholder("Dataset name").fill(dataset_name) + self.page.get_by_role("button", name="Create dataset").click() def select_database_by_name(self, name): self.page.get_by_text(name, exact=True).first.click() @@ -22,15 +23,17 @@ def search_dataset(self, dataset_name): def check_dataset_exists_on_page_by_name(self, dataset_name): expect(self.page.get_by_text(dataset_name).first).to_be_visible() - + def check_dataset_not_exists_on_page_by_name(self, dataset_name): expect(self.page.get_by_text(dataset_name).first).not_to_be_visible() - + def delete_dataset_by_name(self, dataset_name): self.search_dataset(dataset_name) - row = self.page.get_by_role('row').filter(has_text=dataset_name).filter(has=self.page.get_by_role('cell', name=dataset_name, exact=True)) + row = ( + self.page.get_by_role("row") + .filter(has_text=dataset_name) + .filter(has=self.page.get_by_role("cell", name=dataset_name, exact=True)) + ) row.get_by_role("button").click() self.page.get_by_role("menuitem", name="Delete").click() self.page.get_by_role("button", name="Delete dataset").click() - - \ No newline at end of file diff --git a/tests_end_to_end/page_objects/ExperimentItemsPage.py b/tests_end_to_end/page_objects/ExperimentItemsPage.py index 682d4e39a5..904ae2ab7b 100644 --- a/tests_end_to_end/page_objects/ExperimentItemsPage.py +++ b/tests_end_to_end/page_objects/ExperimentItemsPage.py @@ -1,49 +1,52 @@ -from playwright.sync_api import Page, expect, Locator +from playwright.sync_api import Page, Locator import re -class ExperimentItemsPage: +class ExperimentItemsPage: def __init__(self, page: Page): self.page = page - self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)') + self.next_page_button_locator = self.page.locator( + "div:has(> button:nth-of-type(4))" + ).locator("button:nth-of-type(3)") def get_pagination_button(self) -> Locator: - return self.page.get_by_role('button', name='Showing') + return self.page.get_by_role("button", name="Showing") def get_total_number_of_items_in_experiment(self): pagination_button_text = self.get_pagination_button().inner_text() - match = re.search(r'of (\d+)', pagination_button_text) + match = re.search(r"of (\d+)", pagination_button_text) if match: return int(match.group(1)) else: return 0 - + def get_id_of_nth_experiment_item(self, n: int): - row = self.page.locator('tr').nth(n+1) - cell = row.locator('td').first + row = self.page.locator("tr").nth(n + 1) + cell = row.locator("td").first cell.hover() - cell.get_by_role('button').nth(1).click() - id = self.page.evaluate('navigator.clipboard.readText()') + cell.get_by_role("button").nth(1).click() + id = self.page.evaluate("navigator.clipboard.readText()") return id - - + def get_all_item_ids_on_current_page(self): ids = [] - rows = self.page.locator('tr').all() + rows = self.page.locator("tr").all() for row_index, row in enumerate(rows[2:]): - cell = row.locator('td').first + cell = row.locator("td").first cell.hover() - cell.get_by_role('button').nth(1).click() - id = self.page.evaluate('navigator.clipboard.readText()') + cell.get_by_role("button").nth(1).click() + id = self.page.evaluate("navigator.clipboard.readText()") ids.append(id) return ids - def get_all_item_ids_in_experiment(self): ids = [] ids.extend(self.get_all_item_ids_on_current_page()) - while self.next_page_button_locator.is_visible() and self.next_page_button_locator.is_enabled(): + while ( + self.next_page_button_locator.is_visible() + and self.next_page_button_locator.is_enabled() + ): self.next_page_button_locator.click() self.page.wait_for_timeout(500) ids.extend(self.get_all_dataset_items_on_current_page()) diff --git a/tests_end_to_end/page_objects/ExperimentsPage.py b/tests_end_to_end/page_objects/ExperimentsPage.py index 64666263ee..5081b236f2 100644 --- a/tests_end_to_end/page_objects/ExperimentsPage.py +++ b/tests_end_to_end/page_objects/ExperimentsPage.py @@ -1,10 +1,11 @@ from playwright.sync_api import Page, expect + class ExperimentsPage: def __init__(self, page: Page): self.page = page - self.url = '/default/experiments' - self.search_bar = self.page.get_by_test_id('search-input') + self.url = "/default/experiments" + self.search_bar = self.page.get_by_test_id("search-input") def go_to_page(self): self.page.goto(self.url) @@ -15,18 +16,20 @@ def search_experiment_by_name(self, exp_name: str): def click_first_experiment_that_matches_name(self, exp_name: str): self.search_experiment_by_name(exp_name=exp_name) - self.page.get_by_role('link', name=exp_name).first.click() - + self.page.get_by_role("link", name=exp_name).first.click() + def check_experiment_exists_by_name(self, exp_name: str): self.search_experiment_by_name(exp_name) expect(self.page.get_by_text(exp_name).first).to_be_visible() - + def check_experiment_not_exists_by_name(self, exp_name: str): self.search_experiment_by_name(exp_name) expect(self.page.get_by_text(exp_name)).not_to_be_visible() def delete_experiment_by_name(self, exp_name: str): self.search_experiment_by_name(exp_name) - self.page.get_by_role('row', name=exp_name).first.get_by_role('button', name='Actions menu').click() - self.page.get_by_role('menuitem', name='Delete').click() - self.page.get_by_role('button', name='Delete experiment').click() \ No newline at end of file + self.page.get_by_role("row", name=exp_name).first.get_by_role( + "button", name="Actions menu" + ).click() + self.page.get_by_role("menuitem", name="Delete").click() + self.page.get_by_role("button", name="Delete experiment").click() diff --git a/tests_end_to_end/page_objects/IndividualDatasetPage.py b/tests_end_to_end/page_objects/IndividualDatasetPage.py index 42574315b4..a85ebef391 100644 --- a/tests_end_to_end/page_objects/IndividualDatasetPage.py +++ b/tests_end_to_end/page_objects/IndividualDatasetPage.py @@ -1,9 +1,10 @@ from playwright.sync_api import Page, expect + class IndividualDatasetPage: def __init__(self, page: Page): self.page = page - self.traces_table = page.get_by_role('table') + self.traces_table = page.get_by_role("table") def check_cell_exists_by_text(self, text): - expect(self.traces_table.get_by_text(text, exact=True)).to_be_visible() \ No newline at end of file + expect(self.traces_table.get_by_text(text, exact=True)).to_be_visible() diff --git a/tests_end_to_end/page_objects/ProjectsPage.py b/tests_end_to_end/page_objects/ProjectsPage.py index e65d777fd5..8852915806 100644 --- a/tests_end_to_end/page_objects/ProjectsPage.py +++ b/tests_end_to_end/page_objects/ProjectsPage.py @@ -1,27 +1,32 @@ from playwright.sync_api import Page, expect import time + class ProjectsPage: def __init__(self, page: Page): self.page = page - self.url = '/projects' - self.projects_table = self.page.get_by_role('table') + self.url = "/projects" + self.projects_table = self.page.get_by_role("table") def go_to_page(self): self.page.goto(self.url) def click_project(self, project_name): - self.page.get_by_role('link', name=project_name).click() + self.page.get_by_role("link", name=project_name).click() def search_project(self, project_name): self.page.get_by_test_id("search-input").click() self.page.get_by_test_id("search-input").fill(project_name) def check_project_exists_on_current_page(self, project_name): - expect(self.page.get_by_role('cell', name=project_name, exact=True)).to_be_visible() + expect( + self.page.get_by_role("cell", name=project_name, exact=True) + ).to_be_visible() def check_project_not_exists_on_current_page(self, project_name): - expect(self.page.get_by_role('cell', name=project_name, exact=True)).not_to_be_visible() + expect( + self.page.get_by_role("cell", name=project_name, exact=True) + ).not_to_be_visible() def check_project_exists_on_current_page_with_retry(self, project_name, timeout): start_time = time.time() @@ -32,7 +37,9 @@ def check_project_exists_on_current_page_with_retry(self, project_name, timeout) except AssertionError: self.page.wait_for_timeout(500) else: - raise AssertionError(f'project {project_name} not found in projects list within {timeout} seconds') + raise AssertionError( + f"project {project_name} not found in projects list within {timeout} seconds" + ) def create_new_project(self, project_name): self.page.get_by_role("button", name="Create new project").click() @@ -45,7 +52,11 @@ def create_new_project(self, project_name): def delete_project_by_name(self, project_name): self.search_project(project_name) - row = self.page.get_by_role('row').filter(has_text=project_name).filter(has=self.page.get_by_role('cell', name=project_name, exact=True)) + row = ( + self.page.get_by_role("row") + .filter(has_text=project_name) + .filter(has=self.page.get_by_role("cell", name=project_name, exact=True)) + ) row.get_by_role("button").click() self.page.get_by_role("menuitem", name="Delete").click() - self.page.get_by_role("button", name="Delete project").click() \ No newline at end of file + self.page.get_by_role("button", name="Delete project").click() diff --git a/tests_end_to_end/page_objects/TracesPage.py b/tests_end_to_end/page_objects/TracesPage.py index f915d49eed..aa60917bb2 100644 --- a/tests_end_to_end/page_objects/TracesPage.py +++ b/tests_end_to_end/page_objects/TracesPage.py @@ -1,99 +1,105 @@ from playwright.sync_api import Page, expect, Locator import re + class TracesPage: def __init__(self, page: Page): self.page = page - self.traces_table = self.page.get_by_role('table') - self.trace_names_selector = 'tr td:nth-child(3) div span' - self.trace_id_selector = 'tr:nth-child({}) > td:nth-child(2) > div'.format - self.next_page_button_locator = self.page.locator("div:has(> button:nth-of-type(4))").locator('button:nth-of-type(3)') - self.delete_button_locator = self.page.locator("div").filter(has_text=re.compile(r"^Add to dataset$")).get_by_role("button").nth(2) - - + self.traces_table = self.page.get_by_role("table") + self.trace_names_selector = "tr td:nth-child(3) div span" + self.trace_id_selector = "tr:nth-child({}) > td:nth-child(2) > div".format + self.next_page_button_locator = self.page.locator( + "div:has(> button:nth-of-type(4))" + ).locator("button:nth-of-type(3)") + self.delete_button_locator = ( + self.page.locator("div") + .filter(has_text=re.compile(r"^Add to dataset$")) + .get_by_role("button") + .nth(2) + ) def get_all_trace_names_on_page(self): self.page.wait_for_selector(self.trace_names_selector) names = self.page.locator(self.trace_names_selector).all_inner_texts() return names - - - def click_first_trace_that_has_name(self, trace_name: str): - self.page.get_by_role('row').filter(has_text=trace_name).first.get_by_role('button').first.click() + def click_first_trace_that_has_name(self, trace_name: str): + self.page.get_by_role("row").filter(has_text=trace_name).first.get_by_role( + "button" + ).first.click() def get_first_trace_name_on_page(self): self.page.wait_for_selector(self.trace_names_selector) name = self.page.locator(self.trace_names_selector).first.text_content() return name - def get_all_trace_names_in_project(self): names = [] names.extend(self.get_all_trace_names_on_page()) - while self.next_page_button_locator.is_visible() and self.next_page_button_locator.is_enabled(): + while ( + self.next_page_button_locator.is_visible() + and self.next_page_button_locator.is_enabled() + ): self.next_page_button_locator.click() self.page.wait_for_timeout(500) names.extend(self.get_all_trace_names_on_page()) return names - def get_pagination_button(self) -> Locator: - return self.page.get_by_role('button', name='Showing') - + return self.page.get_by_role("button", name="Showing") def get_number_of_traces_on_page(self): try: - expect(self.page.get_by_role('row').first).to_be_visible() - except Exception as e: + expect(self.page.get_by_role("row").first).to_be_visible() + except Exception: return 0 finally: - return self.page.get_by_role('row').count() - + return self.page.get_by_role("row").count() def get_total_number_of_traces_in_project(self): pagination_button_text = self.get_pagination_button().inner_text() - match = re.search(r'of (\d+)', pagination_button_text) + match = re.search(r"of (\d+)", pagination_button_text) if match: return int(match.group(1)) else: return 0 - def delete_single_trace_by_name(self, name: str): - trace = self.page.get_by_role('row').filter(has_text=name).first - trace.get_by_label('Select row').click() + trace = self.page.get_by_role("row").filter(has_text=name).first + trace.get_by_label("Select row").click() self.delete_button_locator.click() - self.page.get_by_role('button', name='Delete traces').click() - + self.page.get_by_role("button", name="Delete traces").click() def delete_all_traces_that_match_name_contains_filter(self, name: str): - #TODO compact this into smaller functions + # TODO compact this into smaller functions self.page.get_by_role("button", name="Filters").click() - filter_row = self.page.get_by_role('row').filter(has=self.page.get_by_role('cell', name='Where')) - filter_row.get_by_role('cell').filter(has_text=re.compile(r"^Column$")).click() + filter_row = self.page.get_by_role("row").filter( + has=self.page.get_by_role("cell", name="Where") + ) + filter_row.get_by_role("cell").filter(has_text=re.compile(r"^Column$")).click() self.page.get_by_label("Name").click() self.page.get_by_test_id("filter-string-input").click() self.page.get_by_test_id("filter-string-input").fill(name) - expect(self.page.get_by_role('button', name='Filters (1)')).to_be_visible() + expect(self.page.get_by_role("button", name="Filters (1)")).to_be_visible() self.page.keyboard.press(key="Escape") total_traces = self.get_total_number_of_traces_in_project() while total_traces > 0: - expect(self.page.get_by_label('Select all')).to_be_visible() + expect(self.page.get_by_label("Select all")).to_be_visible() self.page.get_by_label("Select all").click() self.delete_button_locator.click() self.page.get_by_role("button", name="Delete traces").click() - + pagination_button = self.get_pagination_button() - expect(pagination_button).not_to_have_text(f'Showing 1-10 of {total_traces}') + expect(pagination_button).not_to_have_text( + f"Showing 1-10 of {total_traces}" + ) total_traces = self.get_total_number_of_traces_in_project() - def add_all_traces_to_new_dataset(self, dataset_name: str): self.page.get_by_label("Select all").click() self.page.get_by_role("button", name="Add to dataset").click() self.page.get_by_role("button", name="Create new dataset").click() self.page.get_by_placeholder("Dataset name").fill(dataset_name) - self.page.get_by_role("button", name="Create dataset").click() \ No newline at end of file + self.page.get_by_role("button", name="Create dataset").click() diff --git a/tests_end_to_end/page_objects/TracesPageSpansMenu.py b/tests_end_to_end/page_objects/TracesPageSpansMenu.py index 853e663230..b5f8cc2b6b 100644 --- a/tests_end_to_end/page_objects/TracesPageSpansMenu.py +++ b/tests_end_to_end/page_objects/TracesPageSpansMenu.py @@ -1,29 +1,30 @@ from playwright.sync_api import Page, expect + class TracesPageSpansMenu: def __init__(self, page: Page): self.page = page - self.input_output_tab = 'Input/Output' - self.feedback_scores_tab = 'Feedback scores' - self.metadata_tab = 'Metadata' + self.input_output_tab = "Input/Output" + self.feedback_scores_tab = "Feedback scores" + self.metadata_tab = "Metadata" def get_first_trace_by_name(self, name): - return self.page.get_by_role('button', name=name).first - + return self.page.get_by_role("button", name=name).first + def get_first_span_by_name(self, name): - return self.page.get_by_role('button', name=name).first + return self.page.get_by_role("button", name=name).first def check_span_exists_by_name(self, name): - expect(self.page.get_by_role('button', name=name)).to_be_visible() + expect(self.page.get_by_role("button", name=name)).to_be_visible() def check_tag_exists_by_name(self, tag_name): expect(self.page.get_by_text(tag_name)).to_be_visible() def get_input_output_tab(self): - return self.page.get_by_role('tab', name=self.input_output_tab) + return self.page.get_by_role("tab", name=self.input_output_tab) def get_feedback_scores_tab(self): - return self.page.get_by_role('tab', name=self.feedback_scores_tab) - + return self.page.get_by_role("tab", name=self.feedback_scores_tab) + def get_metadata_tab(self): - return self.page.get_by_role('tab', name='Metadata') \ No newline at end of file + return self.page.get_by_role("tab", name="Metadata") diff --git a/tests_end_to_end/test_requirements.txt b/tests_end_to_end/test_requirements.txt index ffe90ac21b..9c5d36765a 100644 --- a/tests_end_to_end/test_requirements.txt +++ b/tests_end_to_end/test_requirements.txt @@ -1,3 +1,3 @@ pytest playwright -pytest-playwright \ No newline at end of file +pytest-playwright diff --git a/tests_end_to_end/tests/Datasets/conftest.py b/tests_end_to_end/tests/Datasets/conftest.py index 2c31f219df..2377783528 100644 --- a/tests_end_to_end/tests/Datasets/conftest.py +++ b/tests_end_to_end/tests/Datasets/conftest.py @@ -1,11 +1,6 @@ import pytest -import os from opik import Opik -from playwright.sync_api import Page -from page_objects.DatasetsPage import DatasetsPage -from page_objects.DatasetItemsPage import DatasetItemsPage from datasets_utils import TEST_ITEMS -import json @pytest.fixture diff --git a/tests_end_to_end/tests/Datasets/datasets_utils.py b/tests_end_to_end/tests/Datasets/datasets_utils.py index 08daea584e..9ba4ff93b7 100644 --- a/tests_end_to_end/tests/Datasets/datasets_utils.py +++ b/tests_end_to_end/tests/Datasets/datasets_utils.py @@ -7,30 +7,30 @@ import opik TEST_ITEMS = [ - {'input': 'input0', 'output': 'output0'}, - {'input': 'input1', 'output': 'output1'}, - {'input': 'input2', 'output': 'output2'}, - {'input': 'input3', 'output': 'output3'}, - {'input': 'input4', 'output': 'output4'}, - {'input': 'input5', 'output': 'output5'}, - {'input': 'input6', 'output': 'output6'}, - {'input': 'input7', 'output': 'output7'}, - {'input': 'input8', 'output': 'output8'}, - {'input': 'input9', 'output': 'output9'} + {"input": "input0", "output": "output0"}, + {"input": "input1", "output": "output1"}, + {"input": "input2", "output": "output2"}, + {"input": "input3", "output": "output3"}, + {"input": "input4", "output": "output4"}, + {"input": "input5", "output": "output5"}, + {"input": "input6", "output": "output6"}, + {"input": "input7", "output": "output7"}, + {"input": "input8", "output": "output8"}, + {"input": "input9", "output": "output9"}, ] TEST_ITEMS_UPDATE = [ - {'input': 'update-input0', 'output': 'update-output0'}, - {'input': 'update-input1', 'output': 'update-output1'}, - {'input': 'update-input2', 'output': 'update-output2'}, - {'input': 'update-input3', 'output': 'update-output3'}, - {'input': 'update-input4', 'output': 'update-output4'}, - {'input': 'update-input5', 'output': 'update-output5'}, - {'input': 'update-input6', 'output': 'update-output6'}, - {'input': 'update-input7', 'output': 'update-output7'}, - {'input': 'update-input8', 'output': 'update-output8'}, - {'input': 'update-input9', 'output': 'update-output9'} + {"input": "update-input0", "output": "update-output0"}, + {"input": "update-input1", "output": "update-output1"}, + {"input": "update-input2", "output": "update-output2"}, + {"input": "update-input3", "output": "update-output3"}, + {"input": "update-input4", "output": "update-output4"}, + {"input": "update-input5", "output": "update-output5"}, + {"input": "update-input6", "output": "update-output6"}, + {"input": "update-input7", "output": "update-output7"}, + {"input": "update-input8", "output": "update-output8"}, + {"input": "update-input9", "output": "update-output9"}, ] @@ -45,7 +45,7 @@ def insert_dataset_items_ui(page: Page, dataset_name, items_list): datasets_page.select_database_by_name(dataset_name) dataset_items_page = DatasetItemsPage(page) - + for item in items_list: dataset_items_page.insert_dataset_item(json.dumps(item)) time.sleep(0.2) @@ -54,7 +54,7 @@ def insert_dataset_items_ui(page: Page, dataset_name, items_list): def delete_one_dataset_item_sdk(client: opik.Opik, dataset_name): dataset = client.get_dataset(dataset_name) item = dataset.get_items()[0] - dataset.delete([item['id']]) + dataset.delete([item["id"]]) return item @@ -80,14 +80,16 @@ def wait_for_dataset_to_be_visible(client: Opik, dataset_name: str, timeout=10): if dataset: break time.sleep(0.5) - + if dataset: return dataset else: raise -def wait_for_number_of_items_in_dataset(expected_items_number: int, dataset, timeout=10): +def wait_for_number_of_items_in_dataset( + expected_items_number: int, dataset, timeout=10 +): expected_number_achieved = False start_time = time.time() items = [] @@ -99,20 +101,25 @@ def wait_for_number_of_items_in_dataset(expected_items_number: int, dataset, tim time.sleep(0.5) if not expected_number_achieved: - raise AssertionError(f'expected to see {expected_items_number} in dataset, instead found {len(items)} after {timeout} seconds of retries') + raise AssertionError( + f"expected to see {expected_items_number} in dataset, instead found {len(items)} after {timeout} seconds of retries" + ) def compare_item_lists(expected: list[dict], actual: list[dict]): set_expected = {frozenset(d.items()) for d in expected} - set_actual = {frozenset((key, val) for key, val in d.items() if 'id' not in key.lower()) for d in actual} + set_actual = { + frozenset((key, val) for key, val in d.items() if "id" not in key.lower()) + for d in actual + } return set_expected == set_actual def get_updated_items(current: list[dict], update: list[dict]): result = [ - {'id': item2['id'], 'input': item1['input'], 'output': item1['output']} + {"id": item2["id"], "input": item1["input"], "output": item1["output"]} for item1, item2 in zip(update, current) ] - return result \ No newline at end of file + return result diff --git a/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py b/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py index 50a3f0def7..913f51a85d 100644 --- a/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py +++ b/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py @@ -1,21 +1,37 @@ import pytest from playwright.sync_api import Page, expect from page_objects.DatasetsPage import DatasetsPage -from page_objects.ProjectsPage import ProjectsPage -from page_objects.TracesPage import TracesPage from page_objects.DatasetItemsPage import DatasetItemsPage -from sdk_helpers import delete_dataset_by_name_if_exists, update_dataset_name, get_dataset_by_name -from datasets_utils import TEST_ITEMS, TEST_ITEMS_UPDATE, compare_item_lists, wait_for_dataset_to_be_visible, get_updated_items, insert_dataset_items_sdk, insert_dataset_items_ui, delete_one_dataset_item_sdk, delete_one_dataset_item_ui, wait_for_number_of_items_in_dataset +from datasets_utils import ( + TEST_ITEMS, + TEST_ITEMS_UPDATE, + compare_item_lists, + wait_for_dataset_to_be_visible, + get_updated_items, + insert_dataset_items_sdk, + insert_dataset_items_ui, + delete_one_dataset_item_sdk, + delete_one_dataset_item_ui, + wait_for_number_of_items_in_dataset, +) import opik -import time class TestDatasetItemsCrud: - - @pytest.mark.parametrize('dataset_insert', ['insert_via_ui', 'insert_via_sdk']) - @pytest.mark.parametrize('dataset_creation_fixture', ['create_delete_dataset_sdk', 'create_delete_dataset_ui']) - @pytest.mark.browser_context_args(permissions=['clipboard-read']) - def test_dataset_item_insertion(self, request, page: Page, client: opik.Opik, dataset_creation_fixture, dataset_insert): + @pytest.mark.parametrize("dataset_insert", ["insert_via_ui", "insert_via_sdk"]) + @pytest.mark.parametrize( + "dataset_creation_fixture", + ["create_delete_dataset_sdk", "create_delete_dataset_ui"], + ) + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) + def test_dataset_item_insertion( + self, + request, + page: Page, + client: opik.Opik, + dataset_creation_fixture, + dataset_insert, + ): """ Tests insertion into database in all possible ways of creating dataset and the items themselves (creates 4 test instances), and checks syncing between UI and SDK 1. Create a dataset via either the SDK or the UI @@ -28,18 +44,24 @@ def test_dataset_item_insertion(self, request, page: Page, client: opik.Opik, da - dataset created by UI - items inserted by SDK - check they all appear correctly in both UI and SDK """ - dataset = wait_for_dataset_to_be_visible(client=client, dataset_name=request.getfixturevalue(dataset_creation_fixture), timeout=10) - - if 'ui' in dataset_insert: + dataset = wait_for_dataset_to_be_visible( + client=client, + dataset_name=request.getfixturevalue(dataset_creation_fixture), + timeout=10, + ) + + if "ui" in dataset_insert: insert_dataset_items_ui(page, dataset.name, TEST_ITEMS) - elif 'sdk' in dataset_insert: + elif "sdk" in dataset_insert: insert_dataset_items_sdk(client, dataset.name, TEST_ITEMS) - wait_for_number_of_items_in_dataset(expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=10) + wait_for_number_of_items_in_dataset( + expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=10 + ) items_from_sdk = dataset.get_items() - # CHECK THAT THE ITEMS INSERTED ARE EXACTLY THE ITEMS RETURNED BY THE SDK + # CHECK THAT THE ITEMS INSERTED ARE EXACTLY THE ITEMS RETURNED BY THE SDK assert compare_item_lists(expected=TEST_ITEMS, actual=items_from_sdk) dataset_page = DatasetsPage(page) @@ -51,9 +73,15 @@ def test_dataset_item_insertion(self, request, page: Page, client: opik.Opik, da # CHECK THAT THE ITEMS INSERTED ARE EXACTLY THE ITEMS FOUND IN THE UI assert compare_item_lists(expected=TEST_ITEMS, actual=items_from_ui) - - @pytest.mark.browser_context_args(permissions=['clipboard-read']) - def test_dataset_item_update(self, request, page: Page, client: opik.Opik, create_delete_dataset_sdk, insert_dataset_items_sdk): + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) + def test_dataset_item_update( + self, + request, + page: Page, + client: opik.Opik, + create_delete_dataset_sdk, + insert_dataset_items_sdk, + ): """ Tests updating existing dataset items with new information and the change syncing to both the UI and the SDK 1. Create a dataset via the SDK @@ -61,17 +89,23 @@ def test_dataset_item_update(self, request, page: Page, client: opik.Opik, creat 3. Using dataset.update(), insert new data into the existing dataset items 4. Check that the new data is correct on both the UI and the SDK """ - dataset = wait_for_dataset_to_be_visible(client=client, dataset_name=create_delete_dataset_sdk, timeout=10) + dataset = wait_for_dataset_to_be_visible( + client=client, dataset_name=create_delete_dataset_sdk, timeout=10 + ) - wait_for_number_of_items_in_dataset(expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=15) + wait_for_number_of_items_in_dataset( + expected_items_number=len(TEST_ITEMS), dataset=dataset, timeout=15 + ) items_from_sdk = dataset.get_items() - updated_items = get_updated_items(current=items_from_sdk, update=TEST_ITEMS_UPDATE) + updated_items = get_updated_items( + current=items_from_sdk, update=TEST_ITEMS_UPDATE + ) dataset.update(updated_items) items_from_sdk = dataset.get_items() - # CHECK THAT THE ITEMS FROM THE SDK EXACTLY MATCH THE UPDATED DATASET ITEMS DATA + # CHECK THAT THE ITEMS FROM THE SDK EXACTLY MATCH THE UPDATED DATASET ITEMS DATA assert compare_item_lists(expected=TEST_ITEMS_UPDATE, actual=items_from_sdk) dataset_page = DatasetsPage(page) @@ -80,34 +114,46 @@ def test_dataset_item_update(self, request, page: Page, client: opik.Opik, creat dataset_items_page = DatasetItemsPage(page) items_from_ui = dataset_items_page.get_all_items_in_dataset() - # CHECK THAT THE ITEMS FOUND IN THE UI EXACTLY MATCH THE UPDATED DATASET ITEMS DATA + # CHECK THAT THE ITEMS FOUND IN THE UI EXACTLY MATCH THE UPDATED DATASET ITEMS DATA assert compare_item_lists(expected=TEST_ITEMS_UPDATE, actual=items_from_ui) - - @pytest.mark.browser_context_args(permissions=['clipboard-read']) - @pytest.mark.parametrize('item_deletion', ['delete_via_ui', 'delete_via_sdk']) - def test_dataset_item_deletion(self, request, page: Page, client: opik.Opik, create_delete_dataset_sdk, insert_dataset_items_sdk, item_deletion): + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) + @pytest.mark.parametrize("item_deletion", ["delete_via_ui", "delete_via_sdk"]) + def test_dataset_item_deletion( + self, + request, + page: Page, + client: opik.Opik, + create_delete_dataset_sdk, + insert_dataset_items_sdk, + item_deletion, + ): """ Tests deletion of an item via both the UI and the SDK (2 test instances created) and the change being visible in both the UI and the SDK 1. Create a dataset via the SDK 2. Insert 10 items into it via the SDK 3. Using either the UI or the SDK (2 tests), delete one item from the dataset - 4. Check that the item with that data no longer exists in both the SDK and the UI and that the length of the item list is updated + 4. Check that the item with that data no longer exists in both the SDK and the UI and that the length of the item list is updated """ - dataset = wait_for_dataset_to_be_visible(client=client, dataset_name=create_delete_dataset_sdk, timeout=10) - + dataset = wait_for_dataset_to_be_visible( + client=client, dataset_name=create_delete_dataset_sdk, timeout=10 + ) + item_deleted = {} - if 'ui' in item_deletion: + if "ui" in item_deletion: item_deleted = delete_one_dataset_item_ui(page, dataset.name) - elif 'sdk' in item_deletion: + elif "sdk" in item_deletion: item_deleted = delete_one_dataset_item_sdk(client, dataset.name) - wait_for_number_of_items_in_dataset(expected_items_number=len(TEST_ITEMS)-1, dataset=dataset, timeout=15) + wait_for_number_of_items_in_dataset( + expected_items_number=len(TEST_ITEMS) - 1, dataset=dataset, timeout=15 + ) items_from_sdk = dataset.get_items() deleted_item_not_in_sdk_list = not any( - item['input'] == item_deleted['input'] and item['output'] == item_deleted['output'] + item["input"] == item_deleted["input"] + and item["output"] == item_deleted["output"] for item in items_from_sdk ) @@ -123,16 +169,23 @@ def test_dataset_item_deletion(self, request, page: Page, client: opik.Opik, cre assert len(items_from_ui) == len(TEST_ITEMS) - 1 deleted_item_not_in_ui_list = not any( - item['input'] == item_deleted['input'] and item['output'] == item_deleted['output'] + item["input"] == item_deleted["input"] + and item["output"] == item_deleted["output"] for item in items_from_ui ) # CHECK DATA OF DELETED ITEM NO LONGER PRESENT IN DATASET WHEN GETTING ITEMS FROM UI assert deleted_item_not_in_ui_list - - @pytest.mark.browser_context_args(permissions=['clipboard-read']) - def test_dataset_clear(self, request, page: Page, client: opik.Opik, create_delete_dataset_sdk, insert_dataset_items_sdk): + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) + def test_dataset_clear( + self, + request, + page: Page, + client: opik.Opik, + create_delete_dataset_sdk, + insert_dataset_items_sdk, + ): """ Tests mass deletion from the dataset using dataset.clear() 1. Create a dataset via the SDK @@ -140,11 +193,15 @@ def test_dataset_clear(self, request, page: Page, client: opik.Opik, create_dele 3. Deleting every item from the dataset using dataset.clear() 4. Check that no items exist in the dataset when trying to get them via both the SDK and the UI """ - dataset = wait_for_dataset_to_be_visible(client=client, dataset_name=create_delete_dataset_sdk, timeout=10) + dataset = wait_for_dataset_to_be_visible( + client=client, dataset_name=create_delete_dataset_sdk, timeout=10 + ) dataset.clear() # CHECK NO ITEMS RETURNED FROM THE SDK - wait_for_number_of_items_in_dataset(expected_items_number=0, dataset=dataset, timeout=15) + wait_for_number_of_items_in_dataset( + expected_items_number=0, dataset=dataset, timeout=15 + ) dataset_page = DatasetsPage(page) dataset_page.go_to_page() @@ -152,4 +209,6 @@ def test_dataset_clear(self, request, page: Page, client: opik.Opik, create_dele dataset_items_page = DatasetItemsPage(page) # CHECK -DATASET EMPTY- MESSAGE APPEARS, SIGNIFYING AN EMPTY DATASET - expect(dataset_items_page.page.get_by_text('There are no dataset items yet')).to_be_visible() \ No newline at end of file + expect( + dataset_items_page.page.get_by_text("There are no dataset items yet") + ).to_be_visible() diff --git a/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py b/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py index 6fe1b8af27..ae0d99f4f9 100644 --- a/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py +++ b/tests_end_to_end/tests/Datasets/test_datasets_crud_operations.py @@ -1,15 +1,18 @@ import pytest -from playwright.sync_api import Page, expect +from playwright.sync_api import Page from page_objects.DatasetsPage import DatasetsPage from page_objects.ProjectsPage import ProjectsPage from page_objects.TracesPage import TracesPage -from sdk_helpers import delete_dataset_by_name_if_exists, update_dataset_name, get_dataset_by_name +from sdk_helpers import ( + delete_dataset_by_name_if_exists, + update_dataset_name, + get_dataset_by_name, +) import opik import time class TestDatasetsCrud: - def test_create_dataset_ui_datasets_page(self, page: Page): """ Basic test to check dataset creation via UI. Uses the UI after creation to check the dataset exists @@ -19,18 +22,21 @@ def test_create_dataset_ui_datasets_page(self, page: Page): """ datasets_page = DatasetsPage(page) datasets_page.go_to_page() - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" try: datasets_page.create_dataset_by_name(dataset_name=dataset_name) - datasets_page.check_dataset_exists_on_page_by_name(dataset_name=dataset_name) + datasets_page.check_dataset_exists_on_page_by_name( + dataset_name=dataset_name + ) except Exception as e: - print(f'error during dataset creation: {e}') + print(f"error during dataset creation: {e}") raise finally: delete_dataset_by_name_if_exists(dataset_name=dataset_name) - - def test_create_dataset_ui_add_traces_to_new_dataset(self, page: Page, create_delete_project_sdk, create_10_test_traces): + def test_create_dataset_ui_add_traces_to_new_dataset( + self, page: Page, create_delete_project_sdk, create_10_test_traces + ): """ Basic test to check dataset creation via "add to new dataset" functionality in the traces page. Uses the UI after creation to check the project exists 1. Create a project with some traces @@ -38,7 +44,7 @@ def test_create_dataset_ui_add_traces_to_new_dataset(self, page: Page, create_de 3. Switch to the datasets page, check the dataset exists in the dataset table 4. If no errors raised and dataset exists, test passes """ - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" proj_name = create_delete_project_sdk projects_page = ProjectsPage(page) projects_page.go_to_page() @@ -46,18 +52,19 @@ def test_create_dataset_ui_add_traces_to_new_dataset(self, page: Page, create_de traces_page = TracesPage(page) traces_page.add_all_traces_to_new_dataset(dataset_name=dataset_name) - + try: datasets_page = DatasetsPage(page) datasets_page.go_to_page() - datasets_page.check_dataset_exists_on_page_by_name(dataset_name=dataset_name) + datasets_page.check_dataset_exists_on_page_by_name( + dataset_name=dataset_name + ) except Exception as e: - print(f'error: dataset not created: {e}') + print(f"error: dataset not created: {e}") raise finally: delete_dataset_by_name_if_exists(dataset_name=dataset_name) - def test_create_dataset_sdk_client(self, client: opik.Opik): """ Basic test to check dataset creation via SDK. Uses the SDK to fetch the created dataset to check it exists @@ -65,20 +72,23 @@ def test_create_dataset_sdk_client(self, client: opik.Opik): 2. Get the project via SDK OpikAPI client 3. If dataset creation fails, client.get_dataset will throw an error and the test will fail. """ - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" try: client.create_dataset(name=dataset_name) time.sleep(0.2) assert client.get_dataset(name=dataset_name) is not None except Exception as e: - print(f'error during dataset creation: {e}') + print(f"error during dataset creation: {e}") raise finally: delete_dataset_by_name_if_exists(dataset_name=dataset_name) - - @pytest.mark.parametrize('dataset_fixture', ['create_delete_dataset_ui', 'create_delete_dataset_sdk']) - def test_dataset_visibility(self, request, page: Page, client: opik.Opik, dataset_fixture): + @pytest.mark.parametrize( + "dataset_fixture", ["create_delete_dataset_ui", "create_delete_dataset_sdk"] + ) + def test_dataset_visibility( + self, request, page: Page, client: opik.Opik, dataset_fixture + ): """ Checks a created dataset is visible via both the UI and SDK. Test split in 2: checks on datasets created on both UI and SDK 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) @@ -95,9 +105,13 @@ def test_dataset_visibility(self, request, page: Page, client: opik.Opik, datase dataset_sdk = client.get_dataset(dataset_name) assert dataset_sdk.name == dataset_name - - @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) - def test_dataset_name_update(self, request, page: Page, client: opik.Opik, dataset_fixture): + @pytest.mark.parametrize( + "dataset_fixture", + ["create_dataset_sdk_no_cleanup", "create_dataset_ui_no_cleanup"], + ) + def test_dataset_name_update( + self, request, page: Page, client: opik.Opik, dataset_fixture + ): """ Checks using the SDK update method on a dataset. Test split into 2: checks on dataset created on both UI and SDK 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) @@ -107,7 +121,7 @@ def test_dataset_name_update(self, request, page: Page, client: opik.Opik, datas """ dataset_name = request.getfixturevalue(dataset_fixture) time.sleep(0.5) - new_name = 'updated_test_dataset_name' + new_name = "updated_test_dataset_name" name_updated = False try: @@ -116,16 +130,18 @@ def test_dataset_name_update(self, request, page: Page, client: opik.Opik, datas dataset_new_name = get_dataset_by_name(dataset_name=new_name) - dataset_id_updated_name = dataset_new_name['id'] + dataset_id_updated_name = dataset_new_name["id"] assert dataset_id_updated_name == dataset_id datasets_page = DatasetsPage(page) datasets_page.go_to_page() datasets_page.check_dataset_exists_on_page_by_name(dataset_name=new_name) - datasets_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) + datasets_page.check_dataset_not_exists_on_page_by_name( + dataset_name=dataset_name + ) except Exception as e: - print(f'Error occured during update of project name: {e}') + print(f"Error occured during update of project name: {e}") raise finally: @@ -134,9 +150,13 @@ def test_dataset_name_update(self, request, page: Page, client: opik.Opik, datas else: delete_dataset_by_name_if_exists(dataset_name) - - @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) - def test_dataset_deletion_in_sdk(self, request, page: Page, client: opik.Opik, dataset_fixture): + @pytest.mark.parametrize( + "dataset_fixture", + ["create_dataset_sdk_no_cleanup", "create_dataset_ui_no_cleanup"], + ) + def test_dataset_deletion_in_sdk( + self, request, page: Page, client: opik.Opik, dataset_fixture + ): """ Checks proper deletion of a dataset via the SDK. Test split into 2: checks on datasets created on both UI and SDK 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) @@ -151,16 +171,20 @@ def test_dataset_deletion_in_sdk(self, request, page: Page, client: opik.Opik, d dataset_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) try: _ = client.get_dataset(dataset_name) - assert False, f'datasets {dataset_name} somehow still exists after deletion' + assert False, f"datasets {dataset_name} somehow still exists after deletion" except Exception as e: - if '404' in str(e) or 'not found' in str(e).lower(): + if "404" in str(e) or "not found" in str(e).lower(): pass else: raise - - @pytest.mark.parametrize('dataset_fixture', ['create_dataset_sdk_no_cleanup', 'create_dataset_ui_no_cleanup']) - def test_dataset_deletion_in_ui(self, request, page: Page, client: opik.Opik, dataset_fixture): + @pytest.mark.parametrize( + "dataset_fixture", + ["create_dataset_sdk_no_cleanup", "create_dataset_ui_no_cleanup"], + ) + def test_dataset_deletion_in_ui( + self, request, page: Page, client: opik.Opik, dataset_fixture + ): """ Checks proper deletion of a dataset via the SDK. Test split into 2: checks on datasets created on both UI and SDK 1. Create a dataset via the UI/the SDK (2 "instances" of the test created for each one) @@ -176,9 +200,9 @@ def test_dataset_deletion_in_ui(self, request, page: Page, client: opik.Opik, da try: _ = client.get_dataset(dataset_name) - assert False, f'datasets {dataset_name} somehow still exists after deletion' + assert False, f"datasets {dataset_name} somehow still exists after deletion" except Exception as e: - if '404' in str(e) or 'not found' in str(e).lower(): + if "404" in str(e) or "not found" in str(e).lower(): pass else: raise @@ -186,4 +210,3 @@ def test_dataset_deletion_in_ui(self, request, page: Page, client: opik.Opik, da dataset_page = DatasetsPage(page) dataset_page.go_to_page() dataset_page.check_dataset_not_exists_on_page_by_name(dataset_name=dataset_name) - diff --git a/tests_end_to_end/tests/Experiments/conftest.py b/tests_end_to_end/tests/Experiments/conftest.py index eebed161c6..5356699b42 100644 --- a/tests_end_to_end/tests/Experiments/conftest.py +++ b/tests_end_to_end/tests/Experiments/conftest.py @@ -1,41 +1,31 @@ import pytest -import os from opik import Opik -from playwright.sync_api import Page -from page_objects.DatasetsPage import DatasetsPage -from page_objects.DatasetItemsPage import DatasetItemsPage -from page_objects.ExperimentsPage import ExperimentsPage from opik.evaluation.metrics import Contains from opik.evaluation import evaluate from sdk_helpers import delete_experiment_by_id -import json def eval_task(item: dict): - return { - 'input': item['input'], - 'output': item['output'], - 'reference': 'output' - } + return {"input": item["input"], "output": item["output"], "reference": "output"} + @pytest.fixture() def mock_experiment(client: Opik, create_delete_dataset_sdk, insert_dataset_items_sdk): dataset = client.get_dataset(create_delete_dataset_sdk) - experiment_name='test_experiment' + experiment_name = "test_experiment" eval = evaluate( experiment_name=experiment_name, dataset=dataset, task=eval_task, - scoring_metrics=[Contains()] + scoring_metrics=[Contains()], ) yield { - 'id': eval.experiment_id, - 'name': experiment_name, - 'size': len(dataset.get_items()) + "id": eval.experiment_id, + "name": experiment_name, + "size": len(dataset.get_items()), } try: delete_experiment_by_id(eval.experiment_id) except Exception as e: - print(f'Experiment cleanup error: {e}') + print(f"Experiment cleanup error: {e}") pass - \ No newline at end of file diff --git a/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py b/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py index 18c333b6be..d7c4d30c44 100644 --- a/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py +++ b/tests_end_to_end/tests/Experiments/test_experiment_crud_operations.py @@ -1,14 +1,10 @@ import pytest -from playwright.sync_api import Page, expect -from page_objects.DatasetsPage import DatasetsPage +from playwright.sync_api import Page from page_objects.ExperimentsPage import ExperimentsPage from sdk_helpers import get_experiment_by_id, delete_experiment_by_id -import opik -import time class TestExperimentsCrud: - def test_experiment_visibility(self, page: Page, mock_experiment): """ Tests experiment creation and visibility of experiment in both UI and SDK @@ -18,13 +14,12 @@ def test_experiment_visibility(self, page: Page, mock_experiment): experiments_page = ExperimentsPage(page) experiments_page.go_to_page() - experiments_page.check_experiment_exists_by_name(mock_experiment['name']) + experiments_page.check_experiment_exists_by_name(mock_experiment["name"]) - experiment_sdk = get_experiment_by_id(mock_experiment['id']) - assert experiment_sdk.name == mock_experiment['name'] + experiment_sdk = get_experiment_by_id(mock_experiment["id"]) + assert experiment_sdk.name == mock_experiment["name"] - - @pytest.mark.parametrize('deletion_method', ['ui', 'sdk']) + @pytest.mark.parametrize("deletion_method", ["ui", "sdk"]) def test_experiment_deletion(self, page: Page, mock_experiment, deletion_method): """ Tests deletion of experiment via both the UI and the SDK and checks experiment correctly no longer appears @@ -32,24 +27,22 @@ def test_experiment_deletion(self, page: Page, mock_experiment, deletion_method) 2. Delete the experiment via either the UI or the SDK (2 separate test entitites) 3. Check the experiment does not appear in the UI and that requesting it via the API correctly returns a 404 """ - if deletion_method == 'ui': + if deletion_method == "ui": experiments_page = ExperimentsPage(page) experiments_page.go_to_page() - experiments_page.delete_experiment_by_name(mock_experiment['name']) - elif deletion_method == 'sdk': - delete_experiment_by_id(mock_experiment['id']) + experiments_page.delete_experiment_by_name(mock_experiment["name"]) + elif deletion_method == "sdk": + delete_experiment_by_id(mock_experiment["id"]) experiments_page = ExperimentsPage(page) experiments_page.go_to_page() - experiments_page.check_experiment_not_exists_by_name(mock_experiment['name']) + experiments_page.check_experiment_not_exists_by_name(mock_experiment["name"]) try: - _ = get_experiment_by_id(mock_experiment['id']) + _ = get_experiment_by_id(mock_experiment["id"]) assert False, f"experiment {mock_experiment['name']} somehow still exists after deletion" except Exception as e: - if '404' in str(e) or 'not found' in str(e).lower(): + if "404" in str(e) or "not found" in str(e).lower(): pass else: raise - - \ No newline at end of file diff --git a/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py index 566b0e9a67..bce3c8b570 100644 --- a/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py +++ b/tests_end_to_end/tests/Experiments/test_experiment_items_crud_operations.py @@ -1,17 +1,17 @@ import pytest -from playwright.sync_api import Page, expect -from page_objects.DatasetsPage import DatasetsPage +from playwright.sync_api import Page from page_objects.ExperimentsPage import ExperimentsPage from page_objects.ExperimentItemsPage import ExperimentItemsPage -from sdk_helpers import get_experiment_by_id, delete_experiment_by_id, delete_experiment_items_by_id, experiment_items_stream -import opik -import time +from sdk_helpers import ( + get_experiment_by_id, + delete_experiment_items_by_id, + experiment_items_stream, +) from collections import Counter class TestExperimentItemsCrud: - - @pytest.mark.browser_context_args(permissions=['clipboard-read']) + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) def test_all_experiment_items_created(self, page: Page, mock_experiment): """ Creates an experiment with 10 experiment items, then checks that all items are visible in both UI and backend @@ -24,22 +24,26 @@ def test_all_experiment_items_created(self, page: Page, mock_experiment): """ experiments_page = ExperimentsPage(page) experiments_page.go_to_page() - experiments_page.click_first_experiment_that_matches_name(exp_name=mock_experiment['name']) + experiments_page.click_first_experiment_that_matches_name( + exp_name=mock_experiment["name"] + ) experiment_items_page = ExperimentItemsPage(page) items_on_page = experiment_items_page.get_total_number_of_items_in_experiment() - assert items_on_page == mock_experiment['size'] + assert items_on_page == mock_experiment["size"] - experiment_backend = get_experiment_by_id(mock_experiment['id']) - assert experiment_backend.trace_count == mock_experiment['size'] + experiment_backend = get_experiment_by_id(mock_experiment["id"]) + assert experiment_backend.trace_count == mock_experiment["size"] - ids_on_backend = [item['dataset_item_id'] for item in experiment_items_stream(mock_experiment['name'])] + ids_on_backend = [ + item["dataset_item_id"] + for item in experiment_items_stream(mock_experiment["name"]) + ] ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() assert Counter(ids_on_backend) == Counter(ids_on_frontend) - - @pytest.mark.browser_context_args(permissions=['clipboard-read']) + @pytest.mark.browser_context_args(permissions=["clipboard-read"]) def test_delete_experiment_items(self, page: Page, mock_experiment): """ Deletes a single experiment item and checks that everything gets updated on both the UI and the backend @@ -54,20 +58,27 @@ def test_delete_experiment_items(self, page: Page, mock_experiment): """ experiments_page = ExperimentsPage(page) experiments_page.go_to_page() - experiments_page.click_first_experiment_that_matches_name(exp_name=mock_experiment['name']) + experiments_page.click_first_experiment_that_matches_name( + exp_name=mock_experiment["name"] + ) - id_to_delete = experiment_items_stream(exp_name=mock_experiment['name'], limit=1)[0]['id'] + id_to_delete = experiment_items_stream( + exp_name=mock_experiment["name"], limit=1 + )[0]["id"] delete_experiment_items_by_id(ids=[id_to_delete]) experiment_items_page = ExperimentItemsPage(page) experiment_items_page.page.reload() items_on_page = experiment_items_page.get_total_number_of_items_in_experiment() - assert items_on_page == mock_experiment['size'] - 1 + assert items_on_page == mock_experiment["size"] - 1 - experiment_sdk = get_experiment_by_id(mock_experiment['id']) - assert experiment_sdk.trace_count == mock_experiment['size'] - 1 + experiment_sdk = get_experiment_by_id(mock_experiment["id"]) + assert experiment_sdk.trace_count == mock_experiment["size"] - 1 - ids_on_backend = [item['dataset_item_id'] for item in experiment_items_stream(mock_experiment['name'])] + ids_on_backend = [ + item["dataset_item_id"] + for item in experiment_items_stream(mock_experiment["name"]) + ] ids_on_frontend = experiment_items_page.get_all_item_ids_in_experiment() assert Counter(ids_on_backend) == Counter(ids_on_frontend) diff --git a/tests_end_to_end/tests/Projects/test_projects_crud_operations.py b/tests_end_to_end/tests/Projects/test_projects_crud_operations.py index 379f19b22e..e1f47bcd62 100644 --- a/tests_end_to_end/tests/Projects/test_projects_crud_operations.py +++ b/tests_end_to_end/tests/Projects/test_projects_crud_operations.py @@ -1,6 +1,12 @@ import pytest from playwright.sync_api import Page -from sdk_helpers import create_project_sdk, find_project_by_name_sdk, delete_project_by_name_sdk, wait_for_project_to_be_visible, update_project_by_name_sdk +from sdk_helpers import ( + create_project_sdk, + find_project_by_name_sdk, + delete_project_by_name_sdk, + wait_for_project_to_be_visible, + update_project_by_name_sdk, +) from page_objects.ProjectsPage import ProjectsPage @@ -13,23 +19,22 @@ def test_project_creation_via_sdk(self): 3. If no errors raised and name of fetched project matches, test passes """ - project_name = 'test_project_creation_via_sdk' - + project_name = "test_project_creation_via_sdk" + try: create_project_sdk(name=project_name) wait_for_project_to_be_visible(project_name, timeout=10) - + check_project = find_project_by_name_sdk(project_name) - assert check_project[0]['name'] == project_name - + assert check_project[0]["name"] == project_name + except Exception as e: - print(f'error occured during creation test: {e}') + print(f"error occured during creation test: {e}") raise finally: delete_project_by_name_sdk(project_name) - def test_project_creation_via_ui(self, page: Page): """ Basic test to check project creation via UI. Uses the UI after creation to check the project exists @@ -37,30 +42,33 @@ def test_project_creation_via_ui(self, page: Page): 2. Check the project exists in the projects table 3. If no errors raised, test passes """ - project_name = 'test_project_creation_via_ui' + project_name = "test_project_creation_via_ui" projects_page = ProjectsPage(page) project_created = False try: projects_page.go_to_page() projects_page.create_new_project(project_name=project_name) - projects_page.check_project_exists_on_current_page(project_name=project_name) + projects_page.check_project_exists_on_current_page( + project_name=project_name + ) project_created = True assert project_created - + except Exception as e: - print(f'error during project creation or verification: {e}') + print(f"error during project creation or verification: {e}") raise finally: if project_created: - try: + try: projects_page.delete_project_by_name(project_name=project_name) except Exception as e: - print(f'error during cleanup: {e}') - + print(f"error during cleanup: {e}") - @pytest.mark.parametrize('project_fixture', ['create_delete_project_ui', 'create_delete_project_sdk']) + @pytest.mark.parametrize( + "project_fixture", ["create_delete_project_ui", "create_delete_project_sdk"] + ) def test_project_visibility(self, request, page: Page, project_fixture): """ Checks a created project is visible via both the UI and SDK. Checks on projects created on both UI and SDK @@ -74,14 +82,18 @@ def test_project_visibility(self, request, page: Page, project_fixture): projects_match = find_project_by_name_sdk(project_name) assert len(projects_match) > 0 - assert projects_match[0]['name'] == project_name + assert projects_match[0]["name"] == project_name projects_page = ProjectsPage(page) projects_page.go_to_page() - projects_page.check_project_exists_on_current_page_with_retry(project_name=project_name, timeout=5) - - - @pytest.mark.parametrize('project_fixture', ['create_project_sdk_no_cleanup', 'create_project_ui_no_cleanup']) + projects_page.check_project_exists_on_current_page_with_retry( + project_name=project_name, timeout=5 + ) + + @pytest.mark.parametrize( + "project_fixture", + ["create_project_sdk_no_cleanup", "create_project_ui_no_cleanup"], + ) def test_project_name_update(self, request, page: Page, project_fixture): """ Checks using the SDK update method on a project. Checks on projects created on both UI and SDK @@ -90,28 +102,34 @@ def test_project_name_update(self, request, page: Page, project_fixture): 3. Check on both the SDK and the UI that the project has been renamed (on SDK: check project ID matches. on UI: check project with new name appears and no project with old name appears) """ - + project_name = request.getfixturevalue(project_fixture) - new_name = 'updated_test_project_name' + new_name = "updated_test_project_name" name_updated = False try: - project_id = update_project_by_name_sdk(name=project_name, new_name=new_name) + project_id = update_project_by_name_sdk( + name=project_name, new_name=new_name + ) name_updated = True wait_for_project_to_be_visible(new_name, timeout=10) projects_match = find_project_by_name_sdk(new_name) - project_id_updated_name = projects_match[0]['id'] + project_id_updated_name = projects_match[0]["id"] assert project_id_updated_name == project_id projects_page = ProjectsPage(page) projects_page.go_to_page() - projects_page.check_project_exists_on_current_page_with_retry(project_name=new_name, timeout=5) - projects_page.check_project_not_exists_on_current_page(project_name=project_name) + projects_page.check_project_exists_on_current_page_with_retry( + project_name=new_name, timeout=5 + ) + projects_page.check_project_not_exists_on_current_page( + project_name=project_name + ) except Exception as e: - print(f'Error occured during update of project name: {e}') + print(f"Error occured during update of project name: {e}") raise finally: @@ -120,8 +138,10 @@ def test_project_name_update(self, request, page: Page, project_fixture): else: delete_project_by_name_sdk(project_name) - - @pytest.mark.parametrize('project_fixture', ['create_project_sdk_no_cleanup', 'create_project_ui_no_cleanup']) + @pytest.mark.parametrize( + "project_fixture", + ["create_project_sdk_no_cleanup", "create_project_ui_no_cleanup"], + ) def test_project_deletion_in_sdk(self, request, page: Page, project_fixture): """ Checks proper deletion of a project via the SDK. Checks on projects created on both UI and SDK @@ -134,13 +154,17 @@ def test_project_deletion_in_sdk(self, request, page: Page, project_fixture): projects_page = ProjectsPage(page) projects_page.go_to_page() - projects_page.check_project_not_exists_on_current_page(project_name=project_name) + projects_page.check_project_not_exists_on_current_page( + project_name=project_name + ) projects_found = find_project_by_name_sdk(project_name) assert len(projects_found) == 0 - - @pytest.mark.parametrize('project_fixture', ['create_project_sdk_no_cleanup', 'create_project_ui_no_cleanup']) + @pytest.mark.parametrize( + "project_fixture", + ["create_project_sdk_no_cleanup", "create_project_ui_no_cleanup"], + ) def test_project_deletion_in_ui(self, request, page: Page, project_fixture): """ Checks proper deletion of a project via the UI. Checks on projects created on both UI and SDK @@ -155,7 +179,9 @@ def test_project_deletion_in_ui(self, request, page: Page, project_fixture): projects_page = ProjectsPage(page) projects_page.go_to_page() - projects_page.check_project_not_exists_on_current_page(project_name=project_name) + projects_page.check_project_not_exists_on_current_page( + project_name=project_name + ) projects_found = find_project_by_name_sdk(project_name) - assert len(projects_found) == 0 \ No newline at end of file + assert len(projects_found) == 0 diff --git a/tests_end_to_end/tests/Traces/conftest.py b/tests_end_to_end/tests/Traces/conftest.py index 0c4728820b..80e68da790 100644 --- a/tests_end_to_end/tests/Traces/conftest.py +++ b/tests_end_to_end/tests/Traces/conftest.py @@ -1,46 +1,45 @@ import pytest -import os from opik import opik_context, track from traces_config import PREFIX, PROJECT_NAME -from sdk_helpers import create_traces_sdk, get_traces_of_project_sdk, delete_list_of_traces_sdk, wait_for_traces_to_be_visible, wait_for_number_of_traces_to_be_visible +from sdk_helpers import wait_for_number_of_traces_to_be_visible -@pytest.fixture(scope='function') -def log_x_traces_with_one_span_via_decorator(traces_number): +@pytest.fixture(scope="function") +def log_x_traces_with_one_span_via_decorator(traces_number): @track def f2(input: str): - return 'test output' - + return "test output" + @track def f1(input: str): - opik_context.update_current_trace(name=PREFIX+str(i)) + opik_context.update_current_trace(name=PREFIX + str(i)) return f2(input) for i in range(traces_number): - f1('test input') + f1("test input") yield -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def log_x_traces_with_one_span_via_client(client, traces_number): for i in range(traces_number): client_trace = client.trace( - name=PREFIX+str(i), + name=PREFIX + str(i), project_name=PROJECT_NAME, - input={'input': 'test input'}, - output={'output': 'test output'}, + input={"input": "test input"}, + output={"output": "test output"}, ) _ = client_trace.span( - name='span', - input={'input': 'test input'}, - output={'output': 'test output'} + name="span", input={"input": "test input"}, output={"output": "test output"} ) - wait_for_number_of_traces_to_be_visible(project_name=PROJECT_NAME, number_of_traces=traces_number) + wait_for_number_of_traces_to_be_visible( + project_name=PROJECT_NAME, number_of_traces=traces_number + ) yield - -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def create_traces(request, traces_number): _ = request.getfixturevalue(request.param) yield 0 diff --git a/tests_end_to_end/tests/Traces/test_traces_crud_operations.py b/tests_end_to_end/tests/Traces/test_traces_crud_operations.py index bf4e6e6576..8ea69a593b 100644 --- a/tests_end_to_end/tests/Traces/test_traces_crud_operations.py +++ b/tests_end_to_end/tests/Traces/test_traces_crud_operations.py @@ -4,20 +4,32 @@ from page_objects.TracesPage import TracesPage from traces_config import PREFIX from collections import Counter -from sdk_helpers import get_traces_of_project_sdk, delete_list_of_traces_sdk, wait_for_traces_to_be_visible +from sdk_helpers import ( + get_traces_of_project_sdk, + delete_list_of_traces_sdk, + wait_for_traces_to_be_visible, +) class TestTracesCrud: - - @pytest.mark.parametrize('traces_number', [1, 15]) - @pytest.mark.parametrize('create_traces', ['log_x_traces_with_one_span_via_decorator', 'log_x_traces_with_one_span_via_client'], indirect=True) - def test_trace_creation(self, page: Page, traces_number, create_delete_project_sdk, create_traces): + @pytest.mark.parametrize("traces_number", [1, 15]) + @pytest.mark.parametrize( + "create_traces", + [ + "log_x_traces_with_one_span_via_decorator", + "log_x_traces_with_one_span_via_client", + ], + indirect=True, + ) + def test_trace_creation( + self, page: Page, traces_number, create_delete_project_sdk, create_traces + ): """Testing basic creation of traces via both decorator and low-level client. Test case is split into 4, creating 1 and then 15 traces using both the decorator and the client respectively 1. Create a new project 2. Create the traces using one of the creation methods, following the naming convention of "test-trace-X", where X is from 1 to 25 (so all have unique names) - no errors should occur - 3. In the UI, check that the presented number of traces in the project matches the number of traces created in the test case + 3. In the UI, check that the presented number of traces in the project matches the number of traces created in the test case """ project_name = create_delete_project_sdk projects_page = ProjectsPage(page) @@ -29,14 +41,22 @@ def test_trace_creation(self, page: Page, traces_number, create_delete_project_s traces_created = traces_page.get_total_number_of_traces_in_project() assert traces_created == traces_number - - @pytest.mark.parametrize('traces_number', [25]) - @pytest.mark.parametrize('create_traces', ['log_x_traces_with_one_span_via_decorator', 'log_x_traces_with_one_span_via_client'], indirect=True) - def test_traces_visibility(self, page: Page, traces_number, create_delete_project_sdk, create_traces): + @pytest.mark.parametrize("traces_number", [25]) + @pytest.mark.parametrize( + "create_traces", + [ + "log_x_traces_with_one_span_via_decorator", + "log_x_traces_with_one_span_via_client", + ], + indirect=True, + ) + def test_traces_visibility( + self, page: Page, traces_number, create_delete_project_sdk, create_traces + ): """ Testing visibility within the UI and SDK of traces created via both the decorator and the client Test case is split into 2, creating traces via decorator first, and then via the low level client - + 1. Create a new project 2. Create 25 traces via either the decorator or the client, following the naming convention of "test-trace-X", where X is from 1 to 25 (so all have unique names) 3. Check all the traces are visible in the UI: @@ -53,18 +73,28 @@ def test_traces_visibility(self, page: Page, traces_number, create_delete_projec traces_page = TracesPage(page) _ = create_traces - created_trace_names = Counter([PREFIX+str(i) for i in range(traces_number)]) + created_trace_names = Counter([PREFIX + str(i) for i in range(traces_number)]) traces_ui = traces_page.get_all_trace_names_in_project() assert Counter(traces_ui) == created_trace_names - traces_sdk = get_traces_of_project_sdk(project_name=project_name, size=traces_number) - traces_sdk_names = [trace['name'] for trace in traces_sdk] + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] assert Counter(traces_sdk_names) == created_trace_names - - @pytest.mark.parametrize('traces_number', [10]) - @pytest.mark.parametrize('create_traces', ['log_x_traces_with_one_span_via_decorator', 'log_x_traces_with_one_span_via_client'], indirect=True) - def test_delete_traces_sdk(self, page: Page, traces_number, create_delete_project_sdk, create_traces): + @pytest.mark.parametrize("traces_number", [10]) + @pytest.mark.parametrize( + "create_traces", + [ + "log_x_traces_with_one_span_via_decorator", + "log_x_traces_with_one_span_via_client", + ], + indirect=True, + ) + def test_delete_traces_sdk( + self, page: Page, traces_number, create_delete_project_sdk, create_traces + ): """ Testing trace deletion via the SDK API client (v1/private/traces/delete endpoint) Test case is split into 2, creating traces via the decorator first, then via the client @@ -79,28 +109,46 @@ def test_delete_traces_sdk(self, page: Page, traces_number, create_delete_projec _ = create_traces wait_for_traces_to_be_visible(project_name=project_name, size=traces_number) - traces_sdk = get_traces_of_project_sdk(project_name=project_name, size=traces_number) - traces_sdk_names_ids = [{'id': trace['id'], 'name': trace['name']} for trace in traces_sdk] + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names_ids = [ + {"id": trace["id"], "name": trace["name"]} for trace in traces_sdk + ] traces_to_delete = traces_sdk_names_ids[0:2] - delete_list_of_traces_sdk(trace['id'] for trace in traces_to_delete) + delete_list_of_traces_sdk(trace["id"] for trace in traces_to_delete) projects_page = ProjectsPage(page) projects_page.go_to_page() projects_page.click_project(project_name) traces_page = TracesPage(page) - expect(traces_page.page.get_by_role('row', name=traces_to_delete[0]['name'])).not_to_be_visible() - expect(traces_page.page.get_by_role('row', name=traces_to_delete[1]['name'])).not_to_be_visible() - - traces_sdk = get_traces_of_project_sdk(project_name=project_name, size=traces_number) - traces_sdk_names = [trace['name'] for trace in traces_sdk] + expect( + traces_page.page.get_by_role("row", name=traces_to_delete[0]["name"]) + ).not_to_be_visible() + expect( + traces_page.page.get_by_role("row", name=traces_to_delete[1]["name"]) + ).not_to_be_visible() + + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] assert all(name not in traces_sdk_names for name in traces_to_delete) - - @pytest.mark.parametrize('traces_number', [10]) - @pytest.mark.parametrize('create_traces', ['log_x_traces_with_one_span_via_decorator', 'log_x_traces_with_one_span_via_client'], indirect=True) - def test_delete_traces_ui(self, page: Page, traces_number, create_delete_project_sdk, create_traces): + @pytest.mark.parametrize("traces_number", [10]) + @pytest.mark.parametrize( + "create_traces", + [ + "log_x_traces_with_one_span_via_decorator", + "log_x_traces_with_one_span_via_client", + ], + indirect=True, + ) + def test_delete_traces_ui( + self, page: Page, traces_number, create_delete_project_sdk, create_traces + ): """ Testing trace deletion via the UI Test case is split into 2, creating traces via the decorator first, then via the client @@ -118,19 +166,27 @@ def test_delete_traces_ui(self, page: Page, traces_number, create_delete_project traces_page = TracesPage(page) _ = create_traces - traces_sdk = get_traces_of_project_sdk(project_name=project_name, size=traces_number) - traces_sdk_names = [trace['name'] for trace in traces_sdk] + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] traces_to_delete = traces_sdk_names[0:2] traces_page.delete_single_trace_by_name(traces_to_delete[0]) traces_page.delete_single_trace_by_name(traces_to_delete[1]) traces_page.page.wait_for_timeout(200) - expect(traces_page.page.get_by_role('row', name=traces_to_delete[0])).not_to_be_visible() - expect(traces_page.page.get_by_role('row', name=traces_to_delete[1])).not_to_be_visible() + expect( + traces_page.page.get_by_role("row", name=traces_to_delete[0]) + ).not_to_be_visible() + expect( + traces_page.page.get_by_role("row", name=traces_to_delete[1]) + ).not_to_be_visible() wait_for_traces_to_be_visible(project_name=project_name, size=traces_number) - traces_sdk = get_traces_of_project_sdk(project_name=project_name, size=traces_number) - traces_sdk_names = [trace['name'] for trace in traces_sdk] + traces_sdk = get_traces_of_project_sdk( + project_name=project_name, size=traces_number + ) + traces_sdk_names = [trace["name"] for trace in traces_sdk] - assert all(name not in traces_sdk_names for name in traces_to_delete) \ No newline at end of file + assert all(name not in traces_sdk_names for name in traces_to_delete) diff --git a/tests_end_to_end/tests/Traces/traces_config.py b/tests_end_to_end/tests/Traces/traces_config.py index 972600ec3c..6ef082cbcf 100644 --- a/tests_end_to_end/tests/Traces/traces_config.py +++ b/tests_end_to_end/tests/Traces/traces_config.py @@ -1,2 +1,2 @@ -PREFIX='test-trace-' -PROJECT_NAME='automated_tests_project' \ No newline at end of file +PREFIX = "test-trace-" +PROJECT_NAME = "automated_tests_project" diff --git a/tests_end_to_end/tests/application_sanity/conftest.py b/tests_end_to_end/tests/application_sanity/conftest.py index 092a85aacc..392a76b866 100644 --- a/tests_end_to_end/tests/application_sanity/conftest.py +++ b/tests_end_to_end/tests/application_sanity/conftest.py @@ -3,36 +3,33 @@ import opik import yaml import json -from opik.configurator.configure import configure from opik.evaluation import evaluate from opik.evaluation.metrics import Contains, Equals from opik import opik_context, track -from playwright.sync_api import Page -from page_objects.ProjectsPage import ProjectsPage -from page_objects.TracesPage import TracesPage -from page_objects.DatasetsPage import DatasetsPage -from page_objects.ExperimentsPage import ExperimentsPage - -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def config(): curr_dir = os.path.dirname(__file__) - config_path = os.path.join(curr_dir, 'sanity_config.yaml') + config_path = os.path.join(curr_dir, "sanity_config.yaml") - with open(config_path, 'r') as f: + with open(config_path, "r") as f: conf = yaml.safe_load(f) - - os.environ['OPIK_PROJECT_NAME'] = conf['project']['name'] + + os.environ["OPIK_PROJECT_NAME"] = conf["project"]["name"] return conf -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def client_sanity(config): - return opik.Opik(project_name=config['project']['name'], workspace='default', host='http://localhost:5173/api') + return opik.Opik( + project_name=config["project"]["name"], + workspace="default", + host="http://localhost:5173/api", + ) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def log_traces_and_spans_low_level(client_sanity, config): """ Log 5 traces with spans and subspans using the low level Opik client @@ -40,43 +37,49 @@ def log_traces_and_spans_low_level(client_sanity, config): """ trace_config = { - 'count': config['traces']['count'], - 'prefix': config['traces']['client']['prefix'], - 'tags': config['traces']['client']['tags'], - 'metadata': config['traces']['client']['metadata'], - 'feedback_scores': [{'name': key, 'value': value} for key, value in config['traces']['client']['feedback-scores'].items()] + "count": config["traces"]["count"], + "prefix": config["traces"]["client"]["prefix"], + "tags": config["traces"]["client"]["tags"], + "metadata": config["traces"]["client"]["metadata"], + "feedback_scores": [ + {"name": key, "value": value} + for key, value in config["traces"]["client"]["feedback-scores"].items() + ], } span_config = { - 'count': config['spans']['count'], - 'prefix': config['spans']['client']['prefix'], - 'tags': config['spans']['client']['tags'], - 'metadata': config['spans']['client']['metadata'], - 'feedback_scores': [{'name': key, 'value': value} for key, value in config['spans']['client']['feedback-scores'].items()] + "count": config["spans"]["count"], + "prefix": config["spans"]["client"]["prefix"], + "tags": config["spans"]["client"]["tags"], + "metadata": config["spans"]["client"]["metadata"], + "feedback_scores": [ + {"name": key, "value": value} + for key, value in config["spans"]["client"]["feedback-scores"].items() + ], } - for trace_index in range(trace_config['count']): + for trace_index in range(trace_config["count"]): client_trace = client_sanity.trace( - name=trace_config['prefix'] + str(trace_index), - input={'input': f'input-{trace_index}'}, - output={'output': f'output-{trace_index}'}, - tags=trace_config['tags'], - metadata=trace_config['metadata'], - feedback_scores=trace_config['feedback_scores'] + name=trace_config["prefix"] + str(trace_index), + input={"input": f"input-{trace_index}"}, + output={"output": f"output-{trace_index}"}, + tags=trace_config["tags"], + metadata=trace_config["metadata"], + feedback_scores=trace_config["feedback_scores"], ) - for span_index in range(span_config['count']): + for span_index in range(span_config["count"]): client_span = client_trace.span( - name=span_config['prefix'] + str(span_index), - input={'input': f'input-{span_index}'}, - output={'output': f'output-{span_index}'}, - tags=span_config['tags'], - metadata=span_config['metadata'] + name=span_config["prefix"] + str(span_index), + input={"input": f"input-{span_index}"}, + output={"output": f"output-{span_index}"}, + tags=span_config["tags"], + metadata=span_config["metadata"], ) - for score in span_config['feedback_scores']: - client_span.log_feedback_score(name=score['name'], value=score['value']) + for score in span_config["feedback_scores"]: + client_span.log_feedback_score(name=score["name"], value=score["value"]) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def log_traces_and_spans_decorator(config): """ Log 5 traces with spans and subspans using the low level Opik client @@ -84,118 +87,118 @@ def log_traces_and_spans_decorator(config): """ trace_config = { - 'count': config['traces']['count'], - 'prefix': config['traces']['decorator']['prefix'], - 'tags': config['traces']['decorator']['tags'], - 'metadata': config['traces']['decorator']['metadata'], - 'feedback_scores': [{'name': key, 'value': value} for key, value in config['traces']['decorator']['feedback-scores'].items()] + "count": config["traces"]["count"], + "prefix": config["traces"]["decorator"]["prefix"], + "tags": config["traces"]["decorator"]["tags"], + "metadata": config["traces"]["decorator"]["metadata"], + "feedback_scores": [ + {"name": key, "value": value} + for key, value in config["traces"]["decorator"]["feedback-scores"].items() + ], } span_config = { - 'count': config['spans']['count'], - 'prefix': config['spans']['decorator']['prefix'], - 'tags': config['spans']['decorator']['tags'], - 'metadata': config['spans']['decorator']['metadata'], - 'feedback_scores': [{'name': key, 'value': value} for key, value in config['spans']['decorator']['feedback-scores'].items()] + "count": config["spans"]["count"], + "prefix": config["spans"]["decorator"]["prefix"], + "tags": config["spans"]["decorator"]["tags"], + "metadata": config["spans"]["decorator"]["metadata"], + "feedback_scores": [ + {"name": key, "value": value} + for key, value in config["spans"]["decorator"]["feedback-scores"].items() + ], } @track() def make_span(x): opik_context.update_current_span( - name=span_config['prefix'] + str(x), - input={'input': f'input-{x}'}, - metadata=span_config['metadata'], - tags=span_config['tags'], - feedback_scores=span_config['feedback_scores'] + name=span_config["prefix"] + str(x), + input={"input": f"input-{x}"}, + metadata=span_config["metadata"], + tags=span_config["tags"], + feedback_scores=span_config["feedback_scores"], ) - return {'output': f'output-{x}'} - + return {"output": f"output-{x}"} + @track() def make_trace(x): - for spans_no in range(span_config['count']): + for spans_no in range(span_config["count"]): make_span(spans_no) opik_context.update_current_trace( - name=trace_config['prefix'] + str(x), - input={'input': f'input-{x}'}, - metadata=trace_config['metadata'], - tags=trace_config['tags'], - feedback_scores=trace_config['feedback_scores'] + name=trace_config["prefix"] + str(x), + input={"input": f"input-{x}"}, + metadata=trace_config["metadata"], + tags=trace_config["tags"], + feedback_scores=trace_config["feedback_scores"], ) - return {'output': f'output-{x}'} - - for x in range(trace_config['count']): + return {"output": f"output-{x}"} + + for x in range(trace_config["count"]): make_trace(x) -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def dataset(config, client_sanity): dataset_config = { - 'name': config['dataset']['name'], - 'filename': config['dataset']['filename'] + "name": config["dataset"]["name"], + "filename": config["dataset"]["filename"], } - dataset = client_sanity.get_or_create_dataset(dataset_config['name']) + dataset = client_sanity.get_or_create_dataset(dataset_config["name"]) curr_dir = os.path.dirname(__file__) - dataset_filepath = os.path.join(curr_dir, dataset_config['filename']) + dataset_filepath = os.path.join(curr_dir, dataset_config["filename"]) dataset.read_jsonl_from_file(dataset_filepath) return dataset -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def create_experiments(config, dataset): exp_config = { - 'prefix': config['experiments']['prefix'], - 'metrics': config['experiments']['metrics'], - 'dataset_name': config['experiments']['dataset-name'] + "prefix": config["experiments"]["prefix"], + "metrics": config["experiments"]["metrics"], + "dataset_name": config["experiments"]["dataset-name"], } def eval_contains(x): return { - 'input': x['input'], - 'output': x['expected_output'], - 'reference': 'hello' + "input": x["input"], + "output": x["expected_output"], + "reference": "hello", } def eval_equals(x): return { - 'input': x['input'], - 'output': x['expected_output'], - 'reference': 'goodbye' + "input": x["input"], + "output": x["expected_output"], + "reference": "goodbye", } - contains_metric = Contains( - name='Contains', - case_sensitive=False - ) - equals_metric = Equals( - name='Equals', - case_sensitive=False - ) + contains_metric = Contains(name="Contains", case_sensitive=False) + equals_metric = Equals(name="Equals", case_sensitive=False) evaluate( - experiment_name=exp_config['prefix'] + 'Contains', + experiment_name=exp_config["prefix"] + "Contains", dataset=dataset, task=eval_contains, - scoring_metrics=[contains_metric] + scoring_metrics=[contains_metric], ) evaluate( - experiment_name=exp_config['prefix'] + 'Equals', + experiment_name=exp_config["prefix"] + "Equals", dataset=dataset, task=eval_equals, - scoring_metrics=[equals_metric] + scoring_metrics=[equals_metric], ) - -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def dataset_content(config): curr_dir = os.path.dirname(__file__) - dataset_filepath = os.path.join(curr_dir, config['dataset']['filename']) + dataset_filepath = os.path.join(curr_dir, config["dataset"]["filename"]) data = [] - with open(dataset_filepath, 'r') as f: + with open(dataset_filepath, "r") as f: for line in f: data.append(json.loads(line)) - return data \ No newline at end of file + return data diff --git a/tests_end_to_end/tests/application_sanity/sanity_config.yaml b/tests_end_to_end/tests/application_sanity/sanity_config.yaml index 7efebfab0f..904c29fce7 100644 --- a/tests_end_to_end/tests/application_sanity/sanity_config.yaml +++ b/tests_end_to_end/tests/application_sanity/sanity_config.yaml @@ -12,7 +12,7 @@ traces: metadata: c-md1: "val1" c-md2: "val2" - + decorator: prefix: "decorator-trace-" tags: ["d-tag1", "d-tag2"] @@ -53,4 +53,4 @@ dataset: experiments: prefix: "test-experiment-" metrics: ["Equals", "Contains"] - dataset-name: *dataset-name \ No newline at end of file + dataset-name: *dataset-name diff --git a/tests_end_to_end/tests/application_sanity/test_sanity.py b/tests_end_to_end/tests/application_sanity/test_sanity.py index e29bddf6f1..764e7363f8 100644 --- a/tests_end_to_end/tests/application_sanity/test_sanity.py +++ b/tests_end_to_end/tests/application_sanity/test_sanity.py @@ -1,136 +1,179 @@ -import pytest -import json - from playwright.sync_api import Page, expect from page_objects.TracesPageSpansMenu import TracesPageSpansMenu from page_objects.IndividualDatasetPage import IndividualDatasetPage -def test_project_name(projects_page_timeout, log_traces_and_spans_decorator, log_traces_and_spans_low_level): - ''' +def test_project_name( + projects_page_timeout, + log_traces_and_spans_decorator, + log_traces_and_spans_low_level, +): + """ Checks that the project created via the fixtures exists Does a timeout of 5 seconds to wait for the traces to show up in UI for later tests (TODO: figure out a better way to do this) 1. Open projects page 2. Check the created project exists - ''' - projects_page_timeout.check_project_exists_on_current_page('test-project') + """ + projects_page_timeout.check_project_exists_on_current_page("test-project") -def test_traces_created(traces_page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator): - ''' +def test_traces_created( + traces_page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator +): + """ Checks that every trace defined in the sanity_config file is present in the project 1. Open the traces page of the project 2. Grab all the names of the traces (should never set more than 15 in config so 1 page is safe) 3. Check that every possible name of the traces as defined in sanity_config.yaml is present in the names list - ''' + """ trace_names = traces_page.get_all_trace_names_on_page() - client_prefix = config['traces']['client']['prefix'] - decorator_prefix = config['traces']['decorator']['prefix'] + client_prefix = config["traces"]["client"]["prefix"] + decorator_prefix = config["traces"]["decorator"]["prefix"] - for count in range(config['traces']['count']): + for count in range(config["traces"]["count"]): for prefix in [client_prefix, decorator_prefix]: - assert prefix+str(count) in trace_names + assert prefix + str(count) in trace_names -def test_spans_of_traces(page, traces_page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator): - ''' +def test_spans_of_traces( + page, + traces_page, + config, + log_traces_and_spans_low_level, + log_traces_and_spans_decorator, +): + """ Checks that every trace has the correct number and names of spans defined in the sanity_config.yaml file 1. Open the traces page of the project 2. Go through each trace and click it 3. Check that the spans are present in each trace - ''' + """ trace_names = traces_page.get_all_trace_names_on_page() - traces_page.click_first_trace_that_has_name('decorator-trace-1') + traces_page.click_first_trace_that_has_name("decorator-trace-1") for trace in trace_names: traces_page.click_first_trace_that_has_name(trace) spans_menu = TracesPageSpansMenu(page) - trace_type = trace.split('-')[0] # 'client' or 'decorator' - for count in range(config['spans']['count']): - prefix = config['spans'][trace_type]['prefix'] - spans_menu.check_span_exists_by_name(f'{prefix}{count}') - - -def test_trace_and_span_details(page, traces_page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator): - ''' + trace_type = trace.split("-")[0] # 'client' or 'decorator' + for count in range(config["spans"]["count"]): + prefix = config["spans"][trace_type]["prefix"] + spans_menu.check_span_exists_by_name(f"{prefix}{count}") + + +def test_trace_and_span_details( + page, + traces_page, + config, + log_traces_and_spans_low_level, + log_traces_and_spans_decorator, +): + """ Checks that for each trace and spans, the attributes defined in sanity_config.yaml are present 1. Go through each trace of the project 2. Check the created tags are present 3. Check the created feedback scores are present 4. Check the defined metadata is present 5. Go through each span of the traces and repeat 2-4 - ''' + """ trace_names = traces_page.get_all_trace_names_on_page() for trace in trace_names: traces_page.click_first_trace_that_has_name(trace) spans_menu = TracesPageSpansMenu(page) - trace_type = trace.split('-')[0] - tag_names = config['traces'][trace_type]['tags'] + trace_type = trace.split("-")[0] + tag_names = config["traces"][trace_type]["tags"] for tag in tag_names: spans_menu.check_tag_exists_by_name(tag) spans_menu.get_feedback_scores_tab().click() - for score in config['traces'][trace_type]['feedback-scores']: - expect(page.get_by_role('cell', name=score, exact=True).first).to_be_visible() - expect(page.get_by_role('cell', name=str(config['traces'][trace_type]['feedback-scores'][score]), exact=True).first).to_be_visible() - + for score in config["traces"][trace_type]["feedback-scores"]: + expect( + page.get_by_role("cell", name=score, exact=True).first + ).to_be_visible() + expect( + page.get_by_role( + "cell", + name=str(config["traces"][trace_type]["feedback-scores"][score]), + exact=True, + ).first + ).to_be_visible() + spans_menu.get_metadata_tab().click() - for md_key in config['traces'][trace_type]['metadata']: - expect(page.get_by_text(f"{md_key}: {config['traces'][trace_type]['metadata'][md_key]}")).to_be_visible() - - for count in range(config['spans']['count']): - prefix = config['spans'][trace_type]['prefix'] - spans_menu.get_first_span_by_name(f'{prefix}{count}').click() + for md_key in config["traces"][trace_type]["metadata"]: + expect( + page.get_by_text( + f"{md_key}: {config['traces'][trace_type]['metadata'][md_key]}" + ) + ).to_be_visible() + + for count in range(config["spans"]["count"]): + prefix = config["spans"][trace_type]["prefix"] + spans_menu.get_first_span_by_name(f"{prefix}{count}").click() spans_menu.get_feedback_scores_tab().click() - for score in config['spans'][trace_type]['feedback-scores']: - expect(page.get_by_role('cell', name=score, exact=True)).to_be_visible() - expect(page.get_by_role('cell', name=str(config['spans'][trace_type]['feedback-scores'][score]), exact=True)).to_be_visible() - + for score in config["spans"][trace_type]["feedback-scores"]: + expect(page.get_by_role("cell", name=score, exact=True)).to_be_visible() + expect( + page.get_by_role( + "cell", + name=str(config["spans"][trace_type]["feedback-scores"][score]), + exact=True, + ) + ).to_be_visible() + spans_menu.get_metadata_tab().click() - for md_key in config['spans'][trace_type]['metadata']: - expect(page.get_by_text(f"{md_key}: {config['spans'][trace_type]['metadata'][md_key]}")).to_be_visible() + for md_key in config["spans"][trace_type]["metadata"]: + expect( + page.get_by_text( + f"{md_key}: {config['spans'][trace_type]['metadata'][md_key]}" + ) + ).to_be_visible() # provisional patchy solution, sometimes when clicking through spans very fast some of them show up as "no data" and the test fails page.wait_for_timeout(500) def test_dataset_name(datasets_page, config, dataset): - ''' + """ Checks that the dataset created via the fixture as defined in sanity_config.yaml is present on the datasets page - ''' - datasets_page.check_dataset_exists_on_page_by_name(config['dataset']['name']) + """ + datasets_page.check_dataset_exists_on_page_by_name(config["dataset"]["name"]) def test_dataset_items(page: Page, datasets_page, config, dataset_content): - ''' + """ Checks that the traces created via the fixture and defined in sanity_dataset.jsonl are present within the dataset - ''' - datasets_page.select_database_by_name(config['dataset']['name']) + """ + datasets_page.select_database_by_name(config["dataset"]["name"]) individual_dataset_page = IndividualDatasetPage(page) for item in dataset_content: - individual_dataset_page.check_cell_exists_by_text(item['input']) - individual_dataset_page.check_cell_exists_by_text(item['expected_output']) - + individual_dataset_page.check_cell_exists_by_text(item["input"]) + individual_dataset_page.check_cell_exists_by_text(item["expected_output"]) + def test_experiments_exist(experiments_page, config, create_experiments): - ''' + """ Checks that the experiments created via the fixture are present and have the correct values for the metrics (experiments defined in a way to always return the same results) - ''' - experiments_page.check_experiment_exists_by_name('test-experiment-Equals') - experiments_page.page.get_by_role('link', name='test-experiment-Equals').click() - assert 'Equals\n0' == experiments_page.page.get_by_test_id('feedback-score-tag').first.inner_text() - - experiments_page.page.get_by_role('link', name='Experiments').click() - - experiments_page.check_experiment_exists_by_name('test-experiment-Contains') - experiments_page.page.get_by_role('link', name='test-experiment-Contains').click() - assert 'Contains\n1' == experiments_page.page.get_by_test_id('feedback-score-tag').first.inner_text() \ No newline at end of file + """ + experiments_page.check_experiment_exists_by_name("test-experiment-Equals") + experiments_page.page.get_by_role("link", name="test-experiment-Equals").click() + assert ( + "Equals\n0" + == experiments_page.page.get_by_test_id("feedback-score-tag").first.inner_text() + ) + + experiments_page.page.get_by_role("link", name="Experiments").click() + + experiments_page.check_experiment_exists_by_name("test-experiment-Contains") + experiments_page.page.get_by_role("link", name="test-experiment-Contains").click() + assert ( + "Contains\n1" + == experiments_page.page.get_by_test_id("feedback-score-tag").first.inner_text() + ) diff --git a/tests_end_to_end/tests/conftest.py b/tests_end_to_end/tests/conftest.py index a5be6a72d7..d0819dfef2 100644 --- a/tests_end_to_end/tests/conftest.py +++ b/tests_end_to_end/tests/conftest.py @@ -1,26 +1,23 @@ import pytest import os import opik -import yaml -import json -from opik.configurator.configure import configure -from opik.evaluation import evaluate -from opik.evaluation.metrics import Contains, Equals -from opik import opik_context, track -from playwright.sync_api import Page, BrowserContext, Browser -import time +from playwright.sync_api import Page, Browser from page_objects.ProjectsPage import ProjectsPage from page_objects.TracesPage import TracesPage from page_objects.DatasetsPage import DatasetsPage from page_objects.ExperimentsPage import ExperimentsPage -from tests.sdk_helpers import create_project_sdk, delete_project_by_name_sdk, wait_for_number_of_traces_to_be_visible, delete_dataset_by_name_if_exists +from tests.sdk_helpers import ( + create_project_sdk, + delete_project_by_name_sdk, + wait_for_number_of_traces_to_be_visible, +) from utils import TEST_ITEMS @pytest.fixture def browser_clipboard_permissions(browser: Browser): context = browser.new_context() - context.grant_permissions(['clipboard-read', 'clipboard-write']) + context.grant_permissions(["clipboard-read", "clipboard-write"]) yield context context.close() @@ -32,26 +29,25 @@ def page_with_clipboard_perms(browser_clipboard_permissions): page.close() - -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def configure_local(): - os.environ['OPIK_URL_OVERRIDE'] = "http://localhost:5173/api" - os.environ['OPIK_WORKSPACE'] = 'default' + os.environ["OPIK_URL_OVERRIDE"] = "http://localhost:5173/api" + os.environ["OPIK_WORKSPACE"] = "default" -@pytest.fixture(scope='session', autouse=True) +@pytest.fixture(scope="session", autouse=True) def client() -> opik.Opik: - return opik.Opik(workspace='default', host='http://localhost:5173/api') + return opik.Opik(workspace="default", host="http://localhost:5173/api") -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def projects_page(page: Page): projects_page = ProjectsPage(page) projects_page.go_to_page() return projects_page - -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def projects_page_timeout(page: Page) -> ProjectsPage: projects_page = ProjectsPage(page) projects_page.go_to_page() @@ -59,39 +55,38 @@ def projects_page_timeout(page: Page) -> ProjectsPage: return projects_page -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def traces_page(page: Page, projects_page, config): - projects_page.click_project(config['project']['name']) + projects_page.click_project(config["project"]["name"]) traces_page = TracesPage(page) return traces_page -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def datasets_page(page: Page): datasets_page = DatasetsPage(page) datasets_page.go_to_page() return datasets_page -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def experiments_page(page: Page): experiments_page = ExperimentsPage(page) experiments_page.go_to_page() return experiments_page - -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_project_sdk_no_cleanup(): - proj_name = 'projects_crud_tests_sdk' + proj_name = "projects_crud_tests_sdk" create_project_sdk(name=proj_name) yield proj_name -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_project_ui_no_cleanup(page: Page): - proj_name = 'projects_crud_tests_ui' + proj_name = "projects_crud_tests_ui" projects_page = ProjectsPage(page) projects_page.go_to_page() projects_page.create_new_project(project_name=proj_name) @@ -99,19 +94,19 @@ def create_project_ui_no_cleanup(page: Page): yield proj_name -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_delete_project_sdk(): - proj_name = 'automated_tests_project' + proj_name = "automated_tests_project" create_project_sdk(name=proj_name) - os.environ['OPIK_PROJECT_NAME'] = proj_name + os.environ["OPIK_PROJECT_NAME"] = proj_name yield proj_name delete_project_by_name_sdk(name=proj_name) @pytest.fixture def create_delete_project_ui(page: Page): - proj_name = 'automated_tests_project' + proj_name = "automated_tests_project" projects_page = ProjectsPage(page) projects_page.go_to_page() projects_page.create_new_project(project_name=proj_name) @@ -120,35 +115,35 @@ def create_delete_project_ui(page: Page): delete_project_by_name_sdk(name=proj_name) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_delete_dataset_sdk(client: opik.Opik): - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" client.create_dataset(name=dataset_name) yield dataset_name client.delete_dataset(name=dataset_name) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_delete_dataset_ui(page: Page, client: opik.Opik): - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" datasets_page = DatasetsPage(page) datasets_page.go_to_page() datasets_page.create_dataset_by_name(dataset_name=dataset_name) - + yield dataset_name client.delete_dataset(name=dataset_name) -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_dataset_sdk_no_cleanup(client: opik.Opik): - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" client.create_dataset(name=dataset_name) yield dataset_name -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def create_dataset_ui_no_cleanup(page: Page): - dataset_name = 'automated_tests_dataset' + dataset_name = "automated_tests_dataset" datasets_page = DatasetsPage(page) datasets_page.go_to_page() datasets_page.create_dataset_by_name(dataset_name=dataset_name) @@ -160,15 +155,16 @@ def insert_dataset_items_sdk(client: opik.Opik, create_delete_dataset_sdk): dataset = client.get_dataset(create_delete_dataset_sdk) dataset.insert(TEST_ITEMS) + @pytest.fixture def create_10_test_traces(page: Page, client, create_delete_project_sdk): proj_name = create_delete_project_sdk for i in range(10): _ = client.trace( - name=f'trace{i}', + name=f"trace{i}", project_name=proj_name, - input={'input': 'test input'}, - output={'output': 'test output'}, + input={"input": "test input"}, + output={"output": "test output"}, ) wait_for_number_of_traces_to_be_visible(project_name=proj_name, number_of_traces=10) - yield \ No newline at end of file + yield diff --git a/tests_end_to_end/tests/sdk_helpers.py b/tests_end_to_end/tests/sdk_helpers.py index 0e3eff441d..33357f45e8 100644 --- a/tests_end_to_end/tests/sdk_helpers.py +++ b/tests_end_to_end/tests/sdk_helpers.py @@ -1,13 +1,6 @@ -import pytest -import os import time import datetime as dt -import opik -from playwright.sync_api import Page -from page_objects.DatasetItemsPage import DatasetItemsPage -from page_objects.DatasetsPage import DatasetsPage from opik.rest_api.client import OpikApi -from opik import track from typing import Optional import json @@ -20,34 +13,36 @@ def create_project_sdk(name: str): def find_project_by_name_sdk(name: str): client = OpikApi() proj_page = client.projects.find_projects(name=name, page=1, size=1) - return proj_page.dict()['content'] + return proj_page.dict()["content"] def delete_project_by_name_sdk(name: str): client = OpikApi() project = find_project_by_name_sdk(name=name) - client.projects.delete_project_by_id(project[0]['id']) + client.projects.delete_project_by_id(project[0]["id"]) def wait_for_project_to_be_visible(project_name, timeout=10, initial_delay=1): start_time = time.time() delay = initial_delay - + while time.time() - start_time < timeout: if find_project_by_name_sdk(project_name): return True - + time.sleep(delay) - delay = min(delay*2, timeout-(time.time() - start_time)) - - raise TimeoutError(f'could not get created project {project_name} via API within {timeout} seconds') + delay = min(delay * 2, timeout - (time.time() - start_time)) + + raise TimeoutError( + f"could not get created project {project_name} via API within {timeout} seconds" + ) def update_project_by_name_sdk(name: str, new_name: str): client = OpikApi() wait_for_project_to_be_visible(name, timeout=10) projects_match = find_project_by_name_sdk(name) - project_id = projects_match[0]['id'] + project_id = projects_match[0]["id"] client.projects.update_project(id=project_id, name=new_name) @@ -58,45 +53,53 @@ def create_traces_sdk(prefix: str, project_name: str, qty: int): client = OpikApi() for i in range(qty): client.traces.create_trace( - name=prefix+str(i), + name=prefix + str(i), project_name=project_name, - start_time=dt.datetime.now() + start_time=dt.datetime.now(), ) def wait_for_traces_to_be_visible(project_name, size, timeout=10, initial_delay=1): start_time = time.time() delay = initial_delay - + while time.time() - start_time < timeout: if get_traces_of_project_sdk(project_name=project_name, size=size): return True - + time.sleep(delay) - delay = min(delay*2, timeout-(time.time() - start_time)) - - raise TimeoutError(f'could not get traces of project {project_name} via API within {timeout} seconds') + delay = min(delay * 2, timeout - (time.time() - start_time)) + + raise TimeoutError( + f"could not get traces of project {project_name} via API within {timeout} seconds" + ) -def wait_for_number_of_traces_to_be_visible(project_name, number_of_traces, timeout=10, initial_delay=1): +def wait_for_number_of_traces_to_be_visible( + project_name, number_of_traces, timeout=10, initial_delay=1 +): start_time = time.time() delay = initial_delay - + while time.time() - start_time < timeout: - traces = get_traces_of_project_sdk(project_name=project_name, size=number_of_traces) + traces = get_traces_of_project_sdk( + project_name=project_name, size=number_of_traces + ) if len(traces) >= number_of_traces: return True - + time.sleep(delay) - delay = min(delay*2, timeout-(time.time() - start_time)) - - raise TimeoutError(f'could not get {number_of_traces} traces of project {project_name} via API within {timeout} seconds') + delay = min(delay * 2, timeout - (time.time() - start_time)) + + raise TimeoutError( + f"could not get {number_of_traces} traces of project {project_name} via API within {timeout} seconds" + ) def get_traces_of_project_sdk(project_name: str, size: int): client = OpikApi() traces = client.traces.get_traces_by_project(project_name=project_name, size=size) - return traces.dict()['content'] + return traces.dict()["content"] def delete_list_of_traces_sdk(ids: list[str]): @@ -106,7 +109,9 @@ def delete_list_of_traces_sdk(ids: list[str]): def update_trace_by_id(id: str): client = OpikApi() - client.traces.update_trace(id=id, ) + client.traces.update_trace( + id=id, + ) def get_dataset_by_name(dataset_name: str): @@ -118,7 +123,7 @@ def get_dataset_by_name(dataset_name: str): def update_dataset_name(name: str, new_name: str): client = OpikApi() dataset = get_dataset_by_name(dataset_name=name) - dataset_id = dataset['id'] + dataset_id = dataset["id"] dataset = client.datasets.update_dataset(id=dataset_id, name=new_name) @@ -130,8 +135,8 @@ def delete_dataset_by_name_if_exists(dataset_name: str): dataset = None try: dataset = get_dataset_by_name(dataset_name) - except Exception as e: - print(f'Trying to delete dataset {dataset_name}, but it does not exist') + except Exception as _: + print(f"Trying to delete dataset {dataset_name}, but it does not exist") finally: if dataset: client.datasets.delete_dataset_by_name(dataset_name=dataset_name) @@ -142,17 +147,24 @@ def get_experiment_by_id(exp_id: str): exp = client.experiments.get_experiment_by_id(exp_id) return exp + def delete_experiment_by_id(exp_id: str): client = OpikApi() client.experiments.delete_experiments_by_id(ids=[exp_id]) + def delete_experiment_items_by_id(ids: list[str]): client = OpikApi() client.experiments.delete_experiment_items(ids=ids) + def experiment_items_stream(exp_name: str, limit: Optional[int] = None): client = OpikApi() - data = b''.join(client.experiments.stream_experiment_items(experiment_name=exp_name, request_options={'chunk_size': 100})) - lines = data.decode('utf-8').split('\r\n') + data = b"".join( + client.experiments.stream_experiment_items( + experiment_name=exp_name, request_options={"chunk_size": 100} + ) + ) + lines = data.decode("utf-8").split("\r\n") dict_list = [json.loads(line) for line in lines if line.strip()] - return dict_list \ No newline at end of file + return dict_list diff --git a/tests_end_to_end/tests/utils.py b/tests_end_to_end/tests/utils.py index fc9cb0b560..55c2311d57 100644 --- a/tests_end_to_end/tests/utils.py +++ b/tests_end_to_end/tests/utils.py @@ -1,26 +1,26 @@ TEST_ITEMS = [ - {'input': 'input0', 'output': 'output0'}, - {'input': 'input1', 'output': 'output1'}, - {'input': 'input2', 'output': 'output2'}, - {'input': 'input3', 'output': 'output3'}, - {'input': 'input4', 'output': 'output4'}, - {'input': 'input5', 'output': 'output5'}, - {'input': 'input6', 'output': 'output6'}, - {'input': 'input7', 'output': 'output7'}, - {'input': 'input8', 'output': 'output8'}, - {'input': 'input9', 'output': 'output9'} + {"input": "input0", "output": "output0"}, + {"input": "input1", "output": "output1"}, + {"input": "input2", "output": "output2"}, + {"input": "input3", "output": "output3"}, + {"input": "input4", "output": "output4"}, + {"input": "input5", "output": "output5"}, + {"input": "input6", "output": "output6"}, + {"input": "input7", "output": "output7"}, + {"input": "input8", "output": "output8"}, + {"input": "input9", "output": "output9"}, ] TEST_ITEMS_UPDATE = [ - {'input': 'update-input0', 'output': 'update-output0'}, - {'input': 'update-input1', 'output': 'update-output1'}, - {'input': 'update-input2', 'output': 'update-output2'}, - {'input': 'update-input3', 'output': 'update-output3'}, - {'input': 'update-input4', 'output': 'update-output4'}, - {'input': 'update-input5', 'output': 'update-output5'}, - {'input': 'update-input6', 'output': 'update-output6'}, - {'input': 'update-input7', 'output': 'update-output7'}, - {'input': 'update-input8', 'output': 'update-output8'}, - {'input': 'update-input9', 'output': 'update-output9'} -] \ No newline at end of file + {"input": "update-input0", "output": "update-output0"}, + {"input": "update-input1", "output": "update-output1"}, + {"input": "update-input2", "output": "update-output2"}, + {"input": "update-input3", "output": "update-output3"}, + {"input": "update-input4", "output": "update-output4"}, + {"input": "update-input5", "output": "update-output5"}, + {"input": "update-input6", "output": "update-output6"}, + {"input": "update-input7", "output": "update-output7"}, + {"input": "update-input8", "output": "update-output8"}, + {"input": "update-input9", "output": "update-output9"}, +] From e8252ac6916694e8a9719d7570964b659c0b7743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 13:21:45 +0200 Subject: [PATCH 16/17] fixed imports --- tests_end_to_end/tests/Datasets/conftest.py | 2 +- .../tests/Datasets/test_dataset_items_crud_operations.py | 2 +- tests_end_to_end/tests/Traces/conftest.py | 2 +- tests_end_to_end/tests/Traces/test_traces_crud_operations.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests_end_to_end/tests/Datasets/conftest.py b/tests_end_to_end/tests/Datasets/conftest.py index 2377783528..280fe2ef3d 100644 --- a/tests_end_to_end/tests/Datasets/conftest.py +++ b/tests_end_to_end/tests/Datasets/conftest.py @@ -1,6 +1,6 @@ import pytest from opik import Opik -from datasets_utils import TEST_ITEMS +from Datasets.datasets_utils import TEST_ITEMS @pytest.fixture diff --git a/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py b/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py index 913f51a85d..710182c6e8 100644 --- a/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py +++ b/tests_end_to_end/tests/Datasets/test_dataset_items_crud_operations.py @@ -2,7 +2,7 @@ from playwright.sync_api import Page, expect from page_objects.DatasetsPage import DatasetsPage from page_objects.DatasetItemsPage import DatasetItemsPage -from datasets_utils import ( +from Datasets.datasets_utils import ( TEST_ITEMS, TEST_ITEMS_UPDATE, compare_item_lists, diff --git a/tests_end_to_end/tests/Traces/conftest.py b/tests_end_to_end/tests/Traces/conftest.py index 80e68da790..fb4d9845ea 100644 --- a/tests_end_to_end/tests/Traces/conftest.py +++ b/tests_end_to_end/tests/Traces/conftest.py @@ -1,6 +1,6 @@ import pytest from opik import opik_context, track -from traces_config import PREFIX, PROJECT_NAME +from Traces.traces_config import PREFIX, PROJECT_NAME from sdk_helpers import wait_for_number_of_traces_to_be_visible diff --git a/tests_end_to_end/tests/Traces/test_traces_crud_operations.py b/tests_end_to_end/tests/Traces/test_traces_crud_operations.py index 8ea69a593b..415cb32e57 100644 --- a/tests_end_to_end/tests/Traces/test_traces_crud_operations.py +++ b/tests_end_to_end/tests/Traces/test_traces_crud_operations.py @@ -2,7 +2,7 @@ from playwright.sync_api import Page, expect from page_objects.ProjectsPage import ProjectsPage from page_objects.TracesPage import TracesPage -from traces_config import PREFIX +from Traces.traces_config import PREFIX from collections import Counter from sdk_helpers import ( get_traces_of_project_sdk, From 38a72c68e7ed8122d92de03222aca75d59bfe31a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrei=20C=C4=83uti=C8=99anu?= Date: Thu, 12 Dec 2024 16:31:42 +0200 Subject: [PATCH 17/17] lint fix --- tests_end_to_end/tests/sdk_helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests_end_to_end/tests/sdk_helpers.py b/tests_end_to_end/tests/sdk_helpers.py index 6b8f98b0af..33357f45e8 100644 --- a/tests_end_to_end/tests/sdk_helpers.py +++ b/tests_end_to_end/tests/sdk_helpers.py @@ -152,6 +152,7 @@ def delete_experiment_by_id(exp_id: str): client = OpikApi() client.experiments.delete_experiments_by_id(ids=[exp_id]) + def delete_experiment_items_by_id(ids: list[str]): client = OpikApi() client.experiments.delete_experiment_items(ids=ids)