OPIK-194 Sanity end-to-end tests - UI tests

.github/workflows/sanity.yml

@@ -0,0 +1,63 @@
+name: Install Local Version of Opik
+
+on:
+    workflow_dispatch:
+
+jobs:
+    test_installation:
+        runs-on: ubuntu-20.04
+
+        steps:
+            - name: Checkout repo
+              uses: actions/checkout@v3
+              with:
+                ref: ${{ github.ref }}
+
+            - name: Setup Python
+              uses: actions/setup-python@v5
+              with:
+                python-version: 3.12
+
+            - name: Install Opik
+              run: pip install ${{ github.workspace }}/sdks/python
+
+            - name: Install Test Dependencies
+              run: |
+                pip install -r ${{ github.workspace }}/tests_end_to_end/test_requirements.txt
+                playwright install
+
+            - name: Install Opik
+              env:
+                OPIK_USAGE_REPORT_ENABLED: false
+              run: |
+                cd ${{ github.workspace }}/deployment/docker-compose
+                docker compose up -d --build
+
+            - name: Check Docker pods are up
+              run: |
+                chmod +x ./tests_end_to_end/installer/check_docker_compose_pods.sh
+                ./tests_end_to_end/installer/check_docker_compose_pods.sh
+              shell: bash
+
+            - name: Check backend health
+              run: |
+                chmod +x ./tests_end_to_end/installer/check_backend.sh
+                ./tests_end_to_end/installer/check_backend.sh
+              shell: bash
+
+            - name: Check app is up via the UI
+              run: |
+                pytest -v -s ${{ github.workspace }}/tests_end_to_end/installer/test_app_status.py
+
+            - name: Run sanity suite
+              run: |
+                cd ${{ github.workspace }}/tests_end_to_end
+                export PYTHONPATH='.'
+                pytest -s application_sanity/test_sanity.py --browser chromium --base-url http://localhost:5173 --setup-show
+
+            - name: Stop Opik server
+              if: always()
+              run: |
+                cd ${{ github.workspace }}/deployment/docker-compose
+                docker compose down
+                cd -

tests_end_to_end/application_sanity/conftest.py

@@ -20,32 +20,33 @@ def config():
 
 @pytest.fixture(scope='session', autouse=True)
 def configure_local(config):
-    configure(use_local=True)
+    os.environ['OPIK_URL_OVERRIDE'] = "http://localhost:5173/api"
+    os.environ['OPIK_WORKSPACE'] = 'default'
     os.environ['OPIK_PROJECT_NAME'] = config['project']['name']
 
 
 @pytest.fixture(scope='session', autouse=True)
 def client(config):
-    return opik.Opik(project_name=config['project']['name'])
+    return opik.Opik(project_name=config['project']['name'], host='http://localhost:5173/api')
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture(scope='module')
 def log_traces_and_spans_low_level(client, config):
     """
     Log 5 traces with spans and subspans using the low level Opik client
     Each should have their own names, tags, metadata and feedback scores to test integrity of data transmitted
     """
 
     trace_config = {
-        'count': config['traces']['client']['count'],
+        'count': config['traces']['count'],
         'prefix': config['traces']['client']['prefix'],
         'tags': config['traces']['client']['tags'],
         'metadata': config['traces']['client']['metadata'],
         'feedback_scores': [{'name': key, 'value': value} for key, value in config['traces']['client']['feedback-scores'].items()]
     }
 
     span_config = {
-        'count': config['spans']['client']['count'],
+        'count': config['spans']['count'],
         'prefix': config['spans']['client']['prefix'],
         'tags': config['spans']['client']['tags'],
         'metadata': config['spans']['client']['metadata'],
@@ -55,41 +56,41 @@ def log_traces_and_spans_low_level(client, config):
     for trace_index in range(trace_config['count']):
         client_trace = client.trace(
             name=trace_config['prefix'] + str(trace_index),
-            input=f'input-{trace_index}',
-            output=f'output-{trace_index}',
+            input={'input': f'input-{trace_index}'},
+            output={'output': f'output-{trace_index}'},
             tags=trace_config['tags'],
             metadata=trace_config['metadata'],
             feedback_scores=trace_config['feedback_scores']
         )
         for span_index in range(span_config['count']):
             client_span = client_trace.span(
                 name=span_config['prefix'] + str(span_index),
-                input=f'input-{span_index}',
-                output=f'output-{span_index}',
+                input={'input': f'input-{span_index}'},
+                output={'output': f'output-{span_index}'},
                 tags=span_config['tags'],
                 metadata=span_config['metadata']
             )
             for score in span_config['feedback_scores']:
                 client_span.log_feedback_score(name=score['name'], value=score['value'])
 
 
-@pytest.fixture(scope='function')
+@pytest.fixture(scope='module')
 def log_traces_and_spans_decorator(config):
     """
     Log 5 traces with spans and subspans using the low level Opik client
     Each should have their own names, tags, metadata and feedback scores to test integrity of data transmitted
     """
 
     trace_config = {
-        'count': config['traces']['decorator']['count'],
+        'count': config['traces']['count'],
         'prefix': config['traces']['decorator']['prefix'],
         'tags': config['traces']['decorator']['tags'],
         'metadata': config['traces']['decorator']['metadata'],
         'feedback_scores': [{'name': key, 'value': value} for key, value in config['traces']['decorator']['feedback-scores'].items()]
     }
 
     span_config = {
-        'count': config['spans']['decorator']['count'],
+        'count': config['spans']['count'],
         'prefix': config['spans']['decorator']['prefix'],
         'tags': config['spans']['decorator']['tags'],
         'metadata': config['spans']['decorator']['metadata'],
@@ -100,12 +101,12 @@ def log_traces_and_spans_decorator(config):
     def make_span(x):
         opik_context.update_current_span(
             name=span_config['prefix'] + str(x),
-            input=f'input-{x}',
+            input={'input': f'input-{x}'},
             metadata=span_config['metadata'],
             tags=span_config['tags'],
             feedback_scores=span_config['feedback_scores']
         )
-        return f'output-{x}'
+        return {'output': f'output-{x}'}
 
     @track()
     def make_trace(x):
@@ -114,12 +115,12 @@ def make_trace(x):
 
         opik_context.update_current_trace(
             name=trace_config['prefix'] + str(x),
-            input=f'input-{x}',
+            input={'input': f'input-{x}'},
             metadata=trace_config['metadata'],
             tags=trace_config['tags'],
             feedback_scores=trace_config['feedback_scores']
         )
-        return f'output-{x}'
+        return {'output': f'output-{x}'}
 
     for x in range(trace_config['count']):
         make_trace(x)

tests_end_to_end/application_sanity/sanity_config.yaml

@@ -2,9 +2,9 @@ project:
   name: "test-project"
 
 traces:
+  count: 5
   client:
     prefix: "client-trace-"
-    count: 5
     tags: ["c-tag1", "c-tag2"]
     feedback-scores:
       c-score1: 0.5
@@ -15,7 +15,6 @@ traces:
 
   decorator:
     prefix: "decorator-trace-"
-    count: 5
     tags: ["d-tag1", "d-tag2"]
     feedback-scores:
       d-score1: 0.1
@@ -25,9 +24,9 @@ traces:
       d-md2: "val2"
 
 spans:
+  count: 2
   client:
     prefix: "client-span-"
-    count: 2
     tags: ["c-span1", "c-span2"]
     feedback-scores:
       s-score1: 0.2
@@ -38,7 +37,6 @@ spans:
 
   decorator:
     prefix: "decorator-span-"
-    count: 2
     tags: ["d-span1", "d-span2"]
     feedback-scores:
       s-score1: 0.93

tests_end_to_end/application_sanity/test_sanity.py

@@ -0,0 +1,56 @@
+import pytest
+from playwright.sync_api import Page, expect
+from page_objects.ProjectsPage import ProjectsPage
+from page_objects.TracesPage import TracesPage
+from page_objects.TracesPageSpansMenu import TracesPageSpansMenu
+
+
+def test_project_name(page: Page, log_traces_and_spans_decorator, log_traces_and_spans_low_level):
+    projects_page = ProjectsPage(page)
+    projects_page.go_to_page()
+    projects_page.check_project_exists('test-project')
+
+
+def test_traces_created(page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator):
+    #navigate to project
+    projects_page = ProjectsPage(page)
+    projects_page.go_to_page()
+
+    #wait for data to actually arrive to the frontend
+    #TODO: replace this with a smarter waiting mechanism
+    page.wait_for_timeout(5000)
+    projects_page.click_project(config['project']['name'])
+
+    #grab all traces of project
+    traces_page = TracesPage(page)
+    trace_names = traces_page.get_all_trace_names()
+
+    client_prefix = config['traces']['client']['prefix']
+    decorator_prefix = config['traces']['decorator']['prefix']
+
+    for count in range(config['traces']['count']):
+        for prefix in [client_prefix, decorator_prefix]:
+            assert prefix+str(count) in trace_names
+
+
+def test_spans_of_traces(page, config, log_traces_and_spans_low_level, log_traces_and_spans_decorator):
+    projects_page = ProjectsPage(page)
+    projects_page.go_to_page()
+
+    #wait for data to actually arrive to the frontend
+    #TODO: replace this with a smarter waiting mechanism
+    projects_page.click_project(config['project']['name'])
+
+    #grab all traces of project
+    traces_page = TracesPage(page)
+    trace_names = traces_page.get_all_trace_names()
+
+    for trace in trace_names:
+        page.get_by_text(trace).click()
+        spans_menu = TracesPageSpansMenu(page)
+        trace_type = trace.split('-')[0] # 'client' or 'decorator'
+        for count in range(config['spans']['count']):
+            prefix = config['spans'][trace_type]['prefix']
+            spans_menu.check_span_exists_by_name(f'{prefix}{count}')
+
+

tests_end_to_end/page_objects/ProjectsPage.py

@@ -0,0 +1,16 @@
+from playwright.sync_api import Page, expect
+
+class ProjectsPage:
+    def __init__(self, page: Page):
+        self.page = page
+        self.url = '/projects'
+        self.projects_table = page.get_by_role('table')
+
+    def go_to_page(self):
+        self.page.goto(self.url)
+
+    def click_project(self, project_name):
+        self.page.get_by_role('cell', name=project_name).click()
+
+    def check_project_exists(self, project_name):
+        expect(self.projects_table.get_by_role('cell', name=project_name)).to_be_visible()

tests_end_to_end/page_objects/TracesPage.py

@@ -0,0 +1,13 @@
+from playwright.sync_api import Page, expect
+
+class TracesPage:
+    def __init__(self, page: Page):
+        self.page = page
+        self.traces_table = self.page.get_by_role('table')
+        self.trace_names_selector = 'tr td:nth-child(2) div span'
+
+    def get_all_trace_names(self):
+        self.page.wait_for_selector(self.trace_names_selector)
+
+        names = self.page.locator(self.trace_names_selector).all_inner_texts()
+        return names

tests_end_to_end/page_objects/TracesPageSpansMenu.py

@@ -0,0 +1,8 @@
+from playwright.sync_api import Page, expect
+
+class TracesPageSpansMenu:
+    def __init__(self, page: Page):
+        self.page = page
+
+    def check_span_exists_by_name(self, name):
+        expect(self.page.get_by_role('button', name=name)).to_be_visible()