Clean e2e test setup (#1443)

Description
Resolves #805.

Development notes
Ok so the main things done in this PR:-

Added support for the earliest Kedro version that Kedro-viz supports, which is Kedro 0.17.5. The end-to-end (e2e) tests for Kedro 0.17.5 will skip if the Python version is greater than 3.8, as Kedro 18 only works with Python 3.9 and above.
Removed old code related to Kedro 0.16.0 in environment.py, which involved unnecessary package installations.
Implemented a suggestion by @tynandebold to capture Kedro-viz breaking due to old dependencies in our end-to-end (e2e) tests. I created a lower-bound requirements.txt file for testing Kedro-viz functionality. If the tests fail, we know that we need to increase the lower-bound on our dependencies.
Added an additional assert statement to check if kedro-viz is running, as the viz flowchart for Kedro 0.17.5 and Kedro 0.18 are different.
Removed win-e2e-test for Python 3.7 since it was failing. Since we plan to drop support for Python 3.7 soon, we decided not to spend time investigating the issue.
also thanks @astrojuanlu for help on this ticket :)

.circleci/continue_config.yml

@@ -345,7 +345,6 @@ workflows:
       - win_e2e_tests:
           matrix:
             parameters:
-              #3.10 fails with win-build, hence reverting it to 3.10
               python_version: ['3.7', '3.8', '3.9', '3.10']
           filters:
             branches:

CONTRIBUTING.md

@@ -311,6 +311,12 @@ make e2e-tests
 make lint
 ```
 
+### Managing Python dependencies on Kedro-viz 
+
+Python dependencies in Kedro-Viz are usually updated automatically through tools like Dependabot or triggered by deprecation warnings. When updating dependencies, especially the lower-bound version, it's essential to also update the `lower-requirements.txt` file manually. This ensures that our e2e tests reflect the latest dependencies, maintaining project stability and compatibility.
+
+If the lower-bound e2e test fails, it indicates that some dependencies may not work correctly with Kedro-Viz. To resolve this, update the problematic dependency in both `requirements.txt` and `lower-requirements.txt`.
+
 # Release guidelines
 
 - Practice frequent, staggered releases.

package/features/environment.py

@@ -1,12 +1,16 @@
 """Behave environment setup commands"""
 
 import os
+import re
 import shutil
+import sys
 import tempfile
 import venv
 from pathlib import Path
 from typing import Set
 
+from semver import Version
+
 from features.steps.sh_run import run
 
 _PATHS_TO_REMOVE: Set[Path] = set()
@@ -26,6 +30,33 @@ def before_scenario(context, scenario):
     Installs kedro by running pip in the top level directory.
     """
 
+    kedro_version = None
+
+    if sys.version_info < (3, 8) and sys.platform.startswith("win"):
+        if "lower-bound" in scenario.name:
+            print(f"{scenario} will be skipped on Windows with Python 3.7")
+            scenario.skip()
+
+    for step in scenario.steps:
+        if "I have installed kedro version" in step.name:
+            match = re.search(r"\b\d+\.\d+\.\d+\b", step.name)
+            if match:
+                kedro_version = Version.parse(match.group(0))
+                break
+
+    if (
+        kedro_version
+        and kedro_version <= Version.parse("0.18.0")
+        and sys.version_info >= (3, 9)
+    ):
+        print(
+            (
+                f"{scenario} will be skipped as {kedro_version} is not "
+                f"compatible with python {sys.version_info.major}.{sys.version_info.minor}"
+            )
+        )
+        scenario.skip()
+
     # make a venv
     kedro_install_venv_dir = _create_new_venv()
     context.venv_dir = kedro_install_venv_dir

package/features/steps/cli_steps.py

@@ -1,5 +1,6 @@
 """Behave step definitions for the cli_scenarios feature."""
 
+from pathlib import Path
 from time import sleep, time
 
 import requests
@@ -86,6 +87,19 @@ def install_project_requirements(context):
         assert False
 
 
+@given("I have installed the lower-bound Kedro-viz requirements")
+def install_lower_bound_requirements(context):
+    cwd = Path(__file__).resolve().parent
+    requirements_path = cwd / "lower_requirements.txt"
+    cmd = [context.pip, "install", "-r", requirements_path]
+    res = run(cmd, env=context.env)
+
+    if res.returncode != OK_EXIT_CODE:
+        print(res.stdout)
+        print(res.stderr)
+        assert False
+
+
 @given('I have installed kedro version "{version}"')
 def install_kedro(context, version):
     """Install Kedro using pip."""
@@ -129,7 +143,13 @@ def check_kedroviz_up(context):
     try:
         assert context.result.poll() is None
         assert (
-            "X_test" == sorted(data_json["nodes"], key=lambda i: i["name"])[0]["name"]
+            # for Kedro 0.17.5
+            "example_iris_data"
+            == sorted(data_json["nodes"], key=lambda i: i["name"])[0]["name"]
+        ) or (
+            # for Kedro 0.18.0 onwards
+            "X_test"
+            == sorted(data_json["nodes"], key=lambda i: i["name"])[0]["name"]
         )
     finally:
         context.result.terminate()

package/features/steps/lower_requirements.txt

@@ -0,0 +1,15 @@
+semver==3.0
+ipython==7.0.0
+fastapi==0.73.0
+fsspec==2021.4
+aiofiles==22.1.0
+uvicorn[standard]==0.22.0
+watchgod==0.8.2
+plotly==4.0
+pandas==1.3; python_version < '3.10'
+pandas==1.4; python_version >= '3.10'
+sqlalchemy==1.4
+strawberry-graphql==0.192.0
+networkx==2.5
+orjson==3.9
+secure==0.3.0

package/features/viz.feature

@@ -2,9 +2,25 @@ Feature: Viz plugin in new project
     Background:
         Given I have prepared a config file with example code
 
+    Scenario: Execute viz with the earliest Kedro version that it supports 
+        Given I have installed kedro version "0.17.5"
+        And I have run a non-interactive kedro new with pandas-iris starter
+        And I have installed the project's requirements
+        When I execute the kedro viz command
+        Then kedro-viz should start successfully
+
+    Scenario: Execute viz with latest Kedro with lower-bound viz requirements
+        Given I have installed kedro version "latest"
+        And I have installed the lower-bound Kedro-viz requirements
+        And I have run a non-interactive kedro new with pandas-iris starter
+        And I have installed the project's requirements
+        When I execute the kedro viz command
+        Then kedro-viz should start successfully
+
     Scenario: Execute viz with latest Kedro
         Given I have installed kedro version "latest"
         And I have run a non-interactive kedro new with pandas-iris starter
         And I have installed the project's requirements
         When I execute the kedro viz command
         Then kedro-viz should start successfully
+

package/kedro_viz/models/flowchart.py

@@ -21,7 +21,6 @@
 except ImportError:  # pragma: no cover
     # older versions
     from kedro.io.core import DataSetError as DatasetError
-
 try:
     # kedro 0.18.12 onwards
     from kedro.io.core import AbstractDataset

package/requirements.txt

@@ -7,9 +7,9 @@ aiofiles==22.1.0
 uvicorn[standard]~=0.22.0
 watchgod~=0.8.2
 plotly>=4.0
-pandas>=0.24
+pandas>=1.3
 sqlalchemy>=1.4, <3
 strawberry-graphql>=0.192.0, <1.0
-networkx>=1.0
+networkx>=2.5
 orjson~=3.9
 secure>=0.3.0