From bf95a017196befe5ce6c2e22ad4ba336699fe9df Mon Sep 17 00:00:00 2001 From: Viraj <35092918+angrybayblade@users.noreply.github.com> Date: Mon, 19 Aug 2024 18:33:35 +0530 Subject: [PATCH] Run example tests parallelly (#460) --- .github/workflows/examples.yml | 28 +++++++++++++++++++------ .github/workflows/release.yaml | 8 ------- python/plugins/langchain/setup.py | 2 +- python/plugins/openai/setup.py | 2 +- python/tests/test_example.py | 35 ++++++++++++++++++++++--------- 5 files changed, 49 insertions(+), 26 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 09d411463d3..58068c23707 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -17,6 +17,22 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.10"] + package: + [ + autogen, + camel, + claude, + crew_ai, + griptape, + julep, + langchain, + llamaindex, + lyzr, + openai, + praisonai, + langgraph, + phidata, + ] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@master @@ -36,10 +52,10 @@ jobs: sudo apt-get autoclean python -m pip install --upgrade pip pipenv pytest - ls -1 plugins | awk '$0="plugins/"$0"/"' | xargs -L1 pipenv run pip3 install - pipenv install - pipenv run pip3 install '.[all]' - pipenv run pip install 'numpy<2' - pipenv run composio apps update + python -m pip install plugins/${{matrix.package}} + python -m pip install '.[all]' + python -m pip install 'numpy<2' python-dotenv - CI=false COMPOSIO_BASE_URL=https://backend.composio.dev/api pipenv run pytest -vv tests/test_example.py \ No newline at end of file + composio apps update + + CI=false COMPOSIO_BASE_URL=https://backend.composio.dev/api PLUGIN_TO_TEST=${{matrix.package}} pytest -vv tests/test_example.py diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8ed20774e1a..717eaf76cc9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -11,16 +11,8 @@ on: default: false jobs: - # test-examples: - # if: ${{ !inputs.skip_tests }} - # uses: ./.github/workflows/examples.yml - # with: - # working-directory: ./python - # secrets: inherit - publish-core: name: Create Framework & Plugin Releases - # needs: test-examples defaults: run: working-directory: ./python diff --git a/python/plugins/langchain/setup.py b/python/plugins/langchain/setup.py index 11e52713b29..c322af1d8e6 100644 --- a/python/plugins/langchain/setup.py +++ b/python/plugins/langchain/setup.py @@ -23,7 +23,7 @@ ], python_requires=">=3.9,<4", install_requires=[ - "langchain>=0.1.0", + "langchain>=0.1.0,<0.2.0", "langchain-openai>=0.0.2.post1", "pydantic>=2.6.4", "langchainhub>=0.1.15", diff --git a/python/plugins/openai/setup.py b/python/plugins/openai/setup.py index 20f0f397c16..f25db57870c 100644 --- a/python/plugins/openai/setup.py +++ b/python/plugins/openai/setup.py @@ -22,6 +22,6 @@ "Operating System :: OS Independent", ], python_requires=">=3.9,<4", - install_requires=["composio_core==0.4.5"], + install_requires=["composio_core==0.4.5", "openai"], include_package_data=True, ) diff --git a/python/tests/test_example.py b/python/tests/test_example.py index 789fc24e41c..66e94d0f93b 100644 --- a/python/tests/test_example.py +++ b/python/tests/test_example.py @@ -23,6 +23,7 @@ # Plugin test definitions EXAMPLES = { "autogen": { + "plugin": "autogen", "file": PLUGINS / "autogen" / "autogen_demo.py", "match": { "type": "stdout", @@ -36,6 +37,7 @@ }, }, "llamaindex": { + "plugin": "llamaindex", "file": PLUGINS / "llamaindex" / "llamaindex_demo.py", "match": { "type": "stdout", @@ -49,6 +51,7 @@ }, }, "local_tools": { + "plugin": "autogen", "file": EXAMPLES_PATH / "local_tools" / "autogen_math.py", "match": { "type": "stdout", @@ -59,6 +62,7 @@ "env": {"OPENAI_API_KEY": OPENAI_API_KEY}, }, "runtime_tools": { + "plugin": "langchain", "file": EXAMPLES_PATH / "runtime_tools" / "langchain_math.py", "match": { "type": "stdout", @@ -67,6 +71,7 @@ "env": {"OPENAI_API_KEY": OPENAI_API_KEY}, }, "crewai": { + "plugin": "crewai", "file": PLUGINS / "crew_ai" / "crewai_demo.py", "match": { "type": "stdout", @@ -81,6 +86,7 @@ }, # TOFIX(@kaave): httpcore.UnsupportedProtocol: Request URL is missing an 'http://' or 'https://' protocol. # "julep": { + # "plugin": "julep", # "file": PLUGINS / "julep" / "julep_demo.py", # "match": { # "type": "stdout", @@ -94,6 +100,7 @@ # }, # }, "langchain": { + "plugin": "langchain", "file": PLUGINS / "langchain" / "langchain_demo.py", "match": { "type": "stdout", @@ -103,17 +110,19 @@ }, "env": {"OPENAI_API_KEY": OPENAI_API_KEY, "COMPOSIO_API_KEY": COMPOSIO_API_KEY}, }, - # "langgraph": { - # "file": PLUGINS / "langgraph" / "langgraph_demo.py", - # "match": { - # "type": "stdout", - # "values": [ - # "{'execution_details': {'executed': True}, 'response_data': ''}" - # ], - # }, - # "env": {"OPENAI_API_KEY": OPENAI_API_KEY, "COMPOSIO_API_KEY": COMPOSIO_API_KEY}, - # }, + "langgraph": { + "plugin": "langgraph", + "file": PLUGINS / "langgraph" / "langgraph_demo.py", + "match": { + "type": "stdout", + "values": [ + "{'execution_details': {'executed': True}, 'response_data': ''}" + ], + }, + "env": {"OPENAI_API_KEY": OPENAI_API_KEY, "COMPOSIO_API_KEY": COMPOSIO_API_KEY}, + }, "openai": { + "plugin": "openai", "file": PLUGINS / "openai" / "openai_demo.py", "match": { "type": "stdout", @@ -124,6 +133,7 @@ "env": {"OPENAI_API_KEY": OPENAI_API_KEY, "COMPOSIO_API_KEY": COMPOSIO_API_KEY}, }, "lyzr": { + "plugin": "lyzr", "file": PLUGINS / "lyzr" / "lyzr_demo.py", "match": { "type": "stdout", @@ -134,6 +144,7 @@ "env": {"OPENAI_API_KEY": OPENAI_API_KEY, "COMPOSIO_API_KEY": COMPOSIO_API_KEY}, }, # "praisonai": { + # "plugin": "praisonai", # "file": PLUGINS / "praisonai" / "praisonai_demo.py", # "match": { # "type": "stdout", @@ -156,6 +167,10 @@ def test_example( example_name: str, example: dict # pylint: disable=unused-argument ) -> None: """Test an example with given environment.""" + plugin_to_test = os.getenv("PLUGIN_TO_TEST") + if plugin_to_test is not None and plugin_to_test != example["plugin"]: + pytest.skip(f"Skipping {example['plugin']}") + for key, val in example["env"].items(): assert ( val is not None