From 8d047898d889b157d5e0475d0cf0c6310c16d945 Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Tue, 15 Mar 2022 15:34:50 -0700 Subject: [PATCH] Tests for Extensions (#978) * Added tests for extensions. Checking if they work. * dummy commit to see if things work. * fix * debug * bug fix. * possible fix. * tweeking context * Bug fix to tests. * dummy commit * bug fix * Added extension test to core tests. - remove seperate gh action * removing files. * fix * added extension test to py3 context - remove redundant complexity. --- test/core/contexts.json | 6 +- test/core/tests/card_extension_test.py | 58 +++++++++++++++++++ test/extensions/README.md | 5 ++ test/extensions/install_packages.sh | 3 + .../packages/card_via_extinit/README.md | 3 + .../plugins/cards/card_a/__init__.py | 15 +++++ .../plugins/cards/card_b/__init__.py | 15 +++++ .../plugins/cards/mfextinit_X.py | 4 ++ .../packages/card_via_extinit/setup.py | 21 +++++++ .../packages/card_via_init/README.md | 3 + .../card_via_init/plugins/cards/__init__.py | 15 +++++ .../packages/card_via_init/setup.py | 21 +++++++ .../packages/card_via_ns_subpackage/README.md | 3 + .../plugins/cards/nssubpackage/__init__.py | 15 +++++ .../packages/card_via_ns_subpackage/setup.py | 21 +++++++ test_runner | 8 ++- 16 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 test/core/tests/card_extension_test.py create mode 100644 test/extensions/README.md create mode 100644 test/extensions/install_packages.sh create mode 100644 test/extensions/packages/card_via_extinit/README.md create mode 100644 test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/card_a/__init__.py create mode 100644 test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/card_b/__init__.py create mode 100644 test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/mfextinit_X.py create mode 100644 test/extensions/packages/card_via_extinit/setup.py create mode 100644 test/extensions/packages/card_via_init/README.md create mode 100644 test/extensions/packages/card_via_init/metaflow_extensions/card_via_init/plugins/cards/__init__.py create mode 100644 test/extensions/packages/card_via_init/setup.py create mode 100644 test/extensions/packages/card_via_ns_subpackage/README.md create mode 100644 test/extensions/packages/card_via_ns_subpackage/metaflow_extensions/card_via_ns_subpackage/plugins/cards/nssubpackage/__init__.py create mode 100644 test/extensions/packages/card_via_ns_subpackage/setup.py diff --git a/test/core/contexts.json b/test/core/contexts.json index d4c241f0cac..03cab966610 100644 --- a/test/core/contexts.json +++ b/test/core/contexts.json @@ -91,7 +91,8 @@ "BasicUnboundedForeachTest", "NestedUnboundedForeachTest", "DetectSegFaultTest", - "TimeoutDecoratorTest" + "TimeoutDecoratorTest", + "CardExtensionsImportTest" ] }, { @@ -126,7 +127,8 @@ "BasicUnboundedForeachTest", "NestedUnboundedForeachTest", "DetectSegFaultTest", - "TimeoutDecoratorTest" + "TimeoutDecoratorTest", + "CardExtensionsImportTest" ] } ], diff --git a/test/core/tests/card_extension_test.py b/test/core/tests/card_extension_test.py new file mode 100644 index 00000000000..d1efdbe6c20 --- /dev/null +++ b/test/core/tests/card_extension_test.py @@ -0,0 +1,58 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + + +class CardExtensionsImportTest(MetaflowTest): + """ + - Requires on tests/extensions/packages to be installed. + """ + + PRIORITY = 5 + + @tag('card(type="card_ext_init_b",save_errors=False)') + @tag('card(type="card_ext_init_a",save_errors=False)') + @tag('card(type="card_ns_subpackage",save_errors=False)') + @tag('card(type="card_init",save_errors=False)') + @steps(0, ["start"]) + def step_start(self): + from metaflow import current + + self.task = current.pathspec + + @steps(1, ["all"]) + def step_all(self): + pass + + def check_results(self, flow, checker): + run = checker.get_run() + if run is None: + # This means CliCheck is in context. + for step in flow: + if step != "start": + continue + cli_check_dict = checker.artifact_dict(step.name, "task") + for task_pathspec in cli_check_dict: + full_pathspec = "/".join([flow.name, task_pathspec]) + task_id = task_pathspec.split("/")[-1] + cards_info = checker.list_cards(step.name, task_id) + # Just check if the cards are created. + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 4, + True, + ) + else: + # This means MetadataCheck is in context. + for step in flow: + if step.name != "start": + continue + meta_check_dict = checker.artifact_dict(step.name, "task") + for task_id in meta_check_dict: + full_pathspec = meta_check_dict[task_id]["task"] + cards_info = checker.list_cards(step.name, task_id) + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 4, + True, + ) diff --git a/test/extensions/README.md b/test/extensions/README.md new file mode 100644 index 00000000000..f135475d26a --- /dev/null +++ b/test/extensions/README.md @@ -0,0 +1,5 @@ +# Extensions Testing Framework. + +What does this framework do ? It installs the extensions and then runs the test suite which leverages the extensions. + +Currently installs the cards related packages. \ No newline at end of file diff --git a/test/extensions/install_packages.sh b/test/extensions/install_packages.sh new file mode 100644 index 00000000000..0d546ec6eb7 --- /dev/null +++ b/test/extensions/install_packages.sh @@ -0,0 +1,3 @@ +pip install ./packages/card_via_extinit +pip install ./packages/card_via_init +pip install ./packages/card_via_ns_subpackage \ No newline at end of file diff --git a/test/extensions/packages/card_via_extinit/README.md b/test/extensions/packages/card_via_extinit/README.md new file mode 100644 index 00000000000..72dfd8a76ff --- /dev/null +++ b/test/extensions/packages/card_via_extinit/README.md @@ -0,0 +1,3 @@ +# card_via_extinit + +This test will check if card extensions installed with `mfextinit_*.py` work with Metaflow. \ No newline at end of file diff --git a/test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/card_a/__init__.py b/test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/card_a/__init__.py new file mode 100644 index 00000000000..87baa717d28 --- /dev/null +++ b/test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/card_a/__init__.py @@ -0,0 +1,15 @@ +from metaflow.cards import MetaflowCard + + +class TestMockCard(MetaflowCard): + type = "card_ext_init_a" + + def __init__(self, options={"key": "task"}, **kwargs): + self._key = options["key"] if "key" in options else "task" + + def render(self, task): + task_data = task[self._key].data + return "%s" % task_data + + +CARDS = [TestMockCard] diff --git a/test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/card_b/__init__.py b/test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/card_b/__init__.py new file mode 100644 index 00000000000..37cc6268a53 --- /dev/null +++ b/test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/card_b/__init__.py @@ -0,0 +1,15 @@ +from metaflow.cards import MetaflowCard + + +class TestMockCard(MetaflowCard): + type = "card_ext_init_b" + + def __init__(self, options={"key": "task"}, **kwargs): + self._key = options["key"] if "key" in options else "task" + + def render(self, task): + task_data = task[self._key].data + return "%s" % task_data + + +CARDS = [TestMockCard] diff --git a/test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/mfextinit_X.py b/test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/mfextinit_X.py new file mode 100644 index 00000000000..8bcb1a36f0e --- /dev/null +++ b/test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/mfextinit_X.py @@ -0,0 +1,4 @@ +from .card_a import CARDS as a +from .card_b import CARDS as b + +CARDS = a + b diff --git a/test/extensions/packages/card_via_extinit/setup.py b/test/extensions/packages/card_via_extinit/setup.py new file mode 100644 index 00000000000..abc638dabc4 --- /dev/null +++ b/test/extensions/packages/card_via_extinit/setup.py @@ -0,0 +1,21 @@ +from setuptools import find_namespace_packages, setup + + +def get_long_description() -> str: + with open("README.md") as fh: + return fh.read() + + +setup( + name="metaflow-card-via-extinit", + version="1.0.0", + description="A desription of your card", + long_description=get_long_description(), + long_description_content_type="text/markdown", + author="Your Name", + author_email="your_name@yourdomain.com", + license="Apache Software License 2.0", + packages=find_namespace_packages(include=["metaflow_extensions.*"]), + include_package_data=True, + zip_safe=False, +) diff --git a/test/extensions/packages/card_via_init/README.md b/test/extensions/packages/card_via_init/README.md new file mode 100644 index 00000000000..2ac3fcae4d8 --- /dev/null +++ b/test/extensions/packages/card_via_init/README.md @@ -0,0 +1,3 @@ +# card_via_init + +This test checks if card extensions directly with a `plugins/cards` directory structure work as planned. \ No newline at end of file diff --git a/test/extensions/packages/card_via_init/metaflow_extensions/card_via_init/plugins/cards/__init__.py b/test/extensions/packages/card_via_init/metaflow_extensions/card_via_init/plugins/cards/__init__.py new file mode 100644 index 00000000000..d3d4160c118 --- /dev/null +++ b/test/extensions/packages/card_via_init/metaflow_extensions/card_via_init/plugins/cards/__init__.py @@ -0,0 +1,15 @@ +from metaflow.cards import MetaflowCard + + +class TestMockCard(MetaflowCard): + type = "card_init" + + def __init__(self, options={"key": "task"}, **kwargs): + self._key = options["key"] if "key" in options else "task" + + def render(self, task): + task_data = task[self._key].data + return "%s" % task_data + + +CARDS = [TestMockCard] diff --git a/test/extensions/packages/card_via_init/setup.py b/test/extensions/packages/card_via_init/setup.py new file mode 100644 index 00000000000..9927c309e59 --- /dev/null +++ b/test/extensions/packages/card_via_init/setup.py @@ -0,0 +1,21 @@ +from setuptools import find_namespace_packages, setup + + +def get_long_description() -> str: + with open("README.md") as fh: + return fh.read() + + +setup( + name="metaflow-card-via-init", + version="1.0.0", + description="A desription of your card", + long_description=get_long_description(), + long_description_content_type="text/markdown", + author="Your Name", + author_email="your_name@yourdomain.com", + license="Apache Software License 2.0", + packages=find_namespace_packages(include=["metaflow_extensions.*"]), + include_package_data=True, + zip_safe=False, +) diff --git a/test/extensions/packages/card_via_ns_subpackage/README.md b/test/extensions/packages/card_via_ns_subpackage/README.md new file mode 100644 index 00000000000..2edcf43ae0d --- /dev/null +++ b/test/extensions/packages/card_via_ns_subpackage/README.md @@ -0,0 +1,3 @@ +# card_ns_subpackage + +This test will check if card extensions installed subpackages under namespace packages work \ No newline at end of file diff --git a/test/extensions/packages/card_via_ns_subpackage/metaflow_extensions/card_via_ns_subpackage/plugins/cards/nssubpackage/__init__.py b/test/extensions/packages/card_via_ns_subpackage/metaflow_extensions/card_via_ns_subpackage/plugins/cards/nssubpackage/__init__.py new file mode 100644 index 00000000000..36b3bd2dead --- /dev/null +++ b/test/extensions/packages/card_via_ns_subpackage/metaflow_extensions/card_via_ns_subpackage/plugins/cards/nssubpackage/__init__.py @@ -0,0 +1,15 @@ +from metaflow.cards import MetaflowCard + + +class TestMockCard(MetaflowCard): + type = "card_ns_subpackage" + + def __init__(self, options={"key": "task"}, **kwargs): + self._key = options["key"] if "key" in options else "task" + + def render(self, task): + task_data = task[self._key].data + return "%s" % task_data + + +CARDS = [TestMockCard] diff --git a/test/extensions/packages/card_via_ns_subpackage/setup.py b/test/extensions/packages/card_via_ns_subpackage/setup.py new file mode 100644 index 00000000000..b9e6ca07f9e --- /dev/null +++ b/test/extensions/packages/card_via_ns_subpackage/setup.py @@ -0,0 +1,21 @@ +from setuptools import find_namespace_packages, setup + + +def get_long_description() -> str: + with open("README.md") as fh: + return fh.read() + + +setup( + name="metaflow-card-via-nspackage", + version="1.0.0", + description="A desription of your card", + long_description=get_long_description(), + long_description_content_type="text/markdown", + author="Your Name", + author_email="your_name@yourdomain.com", + license="Apache Software License 2.0", + packages=find_namespace_packages(include=["metaflow_extensions.*"]), + include_package_data=True, + zip_safe=False, +) diff --git a/test_runner b/test_runner index 85cc7ab6bab..8de1f3ecdf9 100755 --- a/test_runner +++ b/test_runner @@ -9,8 +9,14 @@ install_deps() { done } +install_extensions() { + cd test/extensions + sh install_packages.sh + cd ../../ +} + run_tests() { cd test/core && PYTHONPATH=`pwd`/../../ python3 run_tests.py --num-parallel 8 } -install_deps && run_tests +install_deps && install_extensions && run_tests