Tests for Extensions (#978)

* Added tests for extensions. Checking if they work.

* dummy commit to see if things work.

* fix

* debug

* bug fix.

* possible fix.

* tweeking context

* Bug fix to tests.

* dummy commit

* bug fix

* Added extension test to core tests.
- remove seperate gh action

* removing files.

* fix

* added extension test to py3 context
- remove redundant complexity.

test/core/contexts.json

@@ -91,7 +91,8 @@
                 "BasicUnboundedForeachTest",
                 "NestedUnboundedForeachTest",
                 "DetectSegFaultTest",
-                "TimeoutDecoratorTest"
+                "TimeoutDecoratorTest",
+                "CardExtensionsImportTest"
             ]
         },
         {
@@ -126,7 +127,8 @@
                 "BasicUnboundedForeachTest",
                 "NestedUnboundedForeachTest",
                 "DetectSegFaultTest",
-                "TimeoutDecoratorTest"
+                "TimeoutDecoratorTest",
+                "CardExtensionsImportTest"
             ]
         }
     ],

test/core/tests/card_extension_test.py

@@ -0,0 +1,58 @@
+from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag
+
+
+class CardExtensionsImportTest(MetaflowTest):
+    """
+    - Requires on tests/extensions/packages to be installed.
+    """
+
+    PRIORITY = 5
+
+    @tag('card(type="card_ext_init_b",save_errors=False)')
+    @tag('card(type="card_ext_init_a",save_errors=False)')
+    @tag('card(type="card_ns_subpackage",save_errors=False)')
+    @tag('card(type="card_init",save_errors=False)')
+    @steps(0, ["start"])
+    def step_start(self):
+        from metaflow import current
+
+        self.task = current.pathspec
+
+    @steps(1, ["all"])
+    def step_all(self):
+        pass
+
+    def check_results(self, flow, checker):
+        run = checker.get_run()
+        if run is None:
+            # This means CliCheck is in context.
+            for step in flow:
+                if step != "start":
+                    continue
+                cli_check_dict = checker.artifact_dict(step.name, "task")
+                for task_pathspec in cli_check_dict:
+                    full_pathspec = "/".join([flow.name, task_pathspec])
+                    task_id = task_pathspec.split("/")[-1]
+                    cards_info = checker.list_cards(step.name, task_id)
+                    # Just check if the cards are created.
+                    assert_equals(
+                        cards_info is not None
+                        and "cards" in cards_info
+                        and len(cards_info["cards"]) == 4,
+                        True,
+                    )
+        else:
+            # This means MetadataCheck is in context.
+            for step in flow:
+                if step.name != "start":
+                    continue
+                meta_check_dict = checker.artifact_dict(step.name, "task")
+                for task_id in meta_check_dict:
+                    full_pathspec = meta_check_dict[task_id]["task"]
+                    cards_info = checker.list_cards(step.name, task_id)
+                    assert_equals(
+                        cards_info is not None
+                        and "cards" in cards_info
+                        and len(cards_info["cards"]) == 4,
+                        True,
+                    )

test/extensions/README.md

@@ -0,0 +1,5 @@
+# Extensions Testing Framework. 
+
+What does this framework do ? It installs the extensions and then runs the test suite which leverages the extensions.
+
+Currently installs the cards related packages. 

test/extensions/install_packages.sh

@@ -0,0 +1,3 @@
+pip install ./packages/card_via_extinit
+pip install ./packages/card_via_init
+pip install ./packages/card_via_ns_subpackage

test/extensions/packages/card_via_extinit/README.md

@@ -0,0 +1,3 @@
+# card_via_extinit
+
+This test will check if card extensions installed with `mfextinit_*.py` work with Metaflow.

test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/card_a/__init__.py

@@ -0,0 +1,15 @@
+from metaflow.cards import MetaflowCard
+
+
+class TestMockCard(MetaflowCard):
+    type = "card_ext_init_a"
+
+    def __init__(self, options={"key": "task"}, **kwargs):
+        self._key = options["key"] if "key" in options else "task"
+
+    def render(self, task):
+        task_data = task[self._key].data
+        return "%s" % task_data
+
+
+CARDS = [TestMockCard]

test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/card_b/__init__.py

@@ -0,0 +1,15 @@
+from metaflow.cards import MetaflowCard
+
+
+class TestMockCard(MetaflowCard):
+    type = "card_ext_init_b"
+
+    def __init__(self, options={"key": "task"}, **kwargs):
+        self._key = options["key"] if "key" in options else "task"
+
+    def render(self, task):
+        task_data = task[self._key].data
+        return "%s" % task_data
+
+
+CARDS = [TestMockCard]

test/extensions/packages/card_via_extinit/metaflow_extensions/card_via_extinit/plugins/cards/mfextinit_X.py

@@ -0,0 +1,4 @@
+from .card_a import CARDS as a
+from .card_b import CARDS as b
+
+CARDS = a + b

test/extensions/packages/card_via_extinit/setup.py

@@ -0,0 +1,21 @@
+from setuptools import find_namespace_packages, setup
+
+
+def get_long_description() -> str:
+    with open("README.md") as fh:
+        return fh.read()
+
+
+setup(
+    name="metaflow-card-via-extinit",
+    version="1.0.0",
+    description="A desription of your card",
+    long_description=get_long_description(),
+    long_description_content_type="text/markdown",
+    author="Your Name",
+    author_email="[email protected]",
+    license="Apache Software License 2.0",
+    packages=find_namespace_packages(include=["metaflow_extensions.*"]),
+    include_package_data=True,
+    zip_safe=False,
+)

test/extensions/packages/card_via_init/README.md

@@ -0,0 +1,3 @@
+# card_via_init
+
+This test checks if card extensions directly with a `plugins/cards` directory structure work as planned. 

test/extensions/packages/card_via_init/metaflow_extensions/card_via_init/plugins/cards/__init__.py

@@ -0,0 +1,15 @@
+from metaflow.cards import MetaflowCard
+
+
+class TestMockCard(MetaflowCard):
+    type = "card_init"
+
+    def __init__(self, options={"key": "task"}, **kwargs):
+        self._key = options["key"] if "key" in options else "task"
+
+    def render(self, task):
+        task_data = task[self._key].data
+        return "%s" % task_data
+
+
+CARDS = [TestMockCard]

test/extensions/packages/card_via_init/setup.py

@@ -0,0 +1,21 @@
+from setuptools import find_namespace_packages, setup
+
+
+def get_long_description() -> str:
+    with open("README.md") as fh:
+        return fh.read()
+
+
+setup(
+    name="metaflow-card-via-init",
+    version="1.0.0",
+    description="A desription of your card",
+    long_description=get_long_description(),
+    long_description_content_type="text/markdown",
+    author="Your Name",
+    author_email="[email protected]",
+    license="Apache Software License 2.0",
+    packages=find_namespace_packages(include=["metaflow_extensions.*"]),
+    include_package_data=True,
+    zip_safe=False,
+)

test/extensions/packages/card_via_ns_subpackage/README.md

@@ -0,0 +1,3 @@
+# card_ns_subpackage
+
+This test will check if card extensions installed subpackages under namespace packages work

test/extensions/packages/card_via_ns_subpackage/metaflow_extensions/card_via_ns_subpackage/plugins/cards/nssubpackage/__init__.py

@@ -0,0 +1,15 @@
+from metaflow.cards import MetaflowCard
+
+
+class TestMockCard(MetaflowCard):
+    type = "card_ns_subpackage"
+
+    def __init__(self, options={"key": "task"}, **kwargs):
+        self._key = options["key"] if "key" in options else "task"
+
+    def render(self, task):
+        task_data = task[self._key].data
+        return "%s" % task_data
+
+
+CARDS = [TestMockCard]

test/extensions/packages/card_via_ns_subpackage/setup.py

@@ -0,0 +1,21 @@
+from setuptools import find_namespace_packages, setup
+
+
+def get_long_description() -> str:
+    with open("README.md") as fh:
+        return fh.read()
+
+
+setup(
+    name="metaflow-card-via-nspackage",
+    version="1.0.0",
+    description="A desription of your card",
+    long_description=get_long_description(),
+    long_description_content_type="text/markdown",
+    author="Your Name",
+    author_email="[email protected]",
+    license="Apache Software License 2.0",
+    packages=find_namespace_packages(include=["metaflow_extensions.*"]),
+    include_package_data=True,
+    zip_safe=False,
+)

test_runner

@@ -9,8 +9,14 @@ install_deps() {
   done
 }
 
+install_extensions() {
+    cd test/extensions
+    sh install_packages.sh
+    cd ../../
+}
+
 run_tests() {
   cd test/core && PYTHONPATH=`pwd`/../../ python3 run_tests.py --num-parallel 8
 }
 
-install_deps && run_tests
+install_deps && install_extensions && run_tests