From 0ad884be402007f8e6241d191cf3e02419b4029d Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Tue, 4 Jan 2022 20:58:24 -0800 Subject: [PATCH 01/12] `card view`/`get` can also now show list as json - added argument for tests --- metaflow/plugins/cards/card_cli.py | 55 ++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/metaflow/plugins/cards/card_cli.py b/metaflow/plugins/cards/card_cli.py index e8969b30b1c..0aa9e9af437 100644 --- a/metaflow/plugins/cards/card_cli.py +++ b/metaflow/plugins/cards/card_cli.py @@ -98,7 +98,13 @@ def resolve_task_from_pathspec(flow_name, pathspec): def resolve_card( - ctx, pathspec, follow_resumed=True, hash=None, type=None, card_id=None + ctx, + pathspec, + follow_resumed=True, + hash=None, + type=None, + card_id=None, + show_list_as_json=False, ): """Resolves the card path based on the arguments provided. We allow identifier to be a pathspec or a id of card. @@ -108,6 +114,7 @@ def resolve_card( hash (optional): This is to specifically resolve the card via the hash. This is useful when there may be many card with same id or type for a pathspec. type : type of card card_id : `id` given to card + show_list_as_json : if set to `True` then supress logs about pathspec resolution. Raises: CardNotPresentException: No card could be found for the pathspec @@ -129,13 +136,9 @@ def resolve_card( origin_taskpathspec = resumed_info(task) if origin_taskpathspec: card_pathspec = origin_taskpathspec - ctx.obj.echo( - "Resolving card resumed from: %s" % origin_taskpathspec, - fg="green", - ) - else: - ctx.obj.echo(print_str, fg="green") - else: + print_str = ("Resolving card resumed from: %s" % origin_taskpathspec,) + + if not show_list_as_json: ctx.obj.echo(print_str, fg="green") # to resolve card_id we first check if the identifier is a pathspec and if it is then we check if the `id` is set or not to resolve card_id # todo : Fix this with `coalesce function` @@ -177,10 +180,20 @@ def raise_timeout(signum, frame): raise TimeoutError -def list_available_cards(ctx, path_spec, card_paths, card_datastore, command="view"): +def list_available_cards( + ctx, path_spec, card_paths, card_datastore, command="view", show_list_as_json=False +): # todo : create nice response messages on the CLI for cards which were found. scriptname = ctx.obj.flow.script_name path_tuples = card_datastore.get_card_names(card_paths) + if show_list_as_json: + json_arr = [ + dict(id=tup.id, hash=tup.hash, type=tup.type, filename=tup.filename) + for tup in path_tuples + ] + ctx.obj.echo(json.dumps(json_arr, indent=4)) + return + ctx.obj.echo( "\nFound %d card matching for your query..." % len(path_tuples), fg="green" ) @@ -275,6 +288,12 @@ def card_read_options_and_arguments(func): show_default=True, help="Follow the origin-task-id of resumed tasks to seek cards stored for resumed tasks.", ) + @click.option( + "--show-list-as-json", + default=False, + is_flag=True, + help="If multiple cards are found then print the list as a json array", + ) @wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -456,6 +475,7 @@ def view( type=None, id=None, follow_resumed=False, + show_list_as_json=False, ): """ View the HTML card in browser based on the pathspec.\n @@ -472,12 +492,18 @@ def view( hash=hash, card_id=card_id, follow_resumed=follow_resumed, + show_list_as_json=show_list_as_json, ) if len(available_card_paths) == 1: open_in_browser(card_datastore.cache_locally(available_card_paths[0])) else: list_available_cards( - ctx, pathspec, available_card_paths, card_datastore, command="view" + ctx, + pathspec, + available_card_paths, + card_datastore, + command="view", + show_list_as_json=show_list_as_json, ) @@ -492,6 +518,7 @@ def get( type=None, id=None, follow_resumed=False, + show_list_as_json=False, ): """ Get the HTML string of the card based on pathspec.\n @@ -508,10 +535,16 @@ def get( hash=hash, card_id=card_id, follow_resumed=follow_resumed, + show_list_as_json=show_list_as_json, ) if len(available_card_paths) == 1: print(card_datastore.get_card_html(available_card_paths[0])) else: list_available_cards( - ctx, pathspec, available_card_paths, card_datastore, command="get" + ctx, + pathspec, + available_card_paths, + card_datastore, + command="get", + show_list_as_json=show_list_as_json, ) From e829efe5278346dc076e11ff3443910a1d72314f Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Tue, 4 Jan 2022 21:01:34 -0800 Subject: [PATCH 02/12] changing `echo` to `print` --- metaflow/plugins/cards/card_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/plugins/cards/card_cli.py b/metaflow/plugins/cards/card_cli.py index 0aa9e9af437..1db2d2a7b6e 100644 --- a/metaflow/plugins/cards/card_cli.py +++ b/metaflow/plugins/cards/card_cli.py @@ -191,7 +191,7 @@ def list_available_cards( dict(id=tup.id, hash=tup.hash, type=tup.type, filename=tup.filename) for tup in path_tuples ] - ctx.obj.echo(json.dumps(json_arr, indent=4)) + print(json.dumps(json_arr, indent=4)) return ctx.obj.echo( From 249da0739f0f1827bc6d71d8dfe67c3c23d23fa8 Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Wed, 5 Jan 2022 09:53:38 -0800 Subject: [PATCH 03/12] Multiple Cards Test Suite Mods (#27) - Added `list_cards` to `CliCheck` and `MetadataCheck` - Bringing Netflix/metaflow#875 into the code - Added a card that prints taskspec with random number --- metaflow/plugins/__init__.py | 8 ++- metaflow/plugins/cards/card_client.py | 3 ++ .../plugins/cards/card_modules/test_cards.py | 9 ++++ test/core/metaflow_test/__init__.py | 3 ++ test/core/metaflow_test/cli_check.py | 49 ++++++++++++++++-- test/core/metaflow_test/metadata_check.py | 51 ++++++++++++++++--- 6 files changed, 111 insertions(+), 12 deletions(-) diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py index 1ef326a73d1..221b03d6282 100644 --- a/metaflow/plugins/__init__.py +++ b/metaflow/plugins/__init__.py @@ -187,7 +187,12 @@ def _merge_lists(base, overrides, attr): # Cards from .cards.card_modules.basic import DefaultCard, TaskSpecCard, ErrorCard, BlankCard -from .cards.card_modules.test_cards import TestErrorCard, TestTimeoutCard, TestMockCard +from .cards.card_modules.test_cards import ( + TestErrorCard, + TestTimeoutCard, + TestMockCard, + TestPathSpecCard, +) CARDS = [ DefaultCard, @@ -196,6 +201,7 @@ def _merge_lists(base, overrides, attr): TestErrorCard, TestTimeoutCard, TestMockCard, + TestPathSpecCard, BlankCard, ] # Sidecars diff --git a/metaflow/plugins/cards/card_client.py b/metaflow/plugins/cards/card_client.py index 76791f6b63d..3e5cb3e90be 100644 --- a/metaflow/plugins/cards/card_client.py +++ b/metaflow/plugins/cards/card_client.py @@ -37,6 +37,7 @@ def __init__( card_ds, type, path, + hash, id=None, html=None, created_on=None, @@ -51,6 +52,7 @@ def __init__( self._card_id = id # public attributes + self.hash = hash self.type = type self.from_resumed = from_resumed self.origin_pathspec = origin_pathspec @@ -131,6 +133,7 @@ def _get_card(self, index): self._card_ds, card_info.type, path, + card_info.hash, id=card_info.id, html=None, created_on=None, diff --git a/metaflow/plugins/cards/card_modules/test_cards.py b/metaflow/plugins/cards/card_modules/test_cards.py index 79fed549abe..9fe4d8b6b7d 100644 --- a/metaflow/plugins/cards/card_modules/test_cards.py +++ b/metaflow/plugins/cards/card_modules/test_cards.py @@ -1,6 +1,15 @@ from .card import MetaflowCard, MetaflowCardComponent +class TestPathSpecCard(MetaflowCard): + type = "test_pathspec_card" + + def render(self, task): + import random + + return "%s %d" % (task.pathspec, random.randint(0, 100)) + + class TestMockCard(MetaflowCard): type = "test_mock_card" diff --git a/test/core/metaflow_test/__init__.py b/test/core/metaflow_test/__init__.py index 1faa7caeca0..d919beae7d2 100644 --- a/test/core/metaflow_test/__init__.py +++ b/test/core/metaflow_test/__init__.py @@ -128,6 +128,9 @@ def assert_log(self, step, logtype, value, exact_match=True): def get_card(self, step, task, card_type): raise NotImplementedError() + def list_cards(self, step, task, card_type=None): + raise NotImplementedError() + def new_checker(flow): from . import cli_check, metadata_check diff --git a/test/core/metaflow_test/cli_check.py b/test/core/metaflow_test/cli_check.py index 13270d3b949..7e3b233e750 100644 --- a/test/core/metaflow_test/cli_check.py +++ b/test/core/metaflow_test/cli_check.py @@ -111,12 +111,23 @@ def assert_log(self, step, logtype, value, exact_match=True): ) return True - def assert_card(self, step, task, card_type, value, exact_match=True): + def assert_card( + self, + step, + task, + card_type, + value, + card_hash=None, + card_id=None, + exact_match=True, + ): from metaflow.plugins.cards.exception import CardNotPresentException no_card_found_message = CardNotPresentException.headline try: - card_data = self.get_card(step, task, card_type) + card_data = self.get_card( + step, task, card_type, card_hash=card_hash, card_id=card_id + ) except subprocess.CalledProcessError as e: if no_card_found_message in e.output.decode("utf-8").strip(): card_data = None @@ -131,7 +142,33 @@ def assert_card(self, step, task, card_type, value, exact_match=True): ) return True - def get_card(self, step, task, card_type): + def list_cards(self, step, task, card_type=None): + from metaflow.plugins.cards.exception import CardNotPresentException + + no_card_found_message = CardNotPresentException.headline + try: + card_data = self._list_cards(step, task=task, card_type=card_type) + card_data = json.loads(card_data) + except subprocess.CalledProcessError as e: + if no_card_found_message in e.output.decode("utf-8").strip(): + card_data = None + else: + raise e + return card_data + + def _list_cards(self, step, task=None, card_type=None): + pathspec = "%s/%s" % (self.run_id, step) + if task is not None: + pathspec = "%s/%s/%s" % (self.run_id, step, task) + cmd = ["--quiet", "card", "list", pathspec, "--as-json"] + if card_type is not None: + cmd.extend(["--type", card_type]) + + return self.run_cli(cmd, capture_output=True, pipe_error_to_output=True).decode( + "utf-8" + ) + + def get_card(self, step, task, card_type, card_hash=None, card_id=None): cmd = [ "--quiet", "card", @@ -140,6 +177,12 @@ def get_card(self, step, task, card_type): "--type", card_type, ] + + if card_hash is not None: + cmd.extend(["--hash", card_hash]) + if card_id is not None: + cmd.extend(["--id", card_id]) + return self.run_cli(cmd, capture_output=True, pipe_error_to_output=True).decode( "utf-8" ) diff --git a/test/core/metaflow_test/metadata_check.py b/test/core/metaflow_test/metadata_check.py index 2ebba314cf3..b2732e7b097 100644 --- a/test/core/metaflow_test/metadata_check.py +++ b/test/core/metaflow_test/metadata_check.py @@ -109,21 +109,56 @@ def assert_log(self, step, logtype, value, exact_match=True): % (step, logtype, repr(value), logtype, repr(log_value)) ) - def assert_card(self, step, task, card_type, value, exact_match=True): + def list_cards(self, step, task, card_type=None): from metaflow.plugins.cards.exception import CardNotPresentException try: card_iter = self.get_card(step, task, card_type) except CardNotPresentException: card_iter = None + + if card_iter is None: + return + pathspec = self.run[step][task].pathspec + list_data = dict(pathspec=pathspec, cards=[]) + if len(card_iter) > 0: + list_data["cards"] = [ + dict( + hash=card.hash, + id=card.id, + type=card.type, + filename=card.path.split("/")[-1], + ) + for card in card_iter + ] + return list_data + + def assert_card( + self, + step, + task, + card_type, + value, + card_hash=None, + card_id=None, + exact_match=True, + ): + from metaflow.plugins.cards.exception import CardNotPresentException + + try: + card_iter = self.get_card(step, task, card_type, card_id=card_id) + except CardNotPresentException: + card_iter = None card_data = None - # FUTURE FIXME: - # We are checking the first card here. - # Not all possible present cards - # This should change in the future when we support many decorator. + # Since there are many cards possible for a taskspec, we check for hash to assert a single card. + # If the id argument is present then there will be a single cards anyways. if card_iter is not None: if len(card_iter) > 0: - card_data = card_iter[0].get() + if card_hash is None: + card_data = card_iter[0].get() + else: + card_filter = [c for c in card_iter if card_hash in c.hash] + card_data = None if len(card_filter) == 0 else card_filter[0].get() if (exact_match and card_data != value) or ( not exact_match and value not in card_data ): @@ -136,8 +171,8 @@ def assert_card(self, step, task, card_type, value, exact_match=True): def get_log(self, step, logtype): return "".join(getattr(task, logtype) for task in self.run[step]) - def get_card(self, step, task, card_type): + def get_card(self, step, task, card_type, card_id=None): from metaflow.cards import get_cards - iterator = get_cards(self.run[step][task], type=card_type) + iterator = get_cards(self.run[step][task], type=card_type, id=card_id) return iterator From ea33004617b8f892dfa5d2764346983b3264f362 Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Wed, 5 Jan 2022 10:09:19 -0800 Subject: [PATCH 04/12] Added test case for multiple cards --- test/core/tests/card_multiple.py | 97 ++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 test/core/tests/card_multiple.py diff --git a/test/core/tests/card_multiple.py b/test/core/tests/card_multiple.py new file mode 100644 index 00000000000..125db2dc40b --- /dev/null +++ b/test/core/tests/card_multiple.py @@ -0,0 +1,97 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + + +class MultipleCardDecoratorTest(MetaflowTest): + """ + Test that checks if the multiple card decorators work with @step code. + - This test adds multiple `test_pathspec_card` cards to a @step + - Each card will contain taskpathspec + - CLI Check: + - List cards and cli will assert multiple cards are present per taskspec using `CliCheck.list_cards` + - Assert the information about the card using the hash and check if taskspec is present in the data + - Metadata Check + - List cards and cli will assert multiple cards are present per taskspec using `MetadataCheck.list_cards` + - Assert the information about the card using the hash and check if taskspec is present in the data + """ + + PRIORITY = 3 + + @tag('card(type="test_pathspec_card")') + @tag('card(type="test_pathspec_card")') + @steps(0, ["start"]) + def step_start(self): + from metaflow import current + + self.task = current.pathspec + + @tag('card(type="test_pathspec_card")') + @tag('card(type="test_pathspec_card")') + @steps(0, ["foreach-nested-inner"]) + def step_foreach_inner(self): + from metaflow import current + + self.task = current.pathspec + + @tag('card(type="test_pathspec_card")') + @tag('card(type="test_pathspec_card")') + @steps(1, ["join"]) + def step_join(self): + from metaflow import current + + self.task = current.pathspec + + @tag('card(type="test_pathspec_card")') + @tag('card(type="test_pathspec_card")') + @steps(1, ["all"]) + def step_all(self): + from metaflow import current + + self.task = current.pathspec + + def check_results(self, flow, checker): + run = checker.get_run() + if run is None: + # This means CliCheck is in context. + for step in flow: + cli_check_dict = checker.artifact_dict(step.name, "task") + for task_pathspec in cli_check_dict: + full_pathspec = "/".join([flow.name, task_pathspec]) + task_id = task_pathspec.split("/")[-1] + cards_info = checker.list_cards(step.name, task_id) + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 2, + True, + ) + for card in cards_info["cards"]: + checker.assert_card( + step.name, + task_id, + "test_pathspec_card", + "%s" % full_pathspec, + card_hash=card["hash"], + exact_match=False, + ) + else: + # This means MetadataCheck is in context. + for step in flow: + meta_check_dict = checker.artifact_dict(step.name, "task") + for task_id in meta_check_dict: + full_pathspec = meta_check_dict[task_id]["task"] + cards_info = checker.list_cards(step.name, task_id) + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 2, + True, + ) + for card in cards_info["cards"]: + checker.assert_card( + step.name, + task_id, + "test_pathspec_card", + "%s" % full_pathspec, + card_hash=card["hash"], + exact_match=False, + ) From a6f467daae8691b99d3d720b3cb4ecf45758489b Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Wed, 5 Jan 2022 11:39:05 -0800 Subject: [PATCH 05/12] Added tests card, summary : - `current.cards['myid']` should be accessible when cards have an `id` argument in decorator - `current.cards.append` should not work when there are no single default editable card. - if a card has `ALLOW_USER_COMPONENTS=False` then it can still be edited via accessing it with `id` property. - adding arbitrary information to `current.cards.append` should not break user code. - Only cards with `ALLOW_USER_COMPONENTS=True` are considered default editable. - If a single @card decorator is present with `id` then it `current.cards.append` should still work - Access of `current.cards` with non existant id should not fail. - `current.cards.append` should be accessible to the card with `customize=True`. - --- metaflow/plugins/__init__.py | 8 +- .../plugins/cards/card_modules/test_cards.py | 48 +++++++ test/core/tests/card_default_editable.py | 117 ++++++++++++++++++ .../tests/card_default_editable_customize.py | 95 ++++++++++++++ .../tests/card_default_editable_with_id.py | 103 +++++++++++++++ test/core/tests/card_id_append.py | 82 ++++++++++++ 6 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 test/core/tests/card_default_editable.py create mode 100644 test/core/tests/card_default_editable_customize.py create mode 100644 test/core/tests/card_default_editable_with_id.py create mode 100644 test/core/tests/card_id_append.py diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py index 221b03d6282..bc52b2d5e32 100644 --- a/metaflow/plugins/__init__.py +++ b/metaflow/plugins/__init__.py @@ -192,17 +192,23 @@ def _merge_lists(base, overrides, attr): TestTimeoutCard, TestMockCard, TestPathSpecCard, + TestEditableCard, + TestEditableCard2, + TestNonEditableCard, ) CARDS = [ DefaultCard, TaskSpecCard, ErrorCard, + BlankCard, TestErrorCard, TestTimeoutCard, TestMockCard, TestPathSpecCard, - BlankCard, + TestEditableCard, + TestEditableCard2, + TestNonEditableCard, ] # Sidecars from ..mflog.save_logs_periodically import SaveLogsPeriodicallySidecar diff --git a/metaflow/plugins/cards/card_modules/test_cards.py b/metaflow/plugins/cards/card_modules/test_cards.py index 9fe4d8b6b7d..c931927310e 100644 --- a/metaflow/plugins/cards/card_modules/test_cards.py +++ b/metaflow/plugins/cards/card_modules/test_cards.py @@ -1,6 +1,14 @@ from .card import MetaflowCard, MetaflowCardComponent +class TestStringComponent(MetaflowCardComponent): + def __init__(self, text): + self._text = text + + def render(self): + return str(self._text) + + class TestPathSpecCard(MetaflowCard): type = "test_pathspec_card" @@ -10,6 +18,46 @@ def render(self, task): return "%s %d" % (task.pathspec, random.randint(0, 100)) +class TestEditableCard(MetaflowCard): + type = "test_editable_card" + + seperator = "$&#!!@*" + + ALLOW_USER_COMPONENTS = True + + def __init__(self, options={}, components=[], graph=None): + self._components = components + + def render(self, task): + return self.seperator.join([str(comp) for comp in self._components]) + + +class TestEditableCard2(MetaflowCard): + type = "test_editable_card_2" + + seperator = "$&#!!@*" + + ALLOW_USER_COMPONENTS = True + + def __init__(self, options={}, components=[], graph=None): + self._components = components + + def render(self, task): + return self.seperator.join([str(comp) for comp in self._components]) + + +class TestNonEditableCard(MetaflowCard): + type = "test_non_editable_card" + + seperator = "$&#!!@*" + + def __init__(self, options={}, components=[], graph=None): + self._components = components + + def render(self, task): + return self.seperator.join([str(comp) for comp in self._components]) + + class TestMockCard(MetaflowCard): type = "test_mock_card" diff --git a/test/core/tests/card_default_editable.py b/test/core/tests/card_default_editable.py new file mode 100644 index 00000000000..acc743d59e2 --- /dev/null +++ b/test/core/tests/card_default_editable.py @@ -0,0 +1,117 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + + +class DefaultEditableCardTest(MetaflowTest): + """ + `current.cards.append` works for one decorator as default editable cards + - adding arbitrary information to `current.cards.append` should not break user code. + - If a single @card decorator is present with `id` then it `current.cards.append` should still work + - Only cards with `ALLOW_USER_COMPONENTS=True` are considered default editable. + """ + + HEADER = """ +class MyNativeType: + at = 0 + def get(self): + return self.at + """ + + PRIORITY = 3 + + @tag('card(type="test_editable_card")') + @steps(0, ["start"]) + def step_start(self): + from metaflow import current + from metaflow.plugins.cards.card_modules.test_cards import TestStringComponent + import random + + self.random_number = random.randint(0, 100) + current.cards.append(current.pathspec) + current.cards.append(TestStringComponent(str(self.random_number))) + empty_list = current.cards.get(type="nonexistingtype") + current.cards.append(MyNativeType()) + + @tag('card(type="test_editable_card", id="xyz")') + @steps(0, ["foreach-nested-inner"]) + def step_foreach_inner(self): + # In this step `test_editable_card` should considered default editable even with `id` + from metaflow import current + from metaflow.plugins.cards.card_modules.test_cards import TestStringComponent + import random + + self.random_number = random.randint(0, 100) + current.cards.append(current.pathspec) + current.cards.append(TestStringComponent(str(self.random_number))) + + @tag('card(type="taskspec_card")') + @tag('card(type="test_editable_card")') + @steps(1, ["join"]) + def step_join(self): + # In this step `taskspec_card` should not be considered default editable + from metaflow import current + from metaflow.plugins.cards.card_modules.test_cards import TestStringComponent + import random + + self.random_number = random.randint(0, 100) + current.cards.append(current.pathspec) + current.cards.append(TestStringComponent(str(self.random_number))) + + @tag('card(type="test_editable_card")') + @steps(1, ["all"]) + def step_all(self): + from metaflow import current + from metaflow.plugins.cards.card_modules.test_cards import TestStringComponent + import random + + self.random_number = random.randint(0, 100) + current.cards.append(current.pathspec) + current.cards.append(TestStringComponent(str(self.random_number))) + + def check_results(self, flow, checker): + run = checker.get_run() + if run is None: + # This means CliCheck is in context. + for step in flow: + cli_check_dict = checker.artifact_dict(step.name, "random_number") + for task_pathspec in cli_check_dict: + # full_pathspec = "/".join([flow.name, task_pathspec]) + task_id = task_pathspec.split("/")[-1] + cards_info = checker.list_cards(step.name, task_id) + number = cli_check_dict[task_pathspec]["random_number"] + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 1, + True, + ) + for card in cards_info["cards"]: + checker.assert_card( + step.name, + task_id, + "test_editable_card", + "%d\n" % number, + card_hash=card["hash"], + exact_match=True, + ) + else: + # This means MetadataCheck is in context. + for step in flow: + meta_check_dict = checker.artifact_dict(step.name, "random_number") + for task_id in meta_check_dict: + random_number = meta_check_dict[task_id]["random_number"] + cards_info = checker.list_cards(step.name, task_id) + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 1, + True, + ) + for card in cards_info["cards"]: + checker.assert_card( + step.name, + task_id, + "test_editable_card", + "%d" % random_number, + card_hash=card["hash"], + exact_match=False, + ) diff --git a/test/core/tests/card_default_editable_customize.py b/test/core/tests/card_default_editable_customize.py new file mode 100644 index 00000000000..4d3581e4872 --- /dev/null +++ b/test/core/tests/card_default_editable_customize.py @@ -0,0 +1,95 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + + +class DefaultEditableCardWithCustomizeTest(MetaflowTest): + """ + `current.cards.append` should be accessible to the card with `customize=True`. + - Even if there are other editable cards without `id` and with `id` + """ + + PRIORITY = 3 + + @tag('card(type="test_editable_card",customize=True)') + @tag('card(type="test_editable_card",id="abc")') + @tag('card(type="test_editable_card_2")') + @tag('card(type="test_editable_card_2")') + @steps(0, ["start"]) + def step_start(self): + from metaflow import current + from metaflow.plugins.cards.card_modules.test_cards import TestStringComponent + import random + + self.random_number = random.randint(0, 100) + current.cards.append(TestStringComponent(str(self.random_number))) + + @steps(1, ["all"]) + def step_all(self): + pass + + def check_results(self, flow, checker): + run = checker.get_run() + if run is None: + # This means CliCheck is in context. + for step in flow: + if step.name != "start": + continue + + cli_check_dict = checker.artifact_dict(step.name, "random_number") + for task_pathspec in cli_check_dict: + task_id = task_pathspec.split("/")[-1] + cards_info = checker.list_cards(step.name, task_id) + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 3, + True, + ) + # Find the card without the id + default_editable_cards = [ + c + for c in cards_info["cards"] + if c["id"] is None and c["type"] == "test_editable_card" + ] + # There should only be one card of and "test_editable_card" with no id. That is the default editable card + assert_equals(len(default_editable_cards) == 1, True) + card = default_editable_cards[0] + number = cli_check_dict[task_pathspec]["random_number"] + checker.assert_card( + step.name, + task_id, + "test_editable_card", + "%d\n" % number, + card_hash=card["hash"], + exact_match=True, + ) + else: + # This means MetadataCheck is in context. + for step in flow: + if step.name != "start": + continue + meta_check_dict = checker.artifact_dict(step.name, "random_number") + for task_id in meta_check_dict: + cards_info = checker.list_cards(step.name, task_id) + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 3, + True, + ) + default_editable_cards = [ + c + for c in cards_info["cards"] + if c["id"] is None and c["type"] == "test_editable_card" + ] + # There should only be one card of and "test_editable_card" with no id. That is the default editable card + assert_equals(len(default_editable_cards) == 1, True) + card = default_editable_cards[0] + random_number = meta_check_dict[task_id]["random_number"] + checker.assert_card( + step.name, + task_id, + "test_editable_card", + "%d" % random_number, + card_hash=card["hash"], + exact_match=True, + ) diff --git a/test/core/tests/card_default_editable_with_id.py b/test/core/tests/card_default_editable_with_id.py new file mode 100644 index 00000000000..129c8a8c00e --- /dev/null +++ b/test/core/tests/card_default_editable_with_id.py @@ -0,0 +1,103 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + + +class DefaultEditableCardWithIdTest(MetaflowTest): + """ + `current.cards.append` should add to default editable card and not the one with `id` when a card with `id` and non id are present + - Access of `current.cards` with non existant id should not fail. + """ + + PRIORITY = 3 + + @tag('card(type="test_editable_card",id="abc")') + @tag('card(type="test_editable_card")') + @steps(0, ["start"]) + def step_start(self): + from metaflow import current + from metaflow.plugins.cards.card_modules.test_cards import TestStringComponent + import random + + self.random_number = random.randint(0, 100) + current.cards.append(current.pathspec) + # This should not fail user code. + current.cards["xyz"].append(TestStringComponent(str(self.random_number))) + current.cards.append(TestStringComponent(str(self.random_number))) + + @steps(0, ["end"], required=True) + def step_end(self): + self.here = True + + @steps(1, ["all"]) + def step_all(self): + pass + + def check_results(self, flow, checker): + run = checker.get_run() + if run is None: + # This means CliCheck is in context. + for step in flow: + if step.name == "end": + # Ensure we reach the `end` even even when a wrong `id` is used with `current.cards` + checker.assert_artifact(step.name, "here", True) + continue + elif step.name != "start": + continue + + cli_check_dict = checker.artifact_dict(step.name, "random_number") + for task_pathspec in cli_check_dict: + # full_pathspec = "/".join([flow.name, task_pathspec]) + task_id = task_pathspec.split("/")[-1] + cards_info = checker.list_cards(step.name, task_id) + number = cli_check_dict[task_pathspec]["random_number"] + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 2, + True, + ) + # Find the card without the id + default_editable_cards = [ + c for c in cards_info["cards"] if c["id"] is None + ] + assert_equals(len(default_editable_cards) == 1, True) + card = default_editable_cards[0] + checker.assert_card( + step.name, + task_id, + "test_editable_card", + "%d\n" % number, + card_hash=card["hash"], + exact_match=True, + ) + else: + # This means MetadataCheck is in context. + for step in flow: + if step.name == "end": + # Ensure we reach the `end` even even when a wrong `id` is used with `current.cards` + checker.assert_artifact(step.name, "here", True) + continue + elif step.name != "start": + continue + meta_check_dict = checker.artifact_dict(step.name, "random_number") + for task_id in meta_check_dict: + random_number = meta_check_dict[task_id]["random_number"] + cards_info = checker.list_cards(step.name, task_id) + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 2, + True, + ) + default_editable_cards = [ + c for c in cards_info["cards"] if c["id"] is None + ] + assert_equals(len(default_editable_cards) == 1, True) + card = default_editable_cards[0] + checker.assert_card( + step.name, + task_id, + "test_editable_card", + "%d" % random_number, + card_hash=card["hash"], + exact_match=False, + ) diff --git a/test/core/tests/card_id_append.py b/test/core/tests/card_id_append.py new file mode 100644 index 00000000000..5f58f013807 --- /dev/null +++ b/test/core/tests/card_id_append.py @@ -0,0 +1,82 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + + +class CardsWithIdTest(MetaflowTest): + """ + `current.cards['myid']` should be accessible when cards have an `id` argument in decorator + - `current.cards.append` should not work when there are no single default editable card. + - if a card has `ALLOW_USER_COMPONENTS=False` then it can still be edited via accessing it with `id` property. + """ + + PRIORITY = 3 + + @tag('card(type="test_editable_card",id="xyz")') + @tag('card(type="test_editable_card",id="abc")') + @steps(0, ["start"]) + def step_start(self): + from metaflow import current + from metaflow.plugins.cards.card_modules.test_cards import TestStringComponent + import random + + self.random_number = random.randint(0, 100) + self.random_number_2 = random.randint(0, 100) + current.cards["abc"].append(TestStringComponent(str(self.random_number))) + # Below line should not work + current.cards.append(TestStringComponent(str(self.random_number_2))) + + @tag('card(type="test_non_editable_card",id="abc")') + @steps(0, ["end"]) + def step_end(self): + # If the card is default non editable, we can still access it via `current.cards[id]` + from metaflow import current + from metaflow.plugins.cards.card_modules.test_cards import TestStringComponent + import random + + self.random_number = random.randint(0, 100) + self.random_number_2 = random.randint(0, 100) + current.cards["abc"].append(TestStringComponent(str(self.random_number))) + + @steps(1, ["all"]) + def step_all(self): + pass + + def check_results(self, flow, checker): + run = checker.get_run() + if run is None: + # This means CliCheck is in context. + for step in flow: + if step.name != "start" or step.name != "end": + continue + + cli_check_dict = checker.artifact_dict(step.name, "random_number") + for task_pathspec in cli_check_dict: + task_id = task_pathspec.split("/")[-1] + number = cli_check_dict[task_pathspec]["random_number"] + checker.assert_card( + step.name, + task_id, + "test_editable_card" + if step.name == "start" + else "test_non_editable_card", + "%d\n" % number, + card_id="abc", + exact_match=True, + ) + else: + # This means MetadataCheck is in context. + for step in flow: + if step.name != "start" or step.name != "end": + continue + meta_check_dict = checker.artifact_dict(step.name, "random_number") + for task_id in meta_check_dict: + random_number = meta_check_dict[task_id]["random_number"] + checker.assert_card( + step.name, + task_id, + "test_editable_card" + if step.name == "start" + else "test_non_editable_card", + "%d" % random_number, + card_id="abc", + exact_match=True, + ) From 713a93e77f85ee16d3d18f62412359d2bf6cdda0 Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Wed, 5 Jan 2022 19:56:10 -0800 Subject: [PATCH 06/12] fixed `DefaultEditableCardTest` test case - Fixed comment - Fixed the `customize=True` test case. --- test/core/tests/card_default_editable.py | 29 ++++++++++--------- .../tests/card_default_editable_customize.py | 29 +++++++++---------- .../tests/card_default_editable_with_id.py | 2 +- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/test/core/tests/card_default_editable.py b/test/core/tests/card_default_editable.py index acc743d59e2..4b47a470058 100644 --- a/test/core/tests/card_default_editable.py +++ b/test/core/tests/card_default_editable.py @@ -45,7 +45,7 @@ def step_foreach_inner(self): @tag('card(type="taskspec_card")') @tag('card(type="test_editable_card")') - @steps(1, ["join"]) + @steps(0, ["join"]) def step_join(self): # In this step `taskspec_card` should not be considered default editable from metaflow import current @@ -69,6 +69,7 @@ def step_all(self): def check_results(self, flow, checker): run = checker.get_run() + card_type = "test_editable_card" if run is None: # This means CliCheck is in context. for step in flow: @@ -76,7 +77,8 @@ def check_results(self, flow, checker): for task_pathspec in cli_check_dict: # full_pathspec = "/".join([flow.name, task_pathspec]) task_id = task_pathspec.split("/")[-1] - cards_info = checker.list_cards(step.name, task_id) + cards_info = checker.list_cards(step.name, task_id, card_type) + number = cli_check_dict[task_pathspec]["random_number"] assert_equals( cards_info is not None @@ -84,22 +86,23 @@ def check_results(self, flow, checker): and len(cards_info["cards"]) == 1, True, ) - for card in cards_info["cards"]: - checker.assert_card( - step.name, - task_id, - "test_editable_card", - "%d\n" % number, - card_hash=card["hash"], - exact_match=True, - ) + card = cards_info["cards"][0] + checker.assert_card( + step.name, + task_id, + card_type, + "%d\n" % number, + card_hash=card["hash"], + exact_match=True, + ) else: # This means MetadataCheck is in context. for step in flow: meta_check_dict = checker.artifact_dict(step.name, "random_number") for task_id in meta_check_dict: random_number = meta_check_dict[task_id]["random_number"] - cards_info = checker.list_cards(step.name, task_id) + cards_info = checker.list_cards(step.name, task_id, card_type) + assert_equals( cards_info is not None and "cards" in cards_info @@ -110,7 +113,7 @@ def check_results(self, flow, checker): checker.assert_card( step.name, task_id, - "test_editable_card", + card_type, "%d" % random_number, card_hash=card["hash"], exact_match=False, diff --git a/test/core/tests/card_default_editable_customize.py b/test/core/tests/card_default_editable_customize.py index 4d3581e4872..a87d90f21e6 100644 --- a/test/core/tests/card_default_editable_customize.py +++ b/test/core/tests/card_default_editable_customize.py @@ -11,7 +11,7 @@ class DefaultEditableCardWithCustomizeTest(MetaflowTest): @tag('card(type="test_editable_card",customize=True)') @tag('card(type="test_editable_card",id="abc")') - @tag('card(type="test_editable_card_2")') + @tag('card(type="taskspec_card")') @tag('card(type="test_editable_card_2")') @steps(0, ["start"]) def step_start(self): @@ -28,6 +28,7 @@ def step_all(self): def check_results(self, flow, checker): run = checker.get_run() + card_type = "test_editable_card" if run is None: # This means CliCheck is in context. for step in flow: @@ -37,27 +38,26 @@ def check_results(self, flow, checker): cli_check_dict = checker.artifact_dict(step.name, "random_number") for task_pathspec in cli_check_dict: task_id = task_pathspec.split("/")[-1] - cards_info = checker.list_cards(step.name, task_id) + cards_info = checker.list_cards(step.name, task_id, card_type) assert_equals( cards_info is not None and "cards" in cards_info - and len(cards_info["cards"]) == 3, + and len(cards_info["cards"]) == 2, True, ) # Find the card without the id default_editable_cards = [ - c - for c in cards_info["cards"] - if c["id"] is None and c["type"] == "test_editable_card" + c for c in cards_info["cards"] if c["id"] is None ] - # There should only be one card of and "test_editable_card" with no id. That is the default editable card + # There should only be one card of type "test_editable_card" with no id. + # That is the default editable card because it has `customize=True` assert_equals(len(default_editable_cards) == 1, True) card = default_editable_cards[0] number = cli_check_dict[task_pathspec]["random_number"] checker.assert_card( step.name, task_id, - "test_editable_card", + card_type, "%d\n" % number, card_hash=card["hash"], exact_match=True, @@ -69,26 +69,25 @@ def check_results(self, flow, checker): continue meta_check_dict = checker.artifact_dict(step.name, "random_number") for task_id in meta_check_dict: - cards_info = checker.list_cards(step.name, task_id) + cards_info = checker.list_cards(step.name, task_id, card_type) assert_equals( cards_info is not None and "cards" in cards_info - and len(cards_info["cards"]) == 3, + and len(cards_info["cards"]) == 2, True, ) default_editable_cards = [ - c - for c in cards_info["cards"] - if c["id"] is None and c["type"] == "test_editable_card" + c for c in cards_info["cards"] if c["id"] is None ] - # There should only be one card of and "test_editable_card" with no id. That is the default editable card + # There should only be one card of type "test_editable_card" with no id. + # That is the default editable card since it has `customize=True` assert_equals(len(default_editable_cards) == 1, True) card = default_editable_cards[0] random_number = meta_check_dict[task_id]["random_number"] checker.assert_card( step.name, task_id, - "test_editable_card", + card_type, "%d" % random_number, card_hash=card["hash"], exact_match=True, diff --git a/test/core/tests/card_default_editable_with_id.py b/test/core/tests/card_default_editable_with_id.py index 129c8a8c00e..1d1da3085c2 100644 --- a/test/core/tests/card_default_editable_with_id.py +++ b/test/core/tests/card_default_editable_with_id.py @@ -45,7 +45,7 @@ def check_results(self, flow, checker): cli_check_dict = checker.artifact_dict(step.name, "random_number") for task_pathspec in cli_check_dict: - # full_pathspec = "/".join([flow.name, task_pathspec]) + task_id = task_pathspec.split("/")[-1] cards_info = checker.list_cards(step.name, task_id) number = cli_check_dict[task_pathspec]["random_number"] From 1d50de893310ea80b83f8106dff0e764f16d4df4 Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Thu, 6 Jan 2022 17:23:13 -0800 Subject: [PATCH 07/12] ensure `test_pathspec_card` has no duplicates - ensure entropy of rendered information is high enough to not overwrite a file. --- metaflow/plugins/cards/card_modules/test_cards.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/metaflow/plugins/cards/card_modules/test_cards.py b/metaflow/plugins/cards/card_modules/test_cards.py index c931927310e..b756fb37dbd 100644 --- a/metaflow/plugins/cards/card_modules/test_cards.py +++ b/metaflow/plugins/cards/card_modules/test_cards.py @@ -14,8 +14,14 @@ class TestPathSpecCard(MetaflowCard): def render(self, task): import random - - return "%s %d" % (task.pathspec, random.randint(0, 100)) + import string + + return "%s %s" % ( + task.pathspec, + "".join( + random.choice(string.ascii_uppercase + string.digits) for _ in range(6) + ), + ) class TestEditableCard(MetaflowCard): From 8907d406eda75a2c899b676fa9932686ce37c458 Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Mon, 10 Jan 2022 13:12:22 -0800 Subject: [PATCH 08/12] test case fix : `current.cards` to `current.card` --- test/core/tests/card_default_editable.py | 26 +++++++++---------- .../tests/card_default_editable_customize.py | 4 +-- .../tests/card_default_editable_with_id.py | 14 +++++----- test/core/tests/card_id_append.py | 12 ++++----- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/test/core/tests/card_default_editable.py b/test/core/tests/card_default_editable.py index 4b47a470058..1da3dfdfc5c 100644 --- a/test/core/tests/card_default_editable.py +++ b/test/core/tests/card_default_editable.py @@ -3,9 +3,9 @@ class DefaultEditableCardTest(MetaflowTest): """ - `current.cards.append` works for one decorator as default editable cards - - adding arbitrary information to `current.cards.append` should not break user code. - - If a single @card decorator is present with `id` then it `current.cards.append` should still work + `current.card.append` works for one decorator as default editable cards + - adding arbitrary information to `current.card.append` should not break user code. + - If a single @card decorator is present with `id` then it `current.card.append` should still work - Only cards with `ALLOW_USER_COMPONENTS=True` are considered default editable. """ @@ -26,10 +26,10 @@ def step_start(self): import random self.random_number = random.randint(0, 100) - current.cards.append(current.pathspec) - current.cards.append(TestStringComponent(str(self.random_number))) - empty_list = current.cards.get(type="nonexistingtype") - current.cards.append(MyNativeType()) + current.card.append(current.pathspec) + current.card.append(TestStringComponent(str(self.random_number))) + empty_list = current.card.get(type="nonexistingtype") + current.card.append(MyNativeType()) @tag('card(type="test_editable_card", id="xyz")') @steps(0, ["foreach-nested-inner"]) @@ -40,8 +40,8 @@ def step_foreach_inner(self): import random self.random_number = random.randint(0, 100) - current.cards.append(current.pathspec) - current.cards.append(TestStringComponent(str(self.random_number))) + current.card.append(current.pathspec) + current.card.append(TestStringComponent(str(self.random_number))) @tag('card(type="taskspec_card")') @tag('card(type="test_editable_card")') @@ -53,8 +53,8 @@ def step_join(self): import random self.random_number = random.randint(0, 100) - current.cards.append(current.pathspec) - current.cards.append(TestStringComponent(str(self.random_number))) + current.card.append(current.pathspec) + current.card.append(TestStringComponent(str(self.random_number))) @tag('card(type="test_editable_card")') @steps(1, ["all"]) @@ -64,8 +64,8 @@ def step_all(self): import random self.random_number = random.randint(0, 100) - current.cards.append(current.pathspec) - current.cards.append(TestStringComponent(str(self.random_number))) + current.card.append(current.pathspec) + current.card.append(TestStringComponent(str(self.random_number))) def check_results(self, flow, checker): run = checker.get_run() diff --git a/test/core/tests/card_default_editable_customize.py b/test/core/tests/card_default_editable_customize.py index a87d90f21e6..3abf6aa906c 100644 --- a/test/core/tests/card_default_editable_customize.py +++ b/test/core/tests/card_default_editable_customize.py @@ -3,7 +3,7 @@ class DefaultEditableCardWithCustomizeTest(MetaflowTest): """ - `current.cards.append` should be accessible to the card with `customize=True`. + `current.card.append` should be accessible to the card with `customize=True`. - Even if there are other editable cards without `id` and with `id` """ @@ -20,7 +20,7 @@ def step_start(self): import random self.random_number = random.randint(0, 100) - current.cards.append(TestStringComponent(str(self.random_number))) + current.card.append(TestStringComponent(str(self.random_number))) @steps(1, ["all"]) def step_all(self): diff --git a/test/core/tests/card_default_editable_with_id.py b/test/core/tests/card_default_editable_with_id.py index 1d1da3085c2..2b54a651e88 100644 --- a/test/core/tests/card_default_editable_with_id.py +++ b/test/core/tests/card_default_editable_with_id.py @@ -3,8 +3,8 @@ class DefaultEditableCardWithIdTest(MetaflowTest): """ - `current.cards.append` should add to default editable card and not the one with `id` when a card with `id` and non id are present - - Access of `current.cards` with non existant id should not fail. + `current.card.append` should add to default editable card and not the one with `id` when a card with `id` and non id are present + - Access of `current.card` with non existant id should not fail. """ PRIORITY = 3 @@ -18,10 +18,10 @@ def step_start(self): import random self.random_number = random.randint(0, 100) - current.cards.append(current.pathspec) + current.card.append(current.pathspec) # This should not fail user code. - current.cards["xyz"].append(TestStringComponent(str(self.random_number))) - current.cards.append(TestStringComponent(str(self.random_number))) + current.card["xyz"].append(TestStringComponent(str(self.random_number))) + current.card.append(TestStringComponent(str(self.random_number))) @steps(0, ["end"], required=True) def step_end(self): @@ -37,7 +37,7 @@ def check_results(self, flow, checker): # This means CliCheck is in context. for step in flow: if step.name == "end": - # Ensure we reach the `end` even even when a wrong `id` is used with `current.cards` + # Ensure we reach the `end` even even when a wrong `id` is used with `current.card` checker.assert_artifact(step.name, "here", True) continue elif step.name != "start": @@ -73,7 +73,7 @@ def check_results(self, flow, checker): # This means MetadataCheck is in context. for step in flow: if step.name == "end": - # Ensure we reach the `end` even even when a wrong `id` is used with `current.cards` + # Ensure we reach the `end` even even when a wrong `id` is used with `current.card` checker.assert_artifact(step.name, "here", True) continue elif step.name != "start": diff --git a/test/core/tests/card_id_append.py b/test/core/tests/card_id_append.py index 5f58f013807..2765c2f8e48 100644 --- a/test/core/tests/card_id_append.py +++ b/test/core/tests/card_id_append.py @@ -3,8 +3,8 @@ class CardsWithIdTest(MetaflowTest): """ - `current.cards['myid']` should be accessible when cards have an `id` argument in decorator - - `current.cards.append` should not work when there are no single default editable card. + `current.card['myid']` should be accessible when cards have an `id` argument in decorator + - `current.card.append` should not work when there are no single default editable card. - if a card has `ALLOW_USER_COMPONENTS=False` then it can still be edited via accessing it with `id` property. """ @@ -20,21 +20,21 @@ def step_start(self): self.random_number = random.randint(0, 100) self.random_number_2 = random.randint(0, 100) - current.cards["abc"].append(TestStringComponent(str(self.random_number))) + current.card["abc"].append(TestStringComponent(str(self.random_number))) # Below line should not work - current.cards.append(TestStringComponent(str(self.random_number_2))) + current.card.append(TestStringComponent(str(self.random_number_2))) @tag('card(type="test_non_editable_card",id="abc")') @steps(0, ["end"]) def step_end(self): - # If the card is default non editable, we can still access it via `current.cards[id]` + # If the card is default non editable, we can still access it via `current.card[id]` from metaflow import current from metaflow.plugins.cards.card_modules.test_cards import TestStringComponent import random self.random_number = random.randint(0, 100) self.random_number_2 = random.randint(0, 100) - current.cards["abc"].append(TestStringComponent(str(self.random_number))) + current.card["abc"].append(TestStringComponent(str(self.random_number))) @steps(1, ["all"]) def step_all(self): From aec6847448e6dcac50c8fe53484b343ba75ad5e5 Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Fri, 21 Jan 2022 13:26:38 -0800 Subject: [PATCH 09/12] Added Test case to support import of cards. - Test case validates that broken card modules don't break metaflow - test case validates that we are able to import cards from metaflow_extensions - Test case validate that cards can be editable if they are importable. --- .../plugins/cards/brokencard/__init__.py | 13 ++ .../plugins/cards/simplecard/__init__.py | 23 ++++ test/core/tests/card_import.py | 127 ++++++++++++++++++ 3 files changed, 163 insertions(+) create mode 100644 test/core/metaflow_extensions/test_org/plugins/cards/brokencard/__init__.py create mode 100644 test/core/metaflow_extensions/test_org/plugins/cards/simplecard/__init__.py create mode 100644 test/core/tests/card_import.py diff --git a/test/core/metaflow_extensions/test_org/plugins/cards/brokencard/__init__.py b/test/core/metaflow_extensions/test_org/plugins/cards/brokencard/__init__.py new file mode 100644 index 00000000000..784eefdbf2e --- /dev/null +++ b/test/core/metaflow_extensions/test_org/plugins/cards/brokencard/__init__.py @@ -0,0 +1,13 @@ +from metaflow.cards import MetaflowCard + + +class BrokenCard(MetaflowCard): + type = "test_broken_card" + + def render(self, task): + return task.pathspec + + +CARDS = [BrokenCard] + +raise Exception("This module should not be importable") diff --git a/test/core/metaflow_extensions/test_org/plugins/cards/simplecard/__init__.py b/test/core/metaflow_extensions/test_org/plugins/cards/simplecard/__init__.py new file mode 100644 index 00000000000..d94b0642a91 --- /dev/null +++ b/test/core/metaflow_extensions/test_org/plugins/cards/simplecard/__init__.py @@ -0,0 +1,23 @@ +from metaflow.cards import MetaflowCard +from metaflow.plugins.cards.card_modules.test_cards import TestEditableCard + + +class TestNonEditableImportCard(MetaflowCard): + type = "non_editable_import_test_card" + + ALLOW_USER_COMPONENTS = False + + def __init__(self, options={}, components=[], graph=None): + self._options, self._components, self._graph = options, components, graph + + def render(self, task): + return task.pathspec + + +class TestEditableImportCard(TestEditableCard): + type = "editable_import_test_card" + + ALLOW_USER_COMPONENTS = True + + +CARDS = [TestEditableImportCard, TestNonEditableImportCard] diff --git a/test/core/tests/card_import.py b/test/core/tests/card_import.py new file mode 100644 index 00000000000..c462f2f3fdb --- /dev/null +++ b/test/core/tests/card_import.py @@ -0,0 +1,127 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + + +class CardImportTest(MetaflowTest): + """ + This test tries to check if the import scheme for cards works as intended. + - Importing a card and calling it via the `type` should work + - Importable cards could be editable. + - If the submodule has errors while importing then the rest of metaflow should not fail. + """ + + PRIORITY = 4 + + @tag('card(type="editable_import_test_card",save_errors=False)') + @tag('card(type="test_broken_card",save_errors=False)') + @tag('card(type="non_editable_import_test_card",save_errors=False)') + @steps(0, ["start"]) + def step_start(self): + from metaflow import current + from metaflow.plugins.cards.card_modules.test_cards import TestStringComponent + import random + + self.random_number = random.randint(0, 100) + # Adds a card to editable_import_test_card + current.card.append(TestStringComponent(str(self.random_number))) + + @steps(1, ["all"]) + def step_all(self): + pass + + def check_results(self, flow, checker): + run = checker.get_run() + if run is None: + # This means CliCheck is in context. + for step in flow: + if step.name != "start": + continue + + cli_check_dict = checker.artifact_dict(step.name, "random_number") + for task_pathspec in cli_check_dict: + task_id = task_pathspec.split("/")[-1] + random_number = cli_check_dict[task_pathspec]["random_number"] + cards_info = checker.list_cards(step.name, task_id) + # Safely importable cards should be present. + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 2, + True, + ) + impc_e = [ + c + for c in cards_info["cards"] + if c["type"] == "editable_import_test_card" + ] + impc_e = impc_e[0] + impc_ne = [ + c + for c in cards_info["cards"] + if c["type"] == "non_editable_import_test_card" + ] + impc_ne = impc_ne[0] + checker.assert_card( + step.name, + task_id, + impc_ne["type"], + "%s\n" % cards_info["pathspec"], + card_hash=impc_ne["hash"], + exact_match=True, + ) + checker.assert_card( + step.name, + task_id, + impc_e["type"], + "%d\n" % random_number, + card_hash=impc_e["hash"], + exact_match=True, + ) + + else: + # This means MetadataCheck is in context. + for step in flow: + if step.name != "start": + continue + meta_check_dict = checker.artifact_dict(step.name, "random_number") + for task_id in meta_check_dict: + random_number = meta_check_dict[task_id]["random_number"] + cards_info = checker.list_cards( + step.name, + task_id, + ) + assert_equals( + cards_info is not None + and "cards" in cards_info + and len(cards_info["cards"]) == 2, + True, + ) + impc_e = [ + c + for c in cards_info["cards"] + if c["type"] == "editable_import_test_card" + ] + impc_e = impc_e[0] + impc_ne = [ + c + for c in cards_info["cards"] + if c["type"] == "non_editable_import_test_card" + ] + impc_ne = impc_ne[0] + # print() + task_pathspec = cards_info["pathspec"] + checker.assert_card( + step.name, + task_id, + impc_ne["type"], + "%s" % task_pathspec, + card_hash=impc_ne["hash"], + exact_match=True, + ) + checker.assert_card( + step.name, + task_id, + impc_e["type"], + "%d" % random_number, + card_hash=impc_e["hash"], + exact_match=True, + ) From 85ec27858004b570595eaab6cdae9322ead5a01a Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Sun, 23 Jan 2022 10:30:05 -0800 Subject: [PATCH 10/12] Added Env var to tests to avoid warnings added to cards. --- test/core/tests/card_default_editable.py | 2 ++ test/core/tests/card_default_editable_customize.py | 5 +++++ test/core/tests/card_default_editable_with_id.py | 4 ++++ test/core/tests/card_id_append.py | 4 ++++ test/core/tests/card_import.py | 5 +++++ 5 files changed, 20 insertions(+) diff --git a/test/core/tests/card_default_editable.py b/test/core/tests/card_default_editable.py index 1da3dfdfc5c..f900ab0b14a 100644 --- a/test/core/tests/card_default_editable.py +++ b/test/core/tests/card_default_editable.py @@ -10,6 +10,8 @@ class DefaultEditableCardTest(MetaflowTest): """ HEADER = """ +import os +os.environ['METAFLOW_CARD_NO_WARNING'] = 'True' class MyNativeType: at = 0 def get(self): diff --git a/test/core/tests/card_default_editable_customize.py b/test/core/tests/card_default_editable_customize.py index 3abf6aa906c..5fb6b7f3f13 100644 --- a/test/core/tests/card_default_editable_customize.py +++ b/test/core/tests/card_default_editable_customize.py @@ -9,6 +9,11 @@ class DefaultEditableCardWithCustomizeTest(MetaflowTest): PRIORITY = 3 + HEADER = """ +import os +os.environ['METAFLOW_CARD_NO_WARNING'] = 'True' + """ + @tag('card(type="test_editable_card",customize=True)') @tag('card(type="test_editable_card",id="abc")') @tag('card(type="taskspec_card")') diff --git a/test/core/tests/card_default_editable_with_id.py b/test/core/tests/card_default_editable_with_id.py index 2b54a651e88..eca5eb1dfac 100644 --- a/test/core/tests/card_default_editable_with_id.py +++ b/test/core/tests/card_default_editable_with_id.py @@ -8,6 +8,10 @@ class DefaultEditableCardWithIdTest(MetaflowTest): """ PRIORITY = 3 + HEADER = """ +import os +os.environ['METAFLOW_CARD_NO_WARNING'] = 'True' + """ @tag('card(type="test_editable_card",id="abc")') @tag('card(type="test_editable_card")') diff --git a/test/core/tests/card_id_append.py b/test/core/tests/card_id_append.py index 2765c2f8e48..54f53037e42 100644 --- a/test/core/tests/card_id_append.py +++ b/test/core/tests/card_id_append.py @@ -9,6 +9,10 @@ class CardsWithIdTest(MetaflowTest): """ PRIORITY = 3 + HEADER = """ +import os +os.environ['METAFLOW_CARD_NO_WARNING'] = 'True' + """ @tag('card(type="test_editable_card",id="xyz")') @tag('card(type="test_editable_card",id="abc")') diff --git a/test/core/tests/card_import.py b/test/core/tests/card_import.py index c462f2f3fdb..ec53f43dce7 100644 --- a/test/core/tests/card_import.py +++ b/test/core/tests/card_import.py @@ -9,6 +9,11 @@ class CardImportTest(MetaflowTest): - If the submodule has errors while importing then the rest of metaflow should not fail. """ + HEADER = """ +import os +os.environ['METAFLOW_CARD_NO_WARNING'] = 'True' + """ + PRIORITY = 4 @tag('card(type="editable_import_test_card",save_errors=False)') From 21791d2a691ed1efabbc9a2c5afbffc088b6bf9a Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Sun, 23 Jan 2022 13:35:31 -0800 Subject: [PATCH 11/12] Added Test for card resume. --- test/core/tests/card_resume.py | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 test/core/tests/card_resume.py diff --git a/test/core/tests/card_resume.py b/test/core/tests/card_resume.py new file mode 100644 index 00000000000..643ee85d68d --- /dev/null +++ b/test/core/tests/card_resume.py @@ -0,0 +1,49 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + + +class CardResumeTest(MetaflowTest): + """ + Resuming a flow with card decorators should reference a origin task's card when calling `get_cards` or `card get` cli commands. + """ + + RESUME = True + PRIORITY = 4 + + @tag('card(type="taskspec_card")') + @steps(0, ["start"]) + def step_start(self): + from metaflow import current + + self.origin_pathspec = current.pathspec + + @steps(0, ["singleton-end"], required=True) + def step_end(self): + if not is_resumed(): + raise ResumeFromHere() + + @steps(2, ["all"]) + def step_all(self): + pass + + def check_results(self, flow, checker): + run = checker.get_run() + if run is not None: + for step in run.steps(): + if step.id == "start": + task = step.task + checker.assert_card( + step.id, task.id, "taskspec_card", "%s" % task.origin_pathspec + ) + else: + for step in flow: + if step.name != "start": + continue + cli_check_dict = checker.artifact_dict(step.name, "origin_pathspec") + for task_pathspec in cli_check_dict: + task_id = task_pathspec.split("/")[-1] + checker.assert_card( + step.name, + task_id, + "taskspec_card", + "%s\n" % cli_check_dict[task_pathspec]["origin_pathspec"], + ) From ccf4f0394d0103d744f09c651cbaa1277ae6b9b6 Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Mon, 24 Jan 2022 13:46:57 -0800 Subject: [PATCH 12/12] Stacked @card v1.2: Card Dev Docs (#899) - Added Dev Docs --- docs/cards.md | 306 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 docs/cards.md diff --git a/docs/cards.md b/docs/cards.md new file mode 100644 index 00000000000..cf3a5d9fe02 --- /dev/null +++ b/docs/cards.md @@ -0,0 +1,306 @@ +# Metaflow Cards + +Metaflow Cards make it possible to produce human-readable report cards automatically from any Metaflow tasks. You can use the feature to observe results of Metaflow runs, visualize models, and share outcomes with non-technical stakeholders. + +While Metaflow comes with a built-in default card that shows all outputs of a task without any changes in the code, the most exciting use cases are enabled by custom cards: With a few additional lines of Python code, you can change the structure and the content of the report to highlight data that matters to you. For more flexible or advanced reports, you can create custom card templates that generate arbitrary HTML. + +Anyone can create card templates and share them as standard Python packages. Cards can be accessed via the Metaflow CLI even without an internet connection, making it possible to use them in security-conscious environments. Cards are also integrated with the latest release of the Metaflow GUI, allowing you to enrich the existing task view with application-specific information. + +## Technical Details + +### Table Of Contents +* [@card decorator](#card-decorator) + * [Parameters](#parameters) + * [Usage Semantics](#usage-semantics) +* [CardDatastore](#carddatastore) +* [Card CLI](#card-cli) +* [Access cards in notebooks](#access-cards-in-notebooks) +* [MetaflowCard](#metaflowcard) + * [Attributes](#attributes) + * [__init__ Parameters](#__init__-parameters) +* [MetaflowCardComponent](#metaflowcardcomponent) +* [DefaultCard](#defaultcard) +* [Default MetaflowCardComponent](#default-metaflowcardcomponent) +* [Editing MetaflowCard from @step code](#editing-metaflowcard-from-step-code) + * [current.card (CardComponentCollector)](#currentcard-cardcomponentcollector) +* [Creating Custom Installable Cards](#creating-custom-cards) + +Metaflow cards can be created by placing an [`@card` decorator](#@card-decorator) over a `@step`. Cards are created after a metaflow task ( instantiation of each `@step` ) completes execution. You can have multiple `@card` decorators for an individual `@step`. Each decorator takes a `type` argument which defaults to the value `default`. The `type` argument corresponds the [MetaflowCard.type](#metaflowcard). On task completion ,every `@card` decorator creates a separate subprocess to call the [card create cli command](#card-cli). This command will create and [store](#carddatastore) the HTML page for the card. + +Since the cards are stored in the datastore we can access them via the `view/get` commands in the [card_cli](#card-cli) or by using the `get_cards` [function](../metaflow/plugins/cards/card_client.py). + +Metaflow ships with a [DefaultCard](#defaultcard) which visualizes artifacts, images, and `pandas.Dataframe`s. Metaflow also ships custom components like `Image`, `Table`, `Markdown` etc. These can be added to a card at `Task` runtime. Cards can also be edited from `@step` code using the [current.card](#editing-metaflowcard-from-@step-code) interface. `current.card` helps add `MetaflowCardComponent`s from `@step` code to a `MetaflowCard`. `current.card` offers methods like `current.card.append` or `current.card['myid']` to helps add components to a card. Since there can be many `@card`s over a `@step`, `@card` also comes with an `id` argument. The `id` argument helps disambigaute the card a component goes to when using `current.card`. For example, setting `@card(id='myid')` and calling `current.card['myid'].append(x)` will append `MetaflowCardComponent` `x` to the card with `id='myid'`. + +### `@card` decorator +The `@card` [decorator](../metaflow/plugins/cards/card_decorator.py) is implemented by inheriting the `StepDecorator`. The decorator can be placed over `@step` to create an HTML file visualizing information from the task. + +#### Parameters +- `type` `(str)` [Defaults to `default`]: The `type` of `MetaflowCard` to create. More details on `MetaflowCard`s is provided [later in this document](#metaflowcard). +- `options` `(dict)` : options to instantiate a `MetaflowCard`. `MetaflowCard`s will be instantiated with the `options` keyword argument. The value of this argument will be this dictionary. +- `timeout` `(int)` [Defaults to `45`]: Amount of time to wait before killing the card subprocess +- `save_errors` `(bool)` [Defaults to `True`]: If set to `True` then any failure on rendering a `MetaflowCard` will generate an `ErrorCard` instead with the full stack trace of the failure. + +#### Usage Semantics + +```python +from metaflow import FlowSpec,step,card + +class ModelTrainingFlow(FlowSpec): + + @step + def start(self): + self.next(self.train) + + @card( + type='default', + options={"only_repr":False}, + timeout=100, + save_errors = False + ) + @step + def train(self): + import random + import numpy as np + self.loss = np.random.randn(100,100)*100 + self.next(self.end) + + @step + def end(self): + print("Done Computation") + +if __name__ == "__main__": + ModelTrainingFlow() +``` + + + +### `CardDatastore` +The [CardDatastore](../metaflow/plugins/cards/card_datastore.py) is used by the the [card_cli](#card-cli) and the [metaflow card client](#access-cards-in-notebooks) (`get_cards`). It exposes methods to get metadata about a card and the paths to cards for a `pathspec`. + +### Card CLI +Methods exposed by the [card_cli](../metaflow/plugins/cards/.card_cli.py). : + +- `create` : Creates the card in the datastore for a `Task`. Adding a `--render-error-card` will render a `ErrorCard` upon failure to render the card of the selected `type`. If `--render-error-card` is not passed then the CLI will fail loudly with the exception. +```sh +# python myflow.py card create --type --timeout --options "{}" +python myflow.py card create 100/stepname/1000 --type default --timeout 10 --options '{"only_repr":false}' --render-error-card +``` + +- `view/get` : Calling the `view` CLI method will open the card associated for the pathspec in a browser. The `get` method gets the HTML for the card and prints it. You can call the command in the following way. Adding `--follow-resumed` as argument will retrieve the card for the origin resumed task. +```sh +# python myflow.py card view --hash --type +python myflow.py card view 100/stepname/1000 --hash ads34 --type default --follow-resumed +``` + +### Access cards in notebooks +Metaflow also exposes a `get_cards` client that helps resolve cards outside the CLI. Example usage is shown below : +```python +from metaflow import Task +from metaflow.cards import get_cards + +taskspec = 'MyFlow/1000/stepname/100' +task = Task(taskspec) +card_iterator = get_cards(task) # you can even call `get_cards(taskspec)` + +# view card in browser +card = card_iterator[0] +card.view() + +# Get HTML of card +html = card_iterator[0].get() +``` + +### `MetaflowCard` + +The [MetaflowCard](../metaflow/plugins/cards/card_modules/card.py) class is the base class to create custom cards. All subclasses require implementing the `render` function. The `render` function is expected to return a string. Below is an example snippet of usage : +```python +from metaflow.cards import MetaflowCard +# path to the custom html file which is a `mustache` template. +PATH_TO_CUSTOM_HTML = 'myhtml.html' + +class CustomCard(MetaflowCard): + type = "custom_card" + + def __init__(self, options={"no_header": True}, graph=None,components=[]): + super().__init__() + self._no_header = True + self._graph = graph + if "no_header" in options: + self._no_header = options["no_header"] + + def render(self, task): + pt = self._get_mustache() + data = dict( + graph = self._graph, + header = self._no_header + ) + html_template = None + with open(PATH_TO_CUSTOM_HTML) as f: + html_template = f.read() + return pt.render(html_template,data) +``` + +The class consists of the `_get_mustache` method that returns [chevron](https://github.com/noahmorrison/chevron) object ( a `mustache` based [templating engine](http://mustache.github.io/mustache.5.html) ). Using the `mustache` templating engine you can rewrite HTML template file. In the above example the `PATH_TO_CUSTOM_HTML` is the file that holds the `mustache` HTML template. +#### Attributes +- `type (str)` : The `type` of card. Needs to ensure correct resolution. +- `ALLOW_USER_COMPONENTS (bool)` : Setting this to `True` will make the a card be user editable. More information on user editable cards can be found [here](#editing-metaflowcard-from-@step-code). + +#### `__init__` Parameters +- `components` `(List[str])`: `components` is a list of `render`ed `MetaflowCardComponent`s created at `@step` runtime. These are passed to the `card create` cli command via a tempfile path in the `--component-file` argument. +- `graph` `(Dict[str,dict])`: The DAG associated to the flow. It is a dictionary of the form `stepname:step_attributes`. `step_attributes` is a dictionary of metadata about a step , `stepname` is the name of the step in the DAG. +- `options` `(dict)`: helps control the behavior of individual cards. + - For example, the `DefaultCard` supports `options` as dictionary of the form `{"only_repr":True}`. Here setting `only_repr` as `True` will ensure that all artifacts are serialized with `reprlib.repr` function instead of native object serialization. + + +### `MetaflowCardComponent` + +The `render` function of the `MetaflowCardComponent` class returns a `string` or `dict`. It can be called in the `MetaflowCard` class or passed during runtime execution. An example of using `MetaflowCardComponent` inside `MetaflowCard` can be seen below : +```python +from metaflow.cards import MetaflowCard,MetaflowCardComponent + +class Title(MetaflowCardComponent): + def __init__(self,text): + self._text = text + + def render(self): + return "

%s

"%self._text + +class Text(MetaflowCardComponent): + def __init__(self,text): + self._text = text + + def render(self): + return "

%s

"%self._text + +class CustomCard(MetaflowCard): + type = "custom_card" + + HTML = "{data}" + + def __init__(self, options={"no_header": True}, graph=None,components=[]): + super().__init__() + self._no_header = True + self._graph = graph + if "no_header" in options: + self._no_header = options["no_header"] + + def render(self, task): + pt = self._get_mustache() + data = '\n'.join([ + Title("Title 1").render(), + Text("some text comes here").render(), + Title("Title 2").render(), + Text("some text comes here again").render(), + ]) + data = dict( + data = data + ) + html_template = self.HTML + + return pt.render(html_template,data) +``` + +### `DefaultCard` +The [DefaultCard](../metaflow/plugins/cards/card_modules/basic.py) is a default card exposed by metaflow. This will be used when the `@card` decorator is called without any `type` argument or called with `type='default'` argument. It will also be the default card used with cli. The card uses a [HTML template](../metaflow/plugins/cards/card_modules/base.html) along with a [JS](../metaflow/plugins/cards/card_modules/main.js) and a [CSS](../metaflow/plugins/cards/card_modules/bundle.css) files. + +The [HTML](../metaflow/plugins/cards/card_modules/base.html) is a template which works with [JS](../metaflow/plugins/cards/card_modules/main.js) and [CSS](../metaflow/plugins/cards/card_modules/bundle.css). + +The JS and CSS are created after building the JS and CSS from the [cards-ui](../metaflow/plugins/cards/ui/README.md) directory. [cards-ui](../metaflow/plugins/cards/ui/README.md) consists of the JS app that generates the HTML view from a JSON object. + +### Default `MetaflowCardComponent` + +`DefaultCard`/`BlankCard` can be given `MetaflowCardComponent` from `@step` code. The following are the main `MetaflowCardComponent`s available via `metaflow.cards`. +- `Artifact` : A component to help log artifacts at task runtime. + - Example : `Artifact(some_variable,compress=True)` +- `Table` : A component to create a table in the card HTML. Consists of convenience methods : + - `Table.from_dataframe(df)` to make a table from a dataframe. +- `Image` : A component to create an image in the card HTML: + - `Image(bytearr,"my Image from bytes")`: to directly from `bytes` + - `Image.from_pil_image(pilimage,"From PIL Image")` : to create an image from a `PIL.Image` + - `Image.from_matplotlib(plot,"My matplotlib plot")` : to create an image from a plot +- `Error` : A wrapper subcomponent to display errors. Accepts an `exception` and a `title` as arguments. +- `Markdown` : A component that renders markdown in the HTML template +### Editing `MetaflowCard` from `@step` code +`MetaflowCard`s can be edited from `@step` code using the `current.card` interface. The `current.card` interface will only be active when a `@card` decorator is placed over a `@step`. To understand the workings of `current.card` consider the following snippet. +```python +@card(type='blank',id='a') +@card(type='default') +@step +def train(self): + from metaflow.cards import Markdown + from metaflow import current + current.card.append(Markdown('# This is present in the blank card with id "a"')) + current.card['a'].append(Markdown('# This is present in the default card')) + self.t = dict( + hi = 1, + hello = 2 + ) + self.next(self.end) +``` +In the above scenario there are two `@card` decorators which are being customized by `current.card`. The `current.card.append`/ `current.card['a'].append` methods only accepts objects which are subclasses of `MetaflowCardComponent`. The `current.card.append`/ `current.card['a'].append` methods only add a component to **one** card. Since there can be many cards for a `@step`, a **default editabled card** is resolved to disambiguate which card has access to the `append`/`extend` methods within the `@step`. A default editable card is a card that will have access to the `current.card.append`/`current.card.extend` methods. `current.card` resolve the default editable card before a `@step` code gets executed. It sets the default editable card once the last `@card` decorator calls the `task_pre_step` callback. In the above case, `current.card.append` will add a `Markdown` component to the card of type `default`. `current.card['a'].append` will add the `Markdown` to the `blank` card whose `id` is `a`. A `MetaflowCard` can be user editable, if `ALLOW_USER_COMPONENTS` is set to `True`. Since cards can be of many types, **some cards can also be non editable by users** (Cards with `ALLOW_USER_COMPONENTS=False`). Those cards won't be eligible to access the `current.card.append`. A non user editable card can be edited through expicitly setting an `id` and accessing it via `current.card['myid'].append` or by looking it up by its type via `current.card.get(type=’pytorch’)`. + +#### `current.card` (`CardComponentCollector`) + +The `CardComponentCollector` is the object responsible for resolving a `MetaflowCardComponent` to the card referenced in the `@card` decorator. + +Since there can be many cards, `CardComponentCollector` has a `_finalize` function. The `_finalize` function is called once the **last** `@card` decorator calls `task_pre_step`. The `_finalize` function will try to find the **default editable card** from all the `@card` decorators on the `@step`. The default editable card is the card that can access the `current.card.append`/`current.card.extend` methods. If there are multiple editable cards with no `id` then `current.card` will throw warnings when users call `current.card.append`. This is done because `current.card` cannot resolve which card the component belongs. + +The `@card` decorator also exposes another argument called `customize=True`. **Only one `@card` decorator over a `@step` can have `customize=True`**. Since cards can also be added from CLI when running a flow, adding `@card(customize=True)` will set **that particular card** from the decorator as default editable. This means that `current.card.append` will append to the card belonging to `@card` with `customize=True`. If there is more than one `@card` decorator with `customize=True` then `current.card` will throw warnings that `append` won't work. + +One important feature of the `current.card` object is that it will not fail. Even when users try to access `current.card.append` with multiple editable cards, we throw warnings but don't fail. `current.card` will also not fail when a user tries to access a card of a non-existing id via `current.card['mycard']`. Since `current.card['mycard']` gives reference to a `list` of `MetaflowCardComponent`s, `current.card` will return a non-referenced `list` when users try to access the dictionary inteface with a non existing id (`current.card['my_non_existant_card']`). + +Once the `@step` completes execution, every `@card` decorator will call `current.card._serialize` (`CardComponentCollector._serialize`) to get a JSON serializable list of `str`/`dict` objects. The `_serialize` function internally calls all [component's](#metaflowcardcomponent) `render` function. This list is `json.dump`ed to a `tempfile` and passed to the `card create` subprocess where the `MetaflowCard` can use them in the final output. + +### Creating Custom Installable Cards +Custom cards can be installed with the help of the `metaflow_extensions` namespace package. Every `metaflow_extensions` module having custom cards should follow the below directory structure. . You can see an example cookie-cutter card over [here](https://github.com/outerbounds/metaflow-card-html). +``` +your_package/ # the name of this dir doesn't matter +├ setup.py +├ metaflow_extensions/ +│ └ organizationA/ # NO __init__.py file, This is a namespace package. +│ └ plugins/ # NO __init__.py file, This is a namespace package. +│ └ cards/ # NO __init__.py file, This is a namespace package. +│ └ my_card_module/ # Name of card_module +│ └ __init__.py. # This is the __init__.py is required to recoginize `my_card_module` as a package +│ └ somerandomfile.py. # Some file as a part of the package. +. +``` + +The `__init__.py` of the `metaflow_extensions.organizationA.plugins.cards.my_card_module`, requires a `CARDS` attribute which needs to be a `list` of objects inheriting `MetaflowCard` class. For Example, in the below `__init__.py` file exposes a `MetaflowCard` of `type` "y_card2". + +```python +from metaflow.cards import MetaflowCard + +class YCard(MetaflowCard): + type = "y_card2" + + ALLOW_USER_COMPONENTS = True + + def __init__(self, options={}, components=[], graph=None): + self._components = components + + def render(self, task): + return "I am Y card %s" % '\n'.join([comp for comp in self._components]) + +CARDS = [YCard] +``` + +Having this `metaflow_extensions` module present in the PYTHONPATH can also work. Custom cards can also be created by reusing components provided by metaflow. For Example : +```python +from metaflow.cards import BlankCard +from metaflow.cards import Artifact,Table + +class MyCustomCard(BlankCard): + + type = 'my_custom_card' + + def render(self, task): + art_com [ + Table( + [[Artifact(k.data,k.id)] for k in task] + ).render() + ] + return super().render(task,components=[art_com]) + +CARDS = [MyCustomCard] +``` \ No newline at end of file