From d35b1d802a30b1ad64556044c7950173710ed66a Mon Sep 17 00:00:00 2001 From: S Wong Date: Fri, 16 Apr 2021 11:39:06 +0100 Subject: [PATCH] [KED-2436] Create modular pipeline visualisation MVP using the tags concept (#421) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [KED-1951] Backend to send modular pipelines to kedro viz (#394) * WIP add modular pipelines * Expose modular pipelines and add testing data * Lint * undo push of package-lock * Revert package lock * Fix lint * Return modular_pipelines in pipeline data endpoint for selected pipeline + update test data * Address comments on PR * Cleanup and lint * Add modular pipelines to datasets and parameter nodes. Some refactoring for clarity * Temporarily skip js tests to make sure all python stuff works * Put back JS tests for CI * First iteration of addressing comments on PR * Correctly deduce modular pipeline from dataset * Add all modular pipelines for all nodes * Check that dataset namespace is actually a modular pipeline * Undo check if namespace is modular pipeline * updated FE tests to match new animals dataset (#401) * Add modular pipelines to parameter nodes (#402) * WIP add modular pipelines * Expose modular pipelines and add testing data * Lint * undo push of package-lock * Revert package lock * Revert package lock * Fix lint * Return modular_pipelines in pipeline data endpoint for selected pipeline + update test data * Address comments on PR * Cleanup and lint * Add modular pipelines to datasets and parameter nodes. Some refactoring for clarity * Temporarily skip js tests to make sure all python stuff works * Put back JS tests for CI * First iteration of addressing comments on PR * Correctly deduce modular pipeline from dataset * Add all modular pipelines for all nodes * Check that dataset namespace is actually a modular pipeline * Undo check if namespace is modular pipeline * Add modular pipelines for parameter nodes * Verify if modular pipelines listed are actual modular piplines * Temporarily disable JS tests to make sure other steps pass * Put back JS tests, all other checks pass ✅ * Update package/kedro_viz/server.py Co-authored-by: Ivan Danov * Address review comments * Treat dataset node modular pipelines the same as task node modular pipelines. Co-authored-by: Ivan Danov * Send correct format for modular pipelines from /pipelines endpoint (#408) * Modular pipeline tags concept (#410) * set up store functions for incoming modular pipeline data * added additional test for modular pipeline * set up flag for modular pipeline * set up selector to get list of modular pipelines and nodes * add ncheck for node modular pipeline data in selector * set up modular pipeline on sidebar * refactor node-list to enable change of both modular pipeline and tags * further setup reducer and node selection * added item label check * hide modular pipeline features behind a flag * fix failing tests and set up new data set * added tests for modular pipeline actions * further revisions * enable indented styling for lazy list * update readme to explain modular pipeline and tag filtering behaviour * Fix pylint * updates as per PR comments * further adjustments per PR comments * update tests to reflect latest PR changes * refactor getItemLabel in node-list-row-list * fix spelling in random-data * further refactoring of getItemLabel Co-authored-by: Merel Theisen * quick fix to ensure selector works for pipelines with no defined modular pipelines * delete unneeded selector * delete unneeded selector * Bugfix: Ensure JSON backwards-compatibility The application should still work without throwing an error, even when "modular_pipelines" is not present in the JSON dataset Co-authored-by: Merel Theisen <49397448+MerelTheisenQB@users.noreply.github.com> Co-authored-by: Ivan Danov Co-authored-by: Merel Theisen Co-authored-by: Richard Westenra --- README.md | 1 + package.json | 2 +- package/features/steps/cli_steps.py | 6 +- package/kedro_viz/server.py | 187 ++++++++++++---- package/tests/test-format.json | 7 + package/tests/test_server.py | 161 +++++++++---- src/actions/actions.test.js | 36 +++ src/actions/modular-pipelines.js | 33 +++ src/components/node-list/index.js | 105 ++++++--- .../node-list/node-list-groups.test.js | 2 +- src/components/node-list/node-list-items.js | 87 ++++++-- .../node-list/node-list-items.test.js | 15 +- .../node-list/node-list-row-list.js | 16 +- src/components/node-list/node-list.test.js | 25 ++- src/config.js | 42 +++- src/reducers/index.js | 3 +- src/reducers/modular-pipelines.js | 46 ++++ src/selectors/disabled.js | 38 +++- src/selectors/linked-nodes.test.js | 9 +- src/selectors/modular-pipelines.js | 34 +++ src/selectors/nodes.js | 33 ++- src/selectors/pipeline.js | 23 ++ src/store/normalize-data.js | 4 +- src/utils/data/animals.mock.json | 211 ++++++++++++------ src/utils/data/demo.mock.json | 175 ++++++++++----- src/utils/graph/constraints.js | 2 +- src/utils/graph/graph.test.js | 2 +- src/utils/random-data.js | 41 +++- 28 files changed, 1050 insertions(+), 296 deletions(-) create mode 100644 src/actions/modular-pipelines.js create mode 100644 src/reducers/modular-pipelines.js create mode 100644 src/selectors/modular-pipelines.js diff --git a/README.md b/README.md index 0298ffbf05..325c2ad6ea 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ The following flags are available to toggle experimental features: - `oldgraph` - From release v3.8.0. Display old version of graph (dagre algorithm) without improved graphing algorithm. (default `false`) - `sizewarning` - From release v3.9.1. Show a warning before rendering very large graphs. (default `true`) +- `modularpipeline` - From release v3.11.0. Enables filtering of nodes by modular pipelines. Note that selecting both modular pipeline and tag filters will only return nodes that belongs to both categories. (default `false`). Note that newgraph has been removed from v3.8.0 onwards and is now the default functionality. Should there be issues with your project, see the oldgraph flag above. diff --git a/package.json b/package.json index 70c8d2705e..259a1fbc10 100644 --- a/package.json +++ b/package.json @@ -122,4 +122,4 @@ "not op_mini all" ], "snyk": true -} \ No newline at end of file +} diff --git a/package/features/steps/cli_steps.py b/package/features/steps/cli_steps.py index 1625ef626e..f05fcc8c98 100644 --- a/package/features/steps/cli_steps.py +++ b/package/features/steps/cli_steps.py @@ -70,8 +70,7 @@ def create_config_file_with_example(context): @given("I have run a non-interactive kedro new") def create_project_from_config_file(context): - """Behave step to run kedro new given the config I previously created. - """ + """Behave step to run kedro new given the config I previously created.""" res = run( [context.kedro, "new", "-c", str(context.config_file)], env=context.env, @@ -82,8 +81,7 @@ def create_project_from_config_file(context): @given("I have run a non-interactive kedro new with {starter} starter") def create_project_with_starter(context, starter): - """Behave step to run kedro new given the config I previously created. - """ + """Behave step to run kedro new given the config I previously created.""" res = run( [ context.kedro, diff --git a/package/kedro_viz/server.py b/package/kedro_viz/server.py index 70ed839ea5..11c0dbe65f 100644 --- a/package/kedro_viz/server.py +++ b/package/kedro_viz/server.py @@ -296,7 +296,15 @@ def _pretty_name(name: str) -> str: return " ".join(parts) -def format_pipelines_data(pipelines: Dict[str, "Pipeline"]) -> Dict[str, list]: +def _pretty_modular_pipeline_name(modular_pipeline: str) -> str: + """Takes the namespace of a modular pipeline and prettifies the + last part to show as the modular pipeline name.""" + chunks = modular_pipeline.split(".") + last_chunk = chunks[-1] + return _pretty_name(last_chunk) + + +def format_pipelines_data(pipelines: Dict[str, "Pipeline"]) -> Dict[str, Any]: """ Format pipelines and catalog data from Kedro for kedro-viz. @@ -317,6 +325,8 @@ def format_pipelines_data(pipelines: Dict[str, "Pipeline"]) -> Dict[str, list]: # keep track of node_id -> set(child_node_ids) for layers sorting node_dependencies = defaultdict(set) tags = set() + # keep track of modular pipelines + modular_pipelines = set() for pipeline_key, pipeline in pipelines.items(): pipelines_list.append({"id": pipeline_key, "name": _pretty_name(pipeline_key)}) @@ -328,6 +338,7 @@ def format_pipelines_data(pipelines: Dict[str, "Pipeline"]) -> Dict[str, list]: tags, edges_list, nodes_list, + modular_pipelines, ) # sort tags @@ -342,6 +353,9 @@ def format_pipelines_data(pipelines: Dict[str, "Pipeline"]) -> Dict[str, list]: else pipelines_list[0]["id"] ) + sorted_modular_pipelines = _sort_and_format_modular_pipelines(modular_pipelines) + _remove_non_modular_pipelines(nodes_list, modular_pipelines) + return { "nodes": nodes_list, "edges": edges_list, @@ -349,11 +363,29 @@ def format_pipelines_data(pipelines: Dict[str, "Pipeline"]) -> Dict[str, list]: "layers": sorted_layers, "pipelines": pipelines_list, "selected_pipeline": selected_pipeline, + "modular_pipelines": sorted_modular_pipelines, } -def _is_namespace_param(namespace: str) -> bool: - """Returns whether a dataset namespace is a parameter""" +def _remove_non_modular_pipelines(nodes_list, modular_pipelines): + """Check parameter nodes only contain existing modular pipelines from the task nodes + and remove those listed that aren't modular pipelines. + + Args: + nodes_list: List of all nodes. + modular_pipelines: Set of modular pipelines for all nodes. + + """ + for node in nodes_list: + if node["type"] == "parameters" and node["modular_pipelines"]: + pipes = [ + pipe for pipe in node["modular_pipelines"] if pipe in modular_pipelines + ] + node["modular_pipelines"] = sorted(pipes) + + +def _is_dataset_param(namespace: str) -> bool: + """Returns whether a dataset is a parameter""" return namespace.lower().startswith("param") @@ -366,6 +398,7 @@ def format_pipeline_data( tags: Set[str], edges_list: List[dict], nodes_list: List[dict], + modular_pipelines: Set[str], ) -> None: """Format pipeline and catalog data from Kedro for kedro-viz. @@ -376,12 +409,13 @@ def format_pipeline_data( node_dependencies: Dictionary of id and node dependencies. edges_list: List of all edges. nodes_list: List of all nodes. + modular_pipelines: Set of modular pipelines for all nodes. """ - # keep_track of {data_set_namespace -> set(tags)} - namespace_tags = defaultdict(set) - # keep track of {data_set_namespace -> layer it belongs to} - namespace_to_layer = {} + # keep_track of {dataset_full_name -> set(tags)} + dataset_name_tags = defaultdict(set) + # keep track of {dataset_full_name -> layer it belongs to} + dataset_name_to_layer = {} dataset_to_layer = _construct_layer_mapping() @@ -390,6 +424,11 @@ def format_pipeline_data( task_id = _hash(str(node)) tags.update(node.tags) _JSON_NODES[task_id] = {"type": "task", "obj": node} + + # Modular pipelines the current node is part of. + node_modular_pipelines = _expand_namespaces(node.namespace) + modular_pipelines.update(node_modular_pipelines) + if task_id not in nodes: nodes[task_id] = { "type": "task", @@ -398,72 +437,124 @@ def format_pipeline_data( "full_name": getattr(node, "_func_name", str(node)), "tags": sorted(node.tags), "pipelines": [pipeline_key], + "modular_pipelines": sorted(node_modular_pipelines), } nodes_list.append(nodes[task_id]) else: nodes[task_id]["pipelines"].append(pipeline_key) for data_set in node.inputs: - namespace = data_set.split("@")[0] - namespace_to_layer[namespace] = dataset_to_layer.get(data_set) - namespace_id = _hash(namespace) - edge = {"source": namespace_id, "target": task_id} + dataset_full_name = data_set.split("@")[0] + dataset_name_to_layer[dataset_full_name] = dataset_to_layer.get(data_set) + dataset_id = _hash(dataset_full_name) + edge = {"source": dataset_id, "target": task_id} if edge not in edges_list: edges_list.append(edge) - namespace_tags[namespace].update(node.tags) - node_dependencies[namespace_id].add(task_id) + dataset_name_tags[dataset_full_name].update(node.tags) + node_dependencies[dataset_id].add(task_id) # if it is a parameter, add it to the node's data - if _is_namespace_param(namespace): - if "parameters" not in _JSON_NODES[task_id]: - _JSON_NODES[task_id]["parameters"] = {} - - if namespace == "parameters": - _JSON_NODES[task_id]["parameters"] = _get_dataset_data_params( - namespace - ).load() - else: - parameter_name = namespace.replace("params:", "") - parameter_value = _get_dataset_data_params(namespace).load() - _JSON_NODES[task_id]["parameters"][parameter_name] = parameter_value + if _is_dataset_param(dataset_full_name): + _add_parameter_data_to_node(dataset_full_name, task_id) for data_set in node.outputs: - namespace = data_set.split("@")[0] - namespace_to_layer[namespace] = dataset_to_layer.get(data_set) - namespace_id = _hash(namespace) - edge = {"source": task_id, "target": namespace_id} + dataset_full_name = data_set.split("@")[0] + dataset_name_to_layer[dataset_full_name] = dataset_to_layer.get(data_set) + dataset_id = _hash(dataset_full_name) + edge = {"source": task_id, "target": dataset_id} if edge not in edges_list: edges_list.append(edge) - namespace_tags[namespace].update(node.tags) - node_dependencies[task_id].add(namespace_id) + dataset_name_tags[dataset_full_name].update(node.tags) + node_dependencies[task_id].add(dataset_id) + # Parameters and data - for namespace, tag_names in sorted(namespace_tags.items()): - is_param = _is_namespace_param(namespace) - node_id = _hash(namespace) + for dataset_full_name, tag_names in sorted(dataset_name_tags.items()): + is_param = _is_dataset_param(dataset_full_name) + node_id = _hash(dataset_full_name) _JSON_NODES[node_id] = { "type": "parameters" if is_param else "data", - "obj": _get_dataset_data_params(namespace), + "obj": _get_dataset_data_params(dataset_full_name), } - if is_param and namespace != "parameters": + + parameter_name = "" + if is_param and dataset_full_name != "parameters": + parameter_name = dataset_full_name.replace("params:", "") # Add "parameter_name" key only for "params:" prefix. - _JSON_NODES[node_id]["parameter_name"] = namespace.replace("params:", "") + _JSON_NODES[node_id]["parameter_name"] = parameter_name + + if is_param: + dataset_modular_pipelines = _expand_namespaces( + _get_namespace(parameter_name) + ) + else: + dataset_modular_pipelines = _expand_namespaces( + _get_namespace(dataset_full_name) + ) + modular_pipelines.update(dataset_modular_pipelines) if node_id not in nodes: nodes[node_id] = { "type": "parameters" if is_param else "data", "id": node_id, - "name": _pretty_name(namespace), - "full_name": namespace, + "name": _pretty_name(dataset_full_name), + "full_name": dataset_full_name, "tags": sorted(tag_names), - "layer": namespace_to_layer[namespace], + "layer": dataset_name_to_layer[dataset_full_name], "pipelines": [pipeline_key], + "modular_pipelines": dataset_modular_pipelines, } nodes_list.append(nodes[node_id]) else: nodes[node_id]["pipelines"].append(pipeline_key) +def _expand_namespaces(namespace): + """ + Expand a node's namespace to the modular pipelines this node belongs to. + For example, if the node's namespace is: "pipeline1.data_science" + it should be expanded to: ["pipeline1", "pipeline1.data_science"] + """ + if not namespace: + return [] + namespace_list = [] + namespace_chunks = namespace.split(".") + prefix = "" + for chunk in namespace_chunks: + if prefix: + prefix = f"{prefix}.{chunk}" + else: + prefix = chunk + namespace_list.append(prefix) + return namespace_list + + +def _add_parameter_data_to_node(dataset_namespace, task_id): + if "parameters" not in _JSON_NODES[task_id]: + _JSON_NODES[task_id]["parameters"] = {} + + if dataset_namespace == "parameters": + _JSON_NODES[task_id]["parameters"] = _get_dataset_data_params( + dataset_namespace + ).load() + else: + parameter_name = dataset_namespace.replace("params:", "") + parameter_value = _get_dataset_data_params(dataset_namespace).load() + _JSON_NODES[task_id]["parameters"][parameter_name] = parameter_value + + +def _get_namespace(dataset_full_name): + """ + Extract the namespace from the full dataset/parameter name. + """ + if "." in dataset_full_name: + # The last part of the namespace is the actual name of the dataset + # e.g. in pipeline1.data_science.a, "pipeline1.data_science" indicates + # the modular pipelines and "a" the name of the dataset. + return dataset_full_name.rsplit(".", 1)[0] + return None + + def _get_dataset_data_params(namespace: str): if KEDRO_VERSION.match(">=0.16.0"): try: @@ -484,6 +575,16 @@ def _get_parameter_values(node: Dict) -> Any: return parameter_values +def _sort_and_format_modular_pipelines(modular_pipelines): + return [ + { + "id": modular_pipeline, + "name": _pretty_modular_pipeline_name(modular_pipeline), + } + for modular_pipeline in sorted(modular_pipelines) + ] + + @app.route("/api/main") def nodes_json(): """Serve the data from all Kedro pipelines in the project. @@ -501,17 +602,21 @@ def pipeline_data(pipeline_id): pipeline_node_ids = set() pipeline_nodes = [] + modular_pipelines = set() for node in _DATA["nodes"]: if pipeline_id in node["pipelines"]: pipeline_node_ids.add(node["id"]) pipeline_nodes.append(node) + modular_pipelines.update(node["modular_pipelines"]) pipeline_edges = [] for edge in _DATA["edges"]: if {edge["source"], edge["target"]} <= pipeline_node_ids: pipeline_edges.append(edge) + sorted_modular_pipelines = _sort_and_format_modular_pipelines(modular_pipelines) + return jsonify( { "nodes": pipeline_nodes, @@ -520,6 +625,7 @@ def pipeline_data(pipeline_id): "layers": _DATA["layers"], "pipelines": _DATA["pipelines"], "selected_pipeline": current_pipeline["id"], + "modular_pipelines": sorted_modular_pipelines, } ) @@ -723,6 +829,7 @@ def _call_viz( # pylint: disable=invalid-name if __name__ == "__main__": # pragma: no cover import argparse + from kedro.framework.startup import _get_project_metadata parser = argparse.ArgumentParser(description="Launch a development viz server") diff --git a/package/tests/test-format.json b/package/tests/test-format.json index efa5c64b59..8f016bd1c4 100644 --- a/package/tests/test-format.json +++ b/package/tests/test-format.json @@ -3,6 +3,7 @@ { "type": "task", "id": "b2a701fc", + "modular_pipelines": [], "name": "Func1", "full_name": "func1", "tags": [], @@ -13,6 +14,7 @@ { "type": "task", "id": "3751475c", + "modular_pipelines": [], "name": "Func2", "full_name": "func2", "tags": [], @@ -27,6 +29,7 @@ "full_name": "bob_in", "tags": [], "layer": "raw", + "modular_pipelines": [], "pipelines": [ "__default__" ] @@ -38,6 +41,7 @@ "full_name": "bob_out", "tags": [], "layer": null, + "modular_pipelines": [], "pipelines": [ "__default__" ] @@ -49,6 +53,7 @@ "full_name": "params:key", "tags": [], "layer": null, + "modular_pipelines": [], "pipelines": [ "__default__" ] @@ -60,6 +65,7 @@ "full_name": "result", "tags": [], "layer": "final", + "modular_pipelines": [], "pipelines": [ "__default__" ] @@ -92,6 +98,7 @@ "raw", "final" ], + "modular_pipelines": [], "pipelines": [ { "id": "__default__", diff --git a/package/tests/test_server.py b/package/tests/test_server.py index ce0bc34ba6..c02618c7df 100644 --- a/package/tests/test_server.py +++ b/package/tests/test_server.py @@ -40,12 +40,19 @@ import pytest from kedro.extras.datasets.pickle import PickleDataSet from kedro.io import DataCatalog, DataSetNotFoundError, MemoryDataSet -from kedro.pipeline import Pipeline, node +from kedro.pipeline import Pipeline, node, pipeline from toposort import CircularDependencyError import kedro_viz from kedro_viz import server -from kedro_viz.server import _allocate_port, _hash, _sort_layers, format_pipelines_data +from kedro_viz.server import ( + _allocate_port, + _expand_namespaces, + _hash, + _pretty_modular_pipeline_name, + _sort_layers, + format_pipelines_data, +) from kedro_viz.utils import WaitForException input_json_path = ( @@ -62,8 +69,7 @@ def shark(input1, input2, input3, input4): def salmon(dog, rabbit, parameters, cat): - """docstring - """ + """docstring""" return dog, rabbit @@ -71,10 +77,15 @@ def trout(pig, sheep): return pig +def tuna(sheep, plankton1, plankton2): + return sheep + + def get_pipelines(): return { "de": de_pipeline(), - "ds": ds_pipeline(), + "pre_ds": pipeline(pre_ds_pipeline(), namespace="pipeline1.data_science"), + "ds": pipeline(ds_pipeline(), namespace="pipeline2.data_science"), "__default__": create_pipeline(), "empty": Pipeline([]), } @@ -97,15 +108,34 @@ def ds_pipeline(): return ds_pipeline +def pre_ds_pipeline(): + pre_ds_pipeline = Pipeline( + [ + node( + tuna, + inputs=[ + "sheep", + "params:pipeline2.data_science.plankton", + "params:pipeline100.data_science.plankton", + ], + outputs=["dolphin"], + name="tuna", + ) + ] + ) + return pre_ds_pipeline + + def de_pipeline(): de_pipeline = Pipeline( [ node( shark, - inputs=["cat", "weasel", "elephant", "bear"], + inputs=["cat", "nested.weasel", "elephant", "bear"], outputs=["pig", "giraffe"], name="shark", tags=["medium", "large"], + namespace="pipeline1.data_engineering", ), node( salmon, @@ -120,16 +150,20 @@ def de_pipeline(): def create_pipeline(): - return de_pipeline() + ds_pipeline() + return ( + pipeline(pre_ds_pipeline(), namespace="pipeline1.data_science") + + de_pipeline() + + pipeline(ds_pipeline(), namespace="pipeline2.data_science") + ) @pytest.fixture def dummy_layers(): return { - "raw": {"elephant", "bear", "weasel", "cat", "dog"}, + "raw": {"elephant", "bear", "nested.weasel", "cat", "dog"}, "primary": {"sheep"}, "feature": {"pig"}, - "model output": {"horse", "giraffe", "whale"}, + "model output": {"horse", "giraffe", "pipeline2.data_science.whale"}, } @@ -155,6 +189,8 @@ def __init__(self, layers): "cat": PickleDataSet(filepath=str(tmp_path)), "parameters": MemoryDataSet({"name": "value"}), "params:rabbit": MemoryDataSet("value"), + "params:pipeline2.data_science.plankton": MemoryDataSet("value"), + "params:pipeline100.data_science.plankton": MemoryDataSet("value"), } self.layers = layers @@ -176,7 +212,7 @@ def load_context(): mocked_context._get_pipeline = get_pipeline # pylint: disable=protected-access dummy_data_catalog = DummyDataCatalog(dummy_layers) mocked_context.catalog = dummy_data_catalog - mocked_context.pipeline = create_pipeline() + mocked_context.test_pipeline = create_pipeline() return mocked_context mocked_session = mocker.Mock() @@ -198,7 +234,9 @@ def client(): @_USE_PATCHED_CONTEXT -def test_set_port(cli_runner,): +def test_set_port( + cli_runner, +): """Check that port argument is correctly handled.""" result = cli_runner.invoke(server.commands, ["viz", "--port", "8000"]) assert result.exit_code == 0, result.output @@ -231,7 +269,10 @@ def test_no_browser(cli_runner): def test_viz_does_not_need_to_specify_project_path(cli_runner, patched_create_session): cli_runner.invoke(server.commands, ["viz", "--no-browser"]) patched_create_session.assert_called_once_with( - package_name="test", project_path=Path.cwd(), env=None, save_on_close=False, + package_name="test", + project_path=Path.cwd(), + env=None, + save_on_close=False, ) @@ -251,8 +292,7 @@ def test_no_browser_if_not_localhost(cli_runner): def test_load_file_outside_kedro_project(cli_runner, tmp_path): - """Check that running viz with `--load-file` flag works outside of a Kedro project. - """ + """Check that running viz with `--load-file` flag works outside of a Kedro project.""" filepath_json = str(tmp_path / "test.json") data = { "nodes": None, @@ -270,8 +310,7 @@ def test_load_file_outside_kedro_project(cli_runner, tmp_path): @_USE_PATCHED_CONTEXT def test_save_file(cli_runner, tmp_path): - """Check that running with `--save-file` flag saves pipeline JSON file in a specified path. - """ + """Check that running with `--save-file` flag saves pipeline JSON file in a specified path.""" save_path = str(tmp_path / "test.json") result = cli_runner.invoke(server.commands, ["viz", "--save-file", save_path]) @@ -294,8 +333,7 @@ def test_load_file_no_top_level_key(cli_runner, tmp_path): def test_no_load_file(cli_runner): - """Check that running viz without `--load-file` flag should fail outside of a Kedro project. - """ + """Check that running viz without `--load-file` flag should fail outside of a Kedro project.""" result = cli_runner.invoke(server.commands, ["viz"]) assert result.exit_code == 1 @@ -337,14 +375,20 @@ def test_pipelines_endpoint(cli_runner, client): # make sure only edges in the selected pipelines are returned assert data["edges"] == [ - {"source": "2cd4ba93", "target": "e27376a9"}, - {"source": "6525f2e6", "target": "e27376a9"}, - {"source": "e27376a9", "target": "1769e230"}, + {"source": "a761759c", "target": "c8c182ec"}, + {"source": "24e06541", "target": "c8c182ec"}, + {"source": "c8c182ec", "target": "0049a504"}, ] # make sure all tags are returned assert data["tags"] == EXPECTED_PIPELINE_DATA["tags"] + # make sure only the list of modular pipelines is returned for the selected pipeline + assert data["modular_pipelines"] == [ + {"id": "pipeline2", "name": "Pipeline2"}, + {"id": "pipeline2.data_science", "name": "Data Science"}, + ] + @_USE_PATCHED_CONTEXT def test_pipelines_endpoint_invalid_pipeline_id(cli_runner, client): @@ -380,7 +424,6 @@ def test_node_metadata_endpoint_task(cli_runner, client, mocker, tmp_path): assert data["parameters"] == {"name": "value"} - @_USE_PATCHED_CONTEXT def test_node_metadata_endpoint_data_input(cli_runner, client, tmp_path): """Test `/api/nodes/data_id` endpoint is functional and returns a valid JSON.""" @@ -448,43 +491,51 @@ def test_pipeline_flag(cli_runner, client): assert data == { "edges": [ - {"source": "2cd4ba93", "target": "e27376a9"}, - {"source": "6525f2e6", "target": "e27376a9"}, - {"source": "e27376a9", "target": "1769e230"}, + {"source": "a761759c", "target": "c8c182ec"}, + {"source": "24e06541", "target": "c8c182ec"}, + {"source": "c8c182ec", "target": "0049a504"}, + ], + "layers": [], + "modular_pipelines": [ + {"id": "pipeline2", "name": "Pipeline2"}, + {"id": "pipeline2.data_science", "name": "Data Science"}, ], - "layers": ["feature", "primary", "model output"], "nodes": [ { "full_name": "trout", - "id": "e27376a9", + "id": "c8c182ec", + "modular_pipelines": ["pipeline2", "pipeline2.data_science"], "name": "trout", "pipelines": ["ds"], "tags": [], "type": "task", }, { - "full_name": "pig", - "id": "2cd4ba93", - "layer": "feature", - "name": "Pig", + "full_name": "pipeline2.data_science.pig", + "id": "a761759c", + "layer": None, + "modular_pipelines": ["pipeline2", "pipeline2.data_science"], + "name": "Pipeline2.data Science.pig", "pipelines": ["ds"], "tags": [], "type": "data", }, { - "full_name": "sheep", - "id": "6525f2e6", - "layer": "primary", - "name": "Sheep", + "full_name": "pipeline2.data_science.sheep", + "id": "24e06541", + "layer": None, + "modular_pipelines": ["pipeline2", "pipeline2.data_science"], + "name": "Pipeline2.data Science.sheep", "pipelines": ["ds"], "tags": [], "type": "data", }, { - "full_name": "whale", - "id": "1769e230", + "full_name": "pipeline2.data_science.whale", + "id": "0049a504", "layer": "model output", - "name": "Whale", + "modular_pipelines": ["pipeline2", "pipeline2.data_science"], + "name": "Pipeline2.data Science.whale", "pipelines": ["ds"], "tags": [], "type": "data", @@ -545,7 +596,10 @@ class TestCallViz: def test_call_viz_without_project_path(self, patched_create_session): server._call_viz() patched_create_session.assert_called_once_with( - package_name="test", project_path=Path.cwd(), env=None, save_on_close=False, + package_name="test", + project_path=Path.cwd(), + env=None, + save_on_close=False, ) def test_call_viz_with_project_path(self, patched_create_session): @@ -686,7 +740,7 @@ def test_allocation_error(self, kwargs, mocker): @pytest.fixture -def pipeline(): +def test_pipeline(): def func1(a, b): # pylint: disable=unused-argument return a @@ -718,18 +772,20 @@ def new_catalog_with_layers(): return catalog -def test_format_pipelines_data(pipeline, new_catalog_with_layers, mocker): +def test_format_pipelines_data(test_pipeline, new_catalog_with_layers, mocker): mocker.patch("kedro_viz.server._CATALOG", new_catalog_with_layers) - result = format_pipelines_data(pipeline) + result = format_pipelines_data(test_pipeline) result_file_path = Path(__file__).parent / "test-format.json" json_data = json.loads(result_file_path.read_text()) assert json_data == result -def test_format_pipelines_data_no_layers(pipeline, new_catalog_with_layers, mocker): +def test_format_pipelines_data_no_layers( + test_pipeline, new_catalog_with_layers, mocker +): mocker.patch("kedro_viz.server._CATALOG", new_catalog_with_layers) setattr(new_catalog_with_layers, "layers", None) - result = format_pipelines_data(pipeline) + result = format_pipelines_data(test_pipeline) assert result["layers"] == [] @@ -889,3 +945,20 @@ def test_sort_layers_should_raise_on_cyclic_layers(): match="Circular dependencies exist among these items: {'int':{'raw'}, 'raw':{'int'}}", ): _sort_layers(nodes, node_dependencies) + + +def test_expand_namespaces(): + node_namespace = "main_pipeline.sub_pipeline.deepest_pipeline" + expected = [ + "main_pipeline", + "main_pipeline.sub_pipeline", + "main_pipeline.sub_pipeline.deepest_pipeline", + ] + result = _expand_namespaces(node_namespace) + assert result == expected + + +def test_pretty_modular_pipeline_name(): + modular_pipeline_name = "main_pipeline.sub_pipeline.deepest_pipeline" + result = _pretty_modular_pipeline_name(modular_pipeline_name) + assert result == "Deepest Pipeline" diff --git a/src/actions/actions.test.js b/src/actions/actions.test.js index 2f7857a3d6..da9755f671 100644 --- a/src/actions/actions.test.js +++ b/src/actions/actions.test.js @@ -35,6 +35,12 @@ import { toggleTagActive, toggleTagFilter, } from '../actions/tags'; +import { + TOGGLE_MODULAR_PIPELINE_ACTIVE, + TOGGLE_MODULAR_PIPELINE_FILTER, + toggleModularPipelineActive, + toggleModularPipelineFilter, +} from '../actions/modular-pipelines'; import { TOGGLE_TYPE_DISABLED, toggleTypeDisabled } from '../actions/node-type'; describe('actions', () => { @@ -155,6 +161,36 @@ describe('actions', () => { expect(toggleTagFilter(tagIDs, enabled)).toEqual(expectedAction); }); + /** + * Tests for modular pipelines related actions + */ + + it('should create an action to toggle an array of modular pipeliness active state on/off', () => { + const modularPipelineIDs = ['12345', '67890']; + const active = false; + const expectedAction = { + type: TOGGLE_MODULAR_PIPELINE_ACTIVE, + modularPipelineIDs, + active, + }; + expect(toggleModularPipelineActive(modularPipelineIDs, active)).toEqual( + expectedAction + ); + }); + + it('should create an action to toggle an array of tags on/off', () => { + const modularPipelineIDs = ['12345', '67890']; + const enabled = false; + const expectedAction = { + type: TOGGLE_MODULAR_PIPELINE_FILTER, + modularPipelineIDs, + enabled, + }; + expect(toggleModularPipelineFilter(modularPipelineIDs, enabled)).toEqual( + expectedAction + ); + }); + it('should create an action to toggle the theme', () => { const theme = 'light'; const expectedAction = { diff --git a/src/actions/modular-pipelines.js b/src/actions/modular-pipelines.js new file mode 100644 index 0000000000..fecd5848f3 --- /dev/null +++ b/src/actions/modular-pipelines.js @@ -0,0 +1,33 @@ +export const TOGGLE_MODULAR_PIPELINE_ACTIVE = 'TOGGLE_MODULAR_PIPELINE_ACTIVE'; + +/** + * Toggle a modular pipeline item's highlighting on/off (or array of modular pipelines) + * @param {string|Array} modularPipelineIDs Modular pipeline id(s) + * @param {Boolean} active True if modualr pipeline(s) active + */ +export function toggleModularPipelineActive(modularPipelineIDs, active) { + return { + type: TOGGLE_MODULAR_PIPELINE_ACTIVE, + modularPipelineIDs: Array.isArray(modularPipelineIDs) + ? modularPipelineIDs + : [modularPipelineIDs], + active, + }; +} + +export const TOGGLE_MODULAR_PIPELINE_FILTER = 'TOGGLE_MODULAR_PIPELINE_FILTER'; + +/** + * Toggle a modular pipeline's filtering on/off (or array of modular pipelines) + * @param {string|Array} modularPipelineIDs Modular pipeline id(s) + * @param {Boolean} enabled True if modular pipeline(s) enabled + */ +export function toggleModularPipelineFilter(modularPipelineIDs, enabled) { + return { + type: TOGGLE_MODULAR_PIPELINE_FILTER, + modularPipelineIDs: Array.isArray(modularPipelineIDs) + ? modularPipelineIDs + : [modularPipelineIDs], + enabled, + }; +} diff --git a/src/components/node-list/index.js b/src/components/node-list/index.js index 3f656e6e62..ca9b2370d6 100644 --- a/src/components/node-list/index.js +++ b/src/components/node-list/index.js @@ -4,9 +4,14 @@ import utils from '@quantumblack/kedro-ui/lib/utils'; import NodeList from './node-list'; import { getFilteredItems, getGroups, getSections } from './node-list-items'; import { toggleTagActive, toggleTagFilter } from '../../actions/tags'; +import { + toggleModularPipelineActive, + toggleModularPipelineFilter, +} from '../../actions/modular-pipelines'; import { toggleTypeDisabled } from '../../actions/node-type'; import { getNodeTypes } from '../../selectors/node-types'; import { getTagData } from '../../selectors/tags'; +import { getModularPipelineData } from '../../selectors/modular-pipelines'; import { getGroupedNodes, getNodeSelected } from '../../selectors/nodes'; import { loadNodeData, @@ -16,6 +21,7 @@ import { import './styles/node-list.css'; const isTagType = (type) => type === 'tag'; +const isModularPipelineType = (type) => type === 'modularPipeline'; /** * Provides data from the store to populate a NodeList component. @@ -31,7 +37,6 @@ const NodeListProvider = ({ faded, nodes, nodeSelected, - sections, tags, tagsEnabled, types, @@ -40,21 +45,30 @@ const NodeListProvider = ({ onToggleNodeActive, onToggleTagActive, onToggleTagFilter, + onToggleModularPipelineActive, + onToggleModularPipelineFilter, onToggleTypeDisabled, + modularPipelines, + modularPipelinesEnabled, + modularPipelineFlag, + sections, }) => { const [searchValue, updateSearchValue] = useState(''); const items = getFilteredItems({ nodes, tags, tagsEnabled, + modularPipelines, + modularPipelinesEnabled, nodeSelected, searchValue, }); + const groups = getGroups({ types, items }); const onItemClick = (item) => { - if (isTagType(item.type)) { - onTagItemChange(item, item.checked); + if (isTagType(item.type) || isModularPipelineType(item.type)) { + onCategoryItemChange(item, item.checked); } else { if (item.faded || item.selected) { onToggleNodeSelected(null); @@ -65,13 +79,12 @@ const NodeListProvider = ({ }; const onItemChange = (item, checked) => { - if (isTagType(item.type)) { - onTagItemChange(item, checked); + if (isTagType(item.type) || isModularPipelineType(item.type)) { + onCategoryItemChange(item, checked); } else { if (checked) { onToggleNodeActive(null); } - onToggleNodesDisabled([item.id], checked); } }; @@ -79,6 +92,8 @@ const NodeListProvider = ({ const onItemMouseEnter = (item) => { if (isTagType(item.type)) { onToggleTagActive(item.id, true); + } else if (isModularPipelineType(item.type)) { + onToggleModularPipelineActive(item.id, true); } else if (item.visible) { onToggleNodeActive(item.id); } @@ -87,41 +102,64 @@ const NodeListProvider = ({ const onItemMouseLeave = (item) => { if (isTagType(item.type)) { onToggleTagActive(item.id, false); + } else if (isModularPipelineType(item.type)) { + onToggleModularPipelineActive(item.id, false); } else if (item.visible) { onToggleNodeActive(null); } }; const onToggleGroupChecked = (type, checked) => { - if (isTagType(type)) { - // Filter all tags if at least one tag item set, otherwise enable all tags - const tagItems = items[type] || []; - const someTagSet = tagItems.some((tagItem) => !tagItem.unset); - const allTagsValue = someTagSet ? undefined : true; - onToggleTagFilter( - tagItems.map((tag) => tag.id), - allTagsValue + if (isTagType(type) || isModularPipelineType(type)) { + // Filter all category items if at least one item set, otherwise enable all items + const categoryItems = items[type] || []; + const someCategoryItemSet = categoryItems.some( + (categoryItem) => !categoryItem.unset ); + const allCategoryItemsValue = someCategoryItemSet ? undefined : true; + + if (isTagType(type)) { + onToggleTagFilter( + categoryItems.map((tag) => tag.id), + allCategoryItemsValue + ); + } else { + onToggleModularPipelineFilter( + categoryItems.map((item) => item.id), + allCategoryItemsValue + ); + } } else { onToggleTypeDisabled(type, checked); } }; - const onTagItemChange = (tagItem, wasChecked) => { - const tagItems = items[tagItem.type] || []; - const oneTagChecked = - tagItems.filter((tagItem) => tagItem.checked).length === 1; - const shouldResetTags = wasChecked && oneTagChecked; - - if (shouldResetTags) { - // Unset all tags - onToggleTagFilter( - tags.map((tag) => tag.id), - undefined - ); + const onCategoryItemChange = (item, wasChecked) => { + const categoryType = item.type; + const categoryTypeItems = items[categoryType] || []; + const oneCategoryItemChecked = + categoryTypeItems.filter((categoryItem) => categoryItem.checked) + .length === 1; + const shouldResetCategoryItems = wasChecked && oneCategoryItemChecked; + + if (shouldResetCategoryItems) { + // Unset all category item + if (categoryType === 'tag') { + onToggleTagFilter( + tags.map((tag) => tag.id), + undefined + ); + } else { + onToggleModularPipelineFilter( + modularPipelines.map((modularPipeline) => modularPipeline.id), + undefined + ); + } } else { - // Toggle the tag - onToggleTagFilter([tagItem.id], !wasChecked); + // Toggle the category item + categoryType === 'tag' + ? onToggleTagFilter([item.id], !wasChecked) + : onToggleModularPipelineFilter([item.id], !wasChecked); } // Reset node selection @@ -162,8 +200,11 @@ export const mapStateToProps = (state) => ({ tagsEnabled: state.tag.enabled, nodes: getGroupedNodes(state), nodeSelected: getNodeSelected(state), - sections: getSections(state), types: getNodeTypes(state), + modularPipelines: getModularPipelineData(state), + modularPipelinesEnabled: state.modularPipeline.enabled, + modularPipelineFlag: state.flags.modularpipeline, + sections: getSections(state), }); export const mapDispatchToProps = (dispatch) => ({ @@ -173,6 +214,12 @@ export const mapDispatchToProps = (dispatch) => ({ onToggleTagFilter: (tagIDs, enabled) => { dispatch(toggleTagFilter(tagIDs, enabled)); }, + onToggleModularPipelineActive: (modularPipelineIDs, active) => { + dispatch(toggleModularPipelineActive(modularPipelineIDs, active)); + }, + onToggleModularPipelineFilter: (modularPipelineIDs, enabled) => { + dispatch(toggleModularPipelineFilter(modularPipelineIDs, enabled)); + }, onToggleTypeDisabled: (typeID, disabled) => { dispatch(toggleTypeDisabled(typeID, disabled)); }, diff --git a/src/components/node-list/node-list-groups.test.js b/src/components/node-list/node-list-groups.test.js index ce725c686d..ddeb6d025d 100644 --- a/src/components/node-list/node-list-groups.test.js +++ b/src/components/node-list/node-list-groups.test.js @@ -9,7 +9,7 @@ describe('NodeListGroups', () => { const mockProps = () => { const items = getGroupedNodes(mockState.animals); const types = getNodeTypes(mockState.animals); - const sections = getSections(); + const sections = getSections({ flags: { modularpipeline: true } }); const groups = getGroups({ types, items }); return { items, sections, groups }; }; diff --git a/src/components/node-list/node-list-items.js b/src/components/node-list/node-list-items.js index ca44871da9..49ff45721f 100644 --- a/src/components/node-list/node-list-items.js +++ b/src/components/node-list/node-list-items.js @@ -65,7 +65,6 @@ export const filterNodes = (nodes, searchValue) => { ...newNodes, [type]: filterNodesByType(type), }); - return Object.keys(nodes).reduce(filterNodeLists, {}); }; @@ -124,6 +123,49 @@ export const getFilteredTagItems = createSelector( }) ); +/** + * Return filtered/highlighted modular pipelines + * @param {object} modularPipelines List of modular pipelines + * @param {string} searchValue Search term + * @return {object} Grouped modular pipelines + */ +export const getFilteredModularPipelines = createSelector( + [(state) => state.modularPipelines, (state) => state.searchValue], + (modularPipelines, searchValue) => + highlightMatch( + filterNodes({ modularPipeline: modularPipelines }, searchValue), + searchValue + ) +); + +/** + * Return filtered/highlighted modular pipeline list items + * @param {object} filteredModularPipelines List of filtered modularPipelines + * @param {object} modularPipelinesEnabled Map of enabled modularPipelines + * @return {array} Node list items + */ +export const getFilteredModularPipelineItems = createSelector( + [getFilteredModularPipelines, (state) => state.modularPipelinesEnabled], + (filteredModularPipelines, modularPipelinesEnabled) => ({ + modularPipeline: filteredModularPipelines.modularPipeline.map( + (modularPipeline) => ({ + ...modularPipeline, + type: 'modularPipeline', + visibleIcon: IndicatorIcon, + invisibleIcon: IndicatorOffIcon, + active: false, + selected: false, + faded: false, + visible: true, + disabled: false, + unset: + typeof modularPipelinesEnabled[modularPipeline.id] === 'undefined', + checked: modularPipelinesEnabled[modularPipeline.id] === true, + }) + ), + }) +); + /** * Compares items for sorting in groups first * by enabled status (by tag) and then alphabeticaly (by name) @@ -153,7 +195,10 @@ export const getFilteredNodeItems = createSelector( result[type] = filteredNodes[type] .map((node) => { const checked = !node.disabled_node; - const disabled = node.disabled_tag || node.disabled_type; + const disabled = + node.disabled_tag || + node.disabled_type || + node.disabled_modularPipeline; return { ...node, visibleIcon: VisibleIcon, @@ -176,13 +221,17 @@ export const getFilteredNodeItems = createSelector( /** * Get formatted list of sections + * @param {boolean} flag value of modularpipeline flag * @return {array} List of sections */ -export const getSections = createSelector(() => - Object.keys(sidebar).map((name) => ({ - name, - types: Object.values(sidebar[name]), - })) + +export const getSections = createSelector( + (state) => sidebar(state.flags.modularpipeline), + (sidebarObject) => + Object.keys(sidebarObject).map((name) => ({ + name, + types: Object.values(sidebarObject[name]), + })) ); /** @@ -210,6 +259,14 @@ export const createGroup = (itemType, itemsOfType = []) => { visibleIcon: group.allChecked ? IndicatorIcon : IndicatorPartialIcon, invisibleIcon: IndicatorOffIcon, }); + } else if (itemType.id === 'modularPipeline') { + Object.assign(group, { + name: 'Modular Pipelines', + kind: 'filter', + checked: !group.allUnset, + visibleIcon: group.allChecked ? IndicatorIcon : IndicatorPartialIcon, + invisibleIcon: IndicatorOffIcon, + }); } else { Object.assign(group, { name: itemType.name, @@ -232,7 +289,7 @@ export const getGroups = createSelector( [(state) => state.types, (state) => state.items], (nodeTypes, items) => { const groups = {}; - const itemTypes = [...nodeTypes, { id: 'tag' }]; + const itemTypes = [...nodeTypes, { id: 'tag' }, { id: 'modularPipeline' }]; for (const itemType of itemTypes) { groups[itemType.id] = createGroup(itemType, items[itemType.id]); } @@ -241,17 +298,19 @@ export const getGroups = createSelector( ); /** - * Returns filtered/highlighted tag and node list items - * @param {object} filteredTags List of filtered tags - * @param {object} tagsEnabled Map of enabled tags - * @return {array} Node list items + * Returns filtered/highlighted items for nodes, tags and modular pipelines + * @param {object} filteredNodeItems List of filtered nodes + * @param {object} filteredTagItems List of filtered tags + * @param {object} filteredModularPipelinesItems List of filtered modularPipelines + * @return {array} final list of all filtered items from the three filtered item sets */ export const getFilteredItems = createSelector( - [getFilteredNodeItems, getFilteredTagItems], - (filteredNodeItems, filteredTagItems) => { + [getFilteredNodeItems, getFilteredTagItems, getFilteredModularPipelineItems], + (filteredNodeItems, filteredTagItems, filteredModularPipelineItems) => { return { ...filteredTagItems, ...filteredNodeItems, + ...filteredModularPipelineItems, }; } ); diff --git a/src/components/node-list/node-list-items.test.js b/src/components/node-list/node-list-items.test.js index 403fdc19b3..5b43c5a9cc 100644 --- a/src/components/node-list/node-list-items.test.js +++ b/src/components/node-list/node-list-items.test.js @@ -14,6 +14,7 @@ import { mockState } from '../../utils/state.mock'; import { getGroupedNodes } from '../../selectors/nodes'; import { getNodeTypes } from '../../selectors/node-types'; import { getTagData } from '../../selectors/tags'; +import { getModularPipelineData } from '../../selectors/modular-pipelines'; const ungroupNodes = (groupedNodes) => Object.keys(groupedNodes).reduce( @@ -118,7 +119,7 @@ describe('node-list-selectors', () => { }); describe('getSections', () => { - const sections = getSections(); + const sections = getSections({ flags: { modularpipeline: false } }); const section = expect.arrayContaining([ expect.objectContaining({ @@ -138,7 +139,9 @@ describe('node-list-selectors', () => { const filteredItems = getFilteredItems({ nodes: getGroupedNodes(mockState.animals), tags: getTagData(mockState.animals), + modularPipelines: getModularPipelineData(mockState.animals), tagsEnabled: {}, + modularPipelinesEnabled: {}, nodeSelected: {}, searchValue, }); @@ -160,10 +163,11 @@ describe('node-list-selectors', () => { ]); it('filters expected number of items', () => { - expect(filteredItems.task).toHaveLength(2); - expect(filteredItems.data).toHaveLength(6); - expect(filteredItems.parameters).toHaveLength(2); + expect(filteredItems.task).toHaveLength(3); + expect(filteredItems.data).toHaveLength(10); + expect(filteredItems.parameters).toHaveLength(4); expect(filteredItems.tag).toHaveLength(2); + expect(filteredItems.modularPipeline).toHaveLength(3); }); it('returns items for each type in the correct format', () => { @@ -180,10 +184,13 @@ describe('node-list-selectors', () => { describe('getGroups', () => { const types = getNodeTypes(mockState.animals); + const items = getFilteredItems({ nodes: getGroupedNodes(mockState.animals), tags: getTagData(mockState.animals), + modularPipelines: getModularPipelineData(mockState.animals), tagsEnabled: {}, + modularPipelinesEnabled: {}, nodeSelected: {}, searchValue: '', }); diff --git a/src/components/node-list/node-list-row-list.js b/src/components/node-list/node-list-row-list.js index c5ac53cad5..6a425ce04d 100644 --- a/src/components/node-list/node-list-row-list.js +++ b/src/components/node-list/node-list-row-list.js @@ -3,6 +3,20 @@ import modifiers from '../../utils/modifiers'; import NodeListRow, { nodeListRowHeight } from './node-list-row'; import LazyList from '../lazy-list'; +// Modify display of labels for modular pipelines to show nested relationship. +// Note: This label indentation could be subject to further design changes. +const getItemLabel = (item) => { + if (item.type === 'modularPipeline') { + // parse depth of modular pipeline from namespace(i.e id) + const levels = item.id.match(/\./g) ? item.id.match(/\./g).length : 0; + const layer = levels ? '・' : ''; + const whiteSpace = '    '; + + return whiteSpace.repeat(levels) + layer + item.highlightedLabel; + } + return item.highlightedLabel; +}; + const NodeRowList = ({ items = [], group, @@ -54,7 +68,7 @@ const NodeRowList = ({ key={item.id} id={item.id} kind={group.kind} - label={item.highlightedLabel} + label={getItemLabel(item)} name={item.name} type={item.type} active={item.active} diff --git a/src/components/node-list/node-list.test.js b/src/components/node-list/node-list.test.js index 313401b554..6d8fb867aa 100644 --- a/src/components/node-list/node-list.test.js +++ b/src/components/node-list/node-list.test.js @@ -234,9 +234,9 @@ describe('NodeList', () => { ['Elephant', true], ['Giraffe', true], ['Horse', true], + ['Nested.weasel', true], ['Pig', true], ['Sheep', true], - ['Weasel', true], ['Parameters', true], ['Params:rabbit', true], ]); @@ -247,25 +247,32 @@ describe('NodeList', () => { changeRows(wrapper, ['Medium'], true); expect(elements(wrapper)).toEqual([ - // Nodes (enabled) + // Tasks (enabled) ['shark', true], - // Nodes (disabled) + // Tasks (disabled) ['salmon', false], ['trout', false], + ['tuna', false], // Datasets (enabled) ['Bear', true], ['Cat', true], ['Elephant', true], ['Giraffe', true], + ['Nested.weasel', true], ['Pig', true], - ['Weasel', true], // Datasets (disabled) ['Dog', false], ['Horse', false], + ['Pipeline1.data Science.dolphin', false], + ['Pipeline1.data Science.sheep', false], + ['Pipeline2.data Science.pig', false], + ['Pipeline2.data Science.sheep', false], + ['Pipeline2.data Science.whale', false], ['Sheep', false], - ['Whale', false], // Parameters ['Parameters', false], + ['Params:pipeline100.data Science.plankton', false], + ['Params:pipeline2.data Science.plankton', false], ['Params:rabbit', false], ]); }); @@ -303,7 +310,7 @@ describe('NodeList', () => { expect(partialIcon(wrapper)).toHaveLength(1); // All tags selected - changeRows(wrapper, ['Large', 'Medium', 'Small'], true); + changeRows(wrapper, ['Medium', 'Small'], true); expect(partialIcon(wrapper)).toHaveLength(0); }); @@ -376,6 +383,7 @@ describe('NodeList', () => { disabled_node: expect.any(Boolean), disabled_tag: expect.any(Boolean), disabled_type: expect.any(Boolean), + disabled_modularPipeline: expect.any(Boolean), id: expect.any(String), name: expect.any(String), type: expect.any(String), @@ -389,8 +397,11 @@ describe('NodeList', () => { task: nodeList, }), nodeSelected: expect.any(Object), - sections: expect.any(Array), types: expect.any(Array), + modularPipelines: expect.any(Object), + modularPipelinesEnabled: expect.any(Object), + modularPipelineFlag: expect.any(Boolean), + sections: expect.any(Object), }); expect(mapStateToProps(mockState.animals)).toEqual(expectedResult); }); diff --git a/src/config.js b/src/config.js index 818dbe5e56..8e0e83245c 100644 --- a/src/config.js +++ b/src/config.js @@ -36,15 +36,37 @@ export const flags = { default: true, icon: '🐳', }, -}; - -export const sidebar = { - Categories: { - Tags: 'tag', - }, - Elements: { - Nodes: 'task', - Datasets: 'data', - Parameters: 'parameters', + modularpipeline: { + description: 'enables modular pipeline features', + default: false, + icon: '⛓️', }, }; + +/** + * returns the sidebar config object + * @param {string} modularPipelineFlag the modular pipeline flag + */ +export const sidebar = (modularPipelineFlag) => + modularPipelineFlag + ? { + Categories: { + Tags: 'tag', + ModularPipelines: 'modularPipeline', + }, + Elements: { + Nodes: 'task', + Datasets: 'data', + Parameters: 'parameters', + }, + } + : { + Categories: { + Tags: 'tag', + }, + Elements: { + Nodes: 'task', + Datasets: 'data', + Parameters: 'parameters', + }, + }; diff --git a/src/reducers/index.js b/src/reducers/index.js index c9654db543..b891b114d8 100644 --- a/src/reducers/index.js +++ b/src/reducers/index.js @@ -7,6 +7,7 @@ import node from './nodes'; import nodeType from './node-type'; import pipeline from './pipeline'; import tag from './tags'; +import modularPipeline from './modular-pipelines'; import visible from './visible'; import { RESET_DATA, @@ -58,11 +59,11 @@ const combinedReducer = combineReducers({ nodeType, pipeline, tag, + modularPipeline, visible, // These props don't have any actions associated with them dataSource: createReducer(null), edge: createReducer({}), - modularPipeline: createReducer({}), // These props have very simple non-nested actions chartSize: createReducer({}, UPDATE_CHART_SIZE, 'chartSize'), zoom: createReducer({}, UPDATE_ZOOM, 'zoom'), diff --git a/src/reducers/modular-pipelines.js b/src/reducers/modular-pipelines.js new file mode 100644 index 0000000000..64a950f4eb --- /dev/null +++ b/src/reducers/modular-pipelines.js @@ -0,0 +1,46 @@ +import { + TOGGLE_MODULAR_PIPELINE_ACTIVE, + TOGGLE_MODULAR_PIPELINE_FILTER, +} from '../actions/modular-pipelines'; + +function modularPipelineReducer(modularPipelineState = {}, action) { + const updateState = (newState) => + Object.assign({}, modularPipelineState, newState); + + /** + * Batch update tags from an array of tag IDs + * @param {string} key Tag action value prop + */ + const batchChanges = (key) => + action.modularPipelineIDs.reduce((result, modularPipelineID) => { + result[modularPipelineID] = action[key]; + return result; + }, {}); + + switch (action.type) { + case TOGGLE_MODULAR_PIPELINE_ACTIVE: { + return updateState({ + active: Object.assign( + {}, + modularPipelineState.active, + batchChanges('active') + ), + }); + } + + case TOGGLE_MODULAR_PIPELINE_FILTER: { + return updateState({ + enabled: Object.assign( + {}, + modularPipelineState.enabled, + batchChanges('enabled') + ), + }); + } + + default: + return modularPipelineState; + } +} + +export default modularPipelineReducer; diff --git a/src/selectors/disabled.js b/src/selectors/disabled.js index 421972c096..ef13144d86 100644 --- a/src/selectors/disabled.js +++ b/src/selectors/disabled.js @@ -2,12 +2,15 @@ import { createSelector } from 'reselect'; import { arrayToObject } from '../utils'; import { getNodeDisabledPipeline, getPipelineNodeIDs } from './pipeline'; import { getTagCount } from './tags'; +import { getModularPipelineCount } from './modular-pipelines'; const getNodeIDs = (state) => state.node.ids; const getNodeDisabledNode = (state) => state.node.disabled; const getNodeTags = (state) => state.node.tags; +const getNodeModularPipelines = (state) => state.node.modularPipelines; const getNodeType = (state) => state.node.type; const getTagEnabled = (state) => state.tag.enabled; +const getModularPipelineEnabled = (state) => state.modularPipeline.enabled; const getNodeTypeDisabled = (state) => state.nodeType.disabled; const getEdgeIDs = (state) => state.edge.ids; const getEdgeSources = (state) => state.edge.sources; @@ -35,13 +38,44 @@ export const getNodeDisabledTag = createSelector( ); /** - * Set disabled status if the node is specifically hidden, and/or via a tag/view/type + * Calculate whether nodes should be disabled based on their modular pipelines + */ +export const getNodeDisabledModularPipeline = createSelector( + [ + getNodeIDs, + getModularPipelineEnabled, + getModularPipelineCount, + getNodeModularPipelines, + ], + ( + nodeIDs, + modularPipelineEnabled, + modularPipelineCount, + nodeModularPipelines + ) => + arrayToObject(nodeIDs, (nodeID) => { + if (modularPipelineCount.enabled === 0) { + return false; + } + if (nodeModularPipelines[nodeID].length) { + // Hide task nodes that don't have at least one modular pipeline filter enabled + return !nodeModularPipelines[nodeID].some( + (modularPipeline) => modularPipelineEnabled[modularPipeline] + ); + } + return true; + }) +); + +/** + * Set disabled status if the node is specifically hidden, and/or via a tag/view/type/modularPipeline */ export const getNodeDisabled = createSelector( [ getNodeIDs, getNodeDisabledNode, getNodeDisabledTag, + getNodeDisabledModularPipeline, getNodeDisabledPipeline, getNodeType, getNodeTypeDisabled, @@ -50,6 +84,7 @@ export const getNodeDisabled = createSelector( nodeIDs, nodeDisabledNode, nodeDisabledTag, + nodeDisabledModularPipeline, nodeDisabledPipeline, nodeType, typeDisabled @@ -58,6 +93,7 @@ export const getNodeDisabled = createSelector( [ nodeDisabledNode[id], nodeDisabledTag[id], + nodeDisabledModularPipeline[id], nodeDisabledPipeline[id], typeDisabled[nodeType[id]], ].some(Boolean) diff --git a/src/selectors/linked-nodes.test.js b/src/selectors/linked-nodes.test.js index 8de4eeff32..b274a6b054 100644 --- a/src/selectors/linked-nodes.test.js +++ b/src/selectors/linked-nodes.test.js @@ -15,12 +15,13 @@ describe('getLinkedNodes function', () => { describe('should return true for ancestor/descendant nodes', () => { test.each([ - ['whale', '1769e230'], - ['horse', '091b5035'], - ['sheep', '6525f2e6'], - ['cat', '9d989e8d'], + ['salmon', '443cf06a'], ['dog', 'e4951252'], + ['params:rabbit', 'c38d4c6a'], ['parameters', 'f1f1425b'], + ['cat', '9d989e8d'], + ['sheep', '6525f2e6'], + ['horse', '091b5035'], ])('node %s should be true', (name, id) => { expect(linkedNodes[id]).toBe(true); }); diff --git a/src/selectors/modular-pipelines.js b/src/selectors/modular-pipelines.js new file mode 100644 index 0000000000..c15237da9b --- /dev/null +++ b/src/selectors/modular-pipelines.js @@ -0,0 +1,34 @@ +import { createSelector } from 'reselect'; +import { getPipelineModularPipelineIDs } from './pipeline'; + +const getModularPipelineIDs = (state) => state.modularPipeline.ids; +const getModularPipelineName = (state) => state.modularPipeline.name; +const getModularPipelineEnabled = (state) => state.modularPipeline.enabled; + +/** + * Retrieve the formatted list of modular pipeline filters + */ +export const getModularPipelineData = createSelector( + [getModularPipelineIDs, getModularPipelineName, getModularPipelineEnabled], + (modularPipelineIDs, modularPipelineName, modularPipelineEnabled) => + modularPipelineIDs + .slice() + .sort() + .map((id) => ({ + id, + name: modularPipelineName[id], + enabled: Boolean(modularPipelineEnabled[id]), + })) +); + +/** + * Get the total and enabled number of modular pipelines + */ +export const getModularPipelineCount = createSelector( + [getPipelineModularPipelineIDs, getModularPipelineEnabled], + (modularPipelineIDs, modularPipelineEnabled) => ({ + total: modularPipelineIDs.length, + enabled: modularPipelineIDs.filter((id) => modularPipelineEnabled[id]) + .length, + }) +); diff --git a/src/selectors/nodes.js b/src/selectors/nodes.js index 0d9ec87ba2..a003194b90 100644 --- a/src/selectors/nodes.js +++ b/src/selectors/nodes.js @@ -6,6 +6,7 @@ import { getNodeDisabled, getNodeDisabledTag, getVisibleNodeIDs, + getNodeDisabledModularPipeline, } from './disabled'; import { getNodeRank } from './ranks'; @@ -13,10 +14,12 @@ const getNodeName = (state) => state.node.name; const getNodeFullName = (state) => state.node.fullName; const getNodeDisabledNode = (state) => state.node.disabled; const getNodeTags = (state) => state.node.tags; +const getNodeModularPipelines = (state) => state.node.modularPipelines; const getNodeType = (state) => state.node.type; const getNodeLayer = (state) => state.node.layer; const getHoveredNode = (state) => state.node.hovered; const getTagActive = (state) => state.tag.active; +const getModularPipelineActive = (state) => state.modularPipeline.active; const getTextLabels = (state) => state.textLabels; const getFontLoaded = (state) => state.fontLoaded; const getNodeTypeDisabled = (state) => state.nodeType.disabled; @@ -35,17 +38,36 @@ export const getGraphNodes = createSelector( ); /** - * Set active status if the node is specifically highlighted, and/or via an associated tag + * Set active status if the node is specifically highlighted, and/or via an associated tag or modular pipeline */ export const getNodeActive = createSelector( - [getPipelineNodeIDs, getHoveredNode, getNodeTags, getTagActive], - (nodeIDs, hoveredNode, nodeTags, tagActive) => + [ + getPipelineNodeIDs, + getHoveredNode, + getNodeTags, + getNodeModularPipelines, + getTagActive, + getModularPipelineActive, + ], + ( + nodeIDs, + hoveredNode, + nodeTags, + nodeModularPipelines, + tagActive, + modularPipelineActive + ) => arrayToObject(nodeIDs, (nodeID) => { if (nodeID === hoveredNode) { return true; } const activeViaTag = nodeTags[nodeID].some((tag) => tagActive[tag]); - return Boolean(activeViaTag); + const activeViaModularPipeline = + nodeModularPipelines[nodeID] && + nodeModularPipelines[nodeID].some( + (modularPipeline) => modularPipelineActive[modularPipeline] + ); + return Boolean(activeViaTag) || Boolean(activeViaModularPipeline); }) ); @@ -72,6 +94,7 @@ export const getNodeData = createSelector( getNodeDisabled, getNodeDisabledNode, getNodeDisabledTag, + getNodeDisabledModularPipeline, getNodeTypeDisabled, ], ( @@ -81,6 +104,7 @@ export const getNodeData = createSelector( nodeDisabled, nodeDisabledNode, nodeDisabledTag, + nodeDisabledModularPipeline, typeDisabled ) => nodeIDs @@ -96,6 +120,7 @@ export const getNodeData = createSelector( disabled: nodeDisabled[id], disabled_node: Boolean(nodeDisabledNode[id]), disabled_tag: nodeDisabledTag[id], + disabled_modularPipeline: nodeDisabledModularPipeline[id], disabled_type: Boolean(typeDisabled[nodeType[id]]), })) ); diff --git a/src/selectors/pipeline.js b/src/selectors/pipeline.js index ccff049a46..71f563cc44 100644 --- a/src/selectors/pipeline.js +++ b/src/selectors/pipeline.js @@ -5,7 +5,9 @@ const getNodeIDs = (state) => state.node.ids; const getNodePipelines = (state) => state.node.pipelines; const getActivePipeline = (state) => state.pipeline.active; const getNodeTags = (state) => state.node.tags; +const getNodeModularPipelines = (state) => state.node.modularPipelines; const getDataSource = (state) => state.dataSource; +const getModularPipelineIDs = (state) => state.modularPipeline.ids; /** * Calculate whether nodes should be disabled based on their tags @@ -49,3 +51,24 @@ export const getPipelineTagIDs = createSelector( return Object.keys(visibleTags); } ); + +/** + * Get IDs of modular pipelines used in the current pipeline + */ +export const getPipelineModularPipelineIDs = createSelector( + [getPipelineNodeIDs, getNodeModularPipelines, getModularPipelineIDs], + (nodeIDs, nodeModularPipelines, modularPipelineIDs) => { + const visibleModularPipelines = {}; + // check if pipeline contains defined modular pipelines + if (modularPipelineIDs.length > 0) { + nodeIDs.forEach((nodeID) => { + nodeModularPipelines[nodeID].forEach((modularPipelineID) => { + if (!visibleModularPipelines[modularPipelineID]) { + visibleModularPipelines[modularPipelineID] = true; + } + }); + }); + } + return Object.keys(visibleModularPipelines); + } +); diff --git a/src/store/normalize-data.js b/src/store/normalize-data.js index 8208999544..261999bd36 100644 --- a/src/store/normalize-data.js +++ b/src/store/normalize-data.js @@ -12,6 +12,8 @@ export const createInitialPipelineState = () => ({ modularPipeline: { ids: [], name: {}, + enabled: {}, + active: {}, }, node: { ids: [], @@ -142,7 +144,7 @@ const addNode = (state) => (node) => { state.node.parameters[id] = node.parameters; state.node.filepath[id] = node.filepath; state.node.datasetType[id] = node.datasetType; - state.node.modularPipelines[id] = node.modular_pipelines; + state.node.modularPipelines[id] = node.modular_pipelines || []; }; /** diff --git a/src/utils/data/animals.mock.json b/src/utils/data/animals.mock.json index bf51d3d854..42f83a0dc8 100644 --- a/src/utils/data/animals.mock.json +++ b/src/utils/data/animals.mock.json @@ -1,82 +1,69 @@ { "edges": [ + { "source": "9d989e8d", "target": "15586b7a" }, + { "source": "518ed240", "target": "15586b7a" }, + { "source": "0ae9e4de", "target": "15586b7a" }, + { "source": "09f5edeb", "target": "15586b7a" }, + { "source": "15586b7a", "target": "2cd4ba93" }, + { "source": "15586b7a", "target": "fac8f1a3" }, + { "source": "e4951252", "target": "443cf06a" }, + { "source": "c38d4c6a", "target": "443cf06a" }, + { "source": "f1f1425b", "target": "443cf06a" }, + { "source": "9d989e8d", "target": "443cf06a" }, + { "source": "443cf06a", "target": "6525f2e6" }, + { "source": "443cf06a", "target": "091b5035" }, + { "source": "735bde08", "target": "2ce32881" }, + { "source": "46734c62", "target": "2ce32881" }, + { "source": "9d8a4d91", "target": "2ce32881" }, + { "source": "2ce32881", "target": "5f7e0e20" }, + { "source": "a761759c", "target": "c8c182ec" }, + { "source": "24e06541", "target": "c8c182ec" }, + { "source": "c8c182ec", "target": "0049a504" } + ], + "layers": ["raw", "feature", "model output", "primary"], + "modular_pipelines": [ { - "source": "e4951252", - "target": "443cf06a" - }, - { - "source": "c38d4c6a", - "target": "443cf06a" - }, - { - "source": "f1f1425b", - "target": "443cf06a" - }, - { - "source": "9d989e8d", - "target": "443cf06a" - }, - { - "source": "443cf06a", - "target": "6525f2e6" - }, - { - "source": "443cf06a", - "target": "091b5035" - }, - { - "source": "9d989e8d", - "target": "f42dab68" - }, - { - "source": "85c4cf64", - "target": "f42dab68" - }, - { - "source": "0ae9e4de", - "target": "f42dab68" - }, - { - "source": "09f5edeb", - "target": "f42dab68" + "id": "nested", + "name": "Nested" }, { - "source": "f42dab68", - "target": "2cd4ba93" + "id": "pipeline1", + "name": "Pipeline1" }, { - "source": "f42dab68", - "target": "fac8f1a3" + "id": "pipeline1.data_engineering", + "name": "Data Engineering" }, { - "source": "2cd4ba93", - "target": "e27376a9" + "id": "pipeline1.data_science", + "name": "Data Science" }, { - "source": "6525f2e6", - "target": "e27376a9" + "id": "pipeline2", + "name": "Pipeline2" }, { - "source": "e27376a9", - "target": "1769e230" + "id": "pipeline2.data_science", + "name": "Data Science" } ], - "layers": ["raw", "feature", "primary", "model output"], "nodes": [ { - "full_name": "salmon", - "id": "443cf06a", - "name": "salmon", + "full_name": "shark", + "id": "15586b7a", + "modular_pipelines": ["pipeline1", "pipeline1.data_engineering"], + "name": "shark", "pipelines": ["de", "__default__"], - "tags": ["small"], + "tags": ["large", "medium"], "type": "task" }, { - "full_name": "shark", - "id": "f42dab68", - "name": "shark", + "full_name": "salmon", + "id": "443cf06a", + "modular_pipelines": [], + "name": "salmon", "pipelines": ["de", "__default__"], - "tags": ["large", "medium"], + "tags": ["small"], "type": "task" }, { @@ -84,6 +71,7 @@ "id": "09f5edeb", "layer": "raw", "name": "Bear", + "modular_pipelines": [], "pipelines": ["de", "__default__"], "tags": ["large", "medium"], "type": "data" @@ -93,6 +81,7 @@ "id": "9d989e8d", "layer": "raw", "name": "Cat", + "modular_pipelines": [], "pipelines": ["de", "__default__"], "tags": ["large", "medium", "small"], "type": "data" @@ -102,6 +91,7 @@ "id": "e4951252", "layer": null, "name": "Dog", + "modular_pipelines": [], "pipelines": ["de", "__default__"], "tags": ["small"], "type": "data" @@ -111,6 +101,7 @@ "id": "0ae9e4de", "layer": "raw", "name": "Elephant", + "modular_pipelines": [], "pipelines": ["de", "__default__"], "tags": ["large", "medium"], "type": "data" @@ -120,6 +111,7 @@ "id": "fac8f1a3", "layer": "model output", "name": "Giraffe", + "modular_pipelines": [], "pipelines": ["de", "__default__"], "tags": ["large", "medium"], "type": "data" @@ -129,15 +121,27 @@ "id": "091b5035", "layer": "model output", "name": "Horse", + "modular_pipelines": [], "pipelines": ["de", "__default__"], "tags": ["small"], "type": "data" }, + { + "full_name": "nested.weasel", + "id": "518ed240", + "layer": "raw", + "modular_pipelines": ["nested"], + "name": "Nested.weasel", + "pipelines": ["de", "__default__"], + "tags": ["large", "medium"], + "type": "data" + }, { "full_name": "parameters", "id": "f1f1425b", "layer": null, "name": "Parameters", + "modular_pipelines": [], "pipelines": ["de", "__default__"], "tags": ["small"], "type": "parameters" @@ -147,6 +151,7 @@ "id": "c38d4c6a", "layer": null, "name": "Params:rabbit", + "modular_pipelines": [], "pipelines": ["de", "__default__"], "tags": ["small"], "type": "parameters" @@ -156,7 +161,8 @@ "id": "2cd4ba93", "layer": "feature", "name": "Pig", - "pipelines": ["de", "ds", "__default__"], + "modular_pipelines": [], + "pipelines": ["de", "__default__"], "tags": ["large", "medium"], "type": "data" }, @@ -165,32 +171,95 @@ "id": "6525f2e6", "layer": "primary", "name": "Sheep", - "pipelines": ["de", "ds", "__default__"], + "modular_pipelines": [], + "pipelines": ["de", "__default__"], "tags": ["small"], "type": "data" }, { - "full_name": "weasel", - "id": "85c4cf64", - "layer": "raw", - "name": "Weasel", - "pipelines": ["de", "__default__"], - "tags": ["large", "medium"], + "full_name": "tuna", + "id": "2ce32881", + "modular_pipelines": ["pipeline1", "pipeline1.data_science"], + "name": "tuna", + "pipelines": ["pre_ds", "__default__"], + "tags": [], + "type": "task" + }, + { + "full_name": "params:pipeline100.data_science.plankton", + "id": "9d8a4d91", + "layer": null, + "modular_pipelines": [], + "name": "Params:pipeline100.data Science.plankton", + "pipelines": ["pre_ds", "__default__"], + "tags": [], + "type": "parameters" + }, + { + "full_name": "params:pipeline2.data_science.plankton", + "id": "46734c62", + "layer": null, + "modular_pipelines": ["pipeline2", "pipeline2.data_science"], + "name": "Params:pipeline2.data Science.plankton", + "pipelines": ["pre_ds", "__default__"], + "tags": [], + "type": "parameters" + }, + { + "full_name": "pipeline1.data_science.dolphin", + "id": "5f7e0e20", + "layer": null, + "name": "Pipeline1.data Science.dolphin", + "pipelines": ["pre_ds", "__default__"], + "modular_pipelines": ["pipeline1", "pipeline1.data_science"], + "tags": [], + "type": "data" + }, + { + "full_name": "pipeline1.data_science.sheep", + "id": "735bde08", + "layer": null, + "modular_pipelines": ["pipeline1", "pipeline1.data_science"], + "name": "Pipeline1.data Science.sheep", + "pipelines": ["pre_ds", "__default__"], + "tags": [], "type": "data" }, { "full_name": "trout", - "id": "e27376a9", + "id": "c8c182ec", + "modular_pipelines": ["pipeline2", "pipeline2.data_science"], "name": "trout", "pipelines": ["ds", "__default__"], "tags": [], "type": "task" }, { - "full_name": "whale", - "id": "1769e230", + "full_name": "pipeline2.data_science.pig", + "id": "a761759c", + "layer": null, + "modular_pipelines": ["pipeline2", "pipeline2.data_science"], + "name": "Pipeline2.data Science.pig", + "pipelines": ["ds", "__default__"], + "tags": [], + "type": "data" + }, + { + "full_name": "pipeline2.data_science.sheep", + "id": "24e06541", + "layer": null, + "modular_pipelines": ["pipeline2", "pipeline2.data_science"], + "name": "Pipeline2.data Science.sheep", + "pipelines": ["ds", "__default__"], + "tags": [], + "type": "data" + }, + { + "full_name": "pipeline2.data_science.whale", + "id": "0049a504", "layer": "model output", - "name": "Whale", + "name": "Pipeline2.data Science.whale", + "modular_pipelines": ["pipeline2", "pipeline2.data_science"], "pipelines": ["ds", "__default__"], "tags": [], "type": "data" @@ -201,6 +270,10 @@ "id": "de", "name": "De" }, + { + "id": "pre_ds", + "name": "Pre Ds" + }, { "id": "ds", "name": "Ds" diff --git a/src/utils/data/demo.mock.json b/src/utils/data/demo.mock.json index 9c915ee2b1..f7d7b9930c 100644 --- a/src/utils/data/demo.mock.json +++ b/src/utils/data/demo.mock.json @@ -14,6 +14,7 @@ "name": "Data science" } ], + "modular_pipelines": [], "layers": [ "Raw", "Intermediate", @@ -334,7 +335,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Raw", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "load_raw_country_data", @@ -343,7 +345,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Raw", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "load_raw_shopper_spend_data", @@ -352,7 +355,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Raw", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "preprocess_intermediate_interaction_data", @@ -361,7 +365,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Intermediate", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "preprocess_intermediate_country_data", @@ -370,7 +375,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Intermediate", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "preprocess_intermediate_shopper_spend_data", @@ -379,7 +385,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Intermediate", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "create_shopper_spend_features", @@ -388,7 +395,8 @@ "tags": ["feature_engineering", "data_engineering"], "layer": "Feature", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "create_shopper_churn_features", @@ -397,7 +405,8 @@ "tags": ["feature_engineering", "data_engineering"], "layer": "Feature", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "prepare_vendor_input", @@ -406,7 +415,8 @@ "tags": ["feature_engineering", "data_engineering"], "layer": "Intermediate", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "prepare_crm_input", @@ -415,7 +425,8 @@ "tags": ["feature_engineering", "data_engineering"], "layer": "Intermediate", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "predictive_sales_model", @@ -424,7 +435,8 @@ "tags": ["model_training", "data_science"], "layer": "Model Input", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "predictive_engagement_model", @@ -433,7 +445,8 @@ "tags": ["model_training", "data_science"], "layer": "Model Input", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "sales_model_explainable_ai", @@ -442,7 +455,8 @@ "tags": ["model_explanation", "data_science"], "layer": "Model Input", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "engagement_model_explainable_ai", @@ -451,7 +465,8 @@ "tags": ["model_explanation", "data_science"], "layer": "Models", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "perform_digital_analysis", @@ -460,7 +475,8 @@ "tags": ["model_training", "data_science"], "layer": "Model Input", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "engagement_recommendation_engine", @@ -469,7 +485,8 @@ "tags": ["model_training", "data_science"], "layer": "Models", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "sales_model_performance_monitoring", @@ -478,7 +495,8 @@ "tags": ["model_performance_monitoring", "data_science"], "layer": "Reporting", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "engagement_model_performance_monitoring", @@ -487,7 +505,8 @@ "tags": ["model_performance_monitoring", "data_science"], "layer": "Reporting", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "multi-channel_optimisation", @@ -496,7 +515,8 @@ "tags": ["optimisation", "data_science"], "layer": "Models", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "content_optimisation", @@ -505,7 +525,8 @@ "tags": ["optimisation", "data_science"], "layer": "Models", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "segment_journeys", @@ -514,7 +535,8 @@ "tags": ["optimisation", "data_science"], "layer": "Model Output", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "generate_dashboard_inputs", @@ -523,7 +545,8 @@ "tags": ["reporting", "data_science"], "layer": "Reporting", "pipelines": ["__default__", "ds"], - "type": "task" + "type": "task", + "modular_pipelines": [] }, { "full_name": "interaction_raw", @@ -532,7 +555,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Raw", "pipelines": ["__default__", "ds"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "interaction_intermediate", @@ -541,7 +565,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Intermediate", "pipelines": ["__default__", "ds"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "country_raw", @@ -550,7 +575,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Raw", "pipelines": ["__default__", "ds"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "country_intermediate", @@ -559,7 +585,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Intermediate", "pipelines": ["__default__", "ds"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "shopper_spend_raw", @@ -568,7 +595,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Raw", "pipelines": ["__default__", "ds"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "shopper_spend_intermediate", @@ -577,7 +605,8 @@ "tags": ["data_engineering", "preprocessing"], "layer": "Intermediate", "pipelines": ["__default__", "ds"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "interaction_primary", @@ -586,7 +615,8 @@ "tags": ["feature_engineering", "data_engineering", "preprocessing"], "layer": "Primary", "pipelines": ["__default__", "ds"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "country_primary", @@ -595,7 +625,8 @@ "tags": ["feature_engineering", "data_engineering", "preprocessing"], "layer": "Primary", "pipelines": ["__default__", "ds"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "shopper_spend_primary", @@ -604,7 +635,8 @@ "tags": ["feature_engineering", "data_engineering", "preprocessing"], "layer": "Primary", "pipelines": ["__default__", "ds"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "crm_predictions", @@ -613,7 +645,8 @@ "tags": ["feature_engineering", "data_engineering"], "layer": "Primary", "pipelines": ["__default__", "ds"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "shopper_spend_features", @@ -628,7 +661,8 @@ ], "layer": "Feature", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "shopper_churn_features", @@ -643,7 +677,8 @@ ], "layer": "Feature", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "vendor_main", @@ -652,7 +687,8 @@ "tags": ["feature_engineering", "data_engineering"], "layer": "Raw", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "salesforce_crm", @@ -661,7 +697,8 @@ "tags": ["feature_engineering", "data_engineering"], "layer": "Raw", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "vendor_predictions", @@ -675,7 +712,8 @@ ], "layer": "Primary", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "salesforce_accounts", @@ -684,7 +722,8 @@ "tags": ["feature_engineering", "data_engineering"], "layer": "Raw", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "params:_sales_model", @@ -693,7 +732,8 @@ "tags": ["data_science", "model_training"], "layer": "Model Input", "pipelines": ["__default__", "de"], - "type": "parameters" + "type": "parameters", + "modular_pipelines": [] }, { "full_name": "sales_validation_results", @@ -706,7 +746,8 @@ ], "layer": "Model Output", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "sales_trained_model", @@ -715,7 +756,8 @@ "tags": ["model_explanation", "data_science", "model_training"], "layer": "Model Input", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "params:_engagement_model", @@ -724,7 +766,8 @@ "tags": ["data_science", "model_training"], "layer": "Model Input", "pipelines": ["__default__", "de"], - "type": "parameters" + "type": "parameters", + "modular_pipelines": [] }, { "full_name": "engagement_validation_results", @@ -737,7 +780,8 @@ ], "layer": "Model Output", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "engagement_trained_model", @@ -746,7 +790,8 @@ "tags": ["model_explanation", "data_science", "model_training"], "layer": "Models", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "sales_model_explanations", @@ -760,7 +805,8 @@ ], "layer": "Model Output", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "engagement_model_explanations", @@ -774,7 +820,8 @@ ], "layer": "Model Output", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "params:_optimisation", @@ -783,7 +830,8 @@ "tags": ["data_science", "model_training", "optimisation"], "layer": "Model Input", "pipelines": ["__default__", "de"], - "type": "parameters" + "type": "parameters", + "modular_pipelines": [] }, { "full_name": "digital_analysis", @@ -792,7 +840,8 @@ "tags": ["data_science", "model_training", "optimisation"], "layer": "Model Input", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "engagement_recommendations", @@ -801,7 +850,8 @@ "tags": ["data_science", "model_training"], "layer": "Model Output", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "action_cost_table", @@ -810,7 +860,8 @@ "tags": ["data_science", "optimisation"], "layer": "Raw", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "multi-channel_resolutions", @@ -819,7 +870,8 @@ "tags": ["reporting", "data_science", "optimisation"], "layer": "Model Output", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "content_resolutions", @@ -828,7 +880,8 @@ "tags": ["reporting", "data_science", "optimisation"], "layer": "Model Output", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "segment_journeys_allocations", @@ -837,7 +890,8 @@ "tags": ["reporting", "data_science", "optimisation"], "layer": "Model Output", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "upselling_readiness_dashboard_input", @@ -846,7 +900,8 @@ "tags": ["reporting", "data_science"], "layer": "Reporting", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "lead_scoring_dashboard_input", @@ -855,7 +910,8 @@ "tags": ["reporting", "data_science"], "layer": "Reporting", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "lifetime_value_prediction_dashboard_input", @@ -864,7 +920,8 @@ "tags": ["reporting", "data_science"], "layer": "Reporting", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "digital_sales_dashboard_input", @@ -873,7 +930,8 @@ "tags": ["reporting", "data_science"], "layer": "Reporting", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] }, { "full_name": "vendor_sales_dashboard_input", @@ -882,7 +940,8 @@ "tags": ["reporting", "data_science"], "layer": "Reporting", "pipelines": ["__default__", "de"], - "type": "data" + "type": "data", + "modular_pipelines": [] } ], "tags": [ diff --git a/src/utils/graph/constraints.js b/src/utils/graph/constraints.js index c49bef4d9f..a637bbdc84 100644 --- a/src/utils/graph/constraints.js +++ b/src/utils/graph/constraints.js @@ -71,7 +71,7 @@ export const crossingConstraint = { const resolveSource = strength * ((edgeA.sourceNode.x - edgeB.sourceNode.x - separationA) / separationA); - + const resolveTarget = strength * ((edgeA.targetNode.x - edgeB.targetNode.x - separationB) / separationB); diff --git a/src/utils/graph/graph.test.js b/src/utils/graph/graph.test.js index be8d9df480..5e07d106c8 100644 --- a/src/utils/graph/graph.test.js +++ b/src/utils/graph/graph.test.js @@ -510,7 +510,7 @@ describe('constraints', () => { (edgeA.sourceNode.x - edgeB.sourceNode.x) * (edgeA.targetNode.x - edgeB.targetNode.x) < 0; - + // Expect edges to be initially crossing expect(isCrossing(testEdgeA, testEdgeB)).toBe(true); diff --git a/src/utils/random-data.js b/src/utils/random-data.js index 760b4cd832..123af33031 100644 --- a/src/utils/random-data.js +++ b/src/utils/random-data.js @@ -17,6 +17,8 @@ const MAX_TAG_COUNT = 20; const PARAMETERS_FREQUENCY = 0.2; const MIN_PIPELINES_COUNT = 2; const MAX_PIPELINES_COUNT = 15; +const MIN_MODULAR_PIPELINES_COUNT = 2; +const MAX_MODULAR_PIPELINES_COUNT = 15; const LAYERS = [ 'Raw', 'Intermediate', @@ -33,6 +35,7 @@ class Pipeline { constructor() { this.utils = randomUtils(); this.pipelines = this.generatePipelines(); + this.modularPipelines = this.generateModularPipelines(); this.rankCount = this.getRankCount(); this.rankLayers = this.getRankLayers(); this.tags = this.generateTags(); @@ -44,7 +47,7 @@ class Pipeline { } /** - * Create the pipelines aray + * Create the pipelines array * @returns {number} Rank count total */ generatePipelines() { @@ -59,6 +62,24 @@ class Pipeline { return pipelines.filter(unique); } + /** + * Create the modular pipelines array + * @returns {number} Rank count total + */ + generateModularPipelines() { + const modularPipelines = ['Data Science']; + const pipelineCount = this.utils.randomNumberBetween( + MIN_MODULAR_PIPELINES_COUNT, + MAX_MODULAR_PIPELINES_COUNT + ); + for (let i = 1; i < pipelineCount; i++) { + modularPipelines.push( + this.utils.getRandomName(this.utils.randomNumber(4), ' ') + ); + } + return modularPipelines.filter(unique); + } + /** * Get the number of ranks (i.e. horizontal bands) * Odd ranks are data, even are task @@ -171,6 +192,7 @@ class Pipeline { rank: initialRank, layer: layer, pipelines: this.getNodePipelines(), + modular_pipelines: this.getNodeModularPipelines(), tags: this.getRandomTags(), _sources: [], _targets: [], @@ -235,6 +257,19 @@ class Pipeline { }, []); } + /** + * Create a list of the modular pipelines that the node will be included in + * @returns {array} Node pipelines + */ + getNodeModularPipelines() { + return this.modularPipelines.reduce((modularPipelines, id, i) => { + if (i === 0 || this.utils.randomIndex(2)) { + return modularPipelines.concat(id); + } + return modularPipelines; + }, []); + } + /** * Select a random number of tags from the list of tags * @returns {array} List of tags @@ -428,6 +463,10 @@ class Pipeline { layers: LAYERS, nodes: this.nodes, pipelines: this.pipelines.map((name) => ({ id: name, name })), + modular_pipelines: this.modularPipelines.map((name) => ({ + id: name, + name, + })), tags: this.tags, }; }