diff --git a/cmflib/cmf.py b/cmflib/cmf.py index 6df08508..fa3e8cb6 100644 --- a/cmflib/cmf.py +++ b/cmflib/cmf.py @@ -141,6 +141,7 @@ def __init__( if is_server: Cmf.__get_neo4j_server_config() if graph is True: + Cmf.__load_neo4j_params() self.driver = graph_wrapper.GraphDriver( Cmf.__neo4j_uri, Cmf.__neo4j_user, Cmf.__neo4j_password ) @@ -148,6 +149,16 @@ def __init__( pipeline_name, self.parent_context.id, custom_properties ) + @staticmethod + def __load_neo4j_params(): + cmf_config = os.environ.get("CONFIG_FILE", ".cmfconfig") + if os.path.exists(cmf_config): + attr_dict = CmfConfig.read_config(cmf_config) + __neo4j_uri = attr_dict.get("neo4j-uri", "") + __neo4j_password = attr_dict.get("neo4j-password", "") + __neo4j_user = attr_dict.get("neo4j-user", "") + + @staticmethod def __get_neo4j_server_config(): Cmf.__neo4j_uri = os.getenv('NEO4J_URI', "") diff --git a/cmflib/cmfquery.py b/cmflib/cmfquery.py index 088d66cc..335d2326 100644 --- a/cmflib/cmfquery.py +++ b/cmflib/cmfquery.py @@ -178,8 +178,8 @@ def _transform_to_dataframe( ) d = CmfQuery._copy( source=node.custom_properties, - target=d#, # renaming custom_properties with prefix custom_properties has impact in server GUI - #key_mapper=_PrefixMapper("custom_properties_", on_collision=_KeyMapper.OnCollision.RESOLVE), + target=d, # renaming custom_properties with prefix custom_properties has impact in server GUI + key_mapper=_PrefixMapper("custom_properties_", on_collision=_KeyMapper.OnCollision.RESOLVE), ) return pd.DataFrame( diff --git a/cmflib/commands/metadata/__init__.py b/cmflib/commands/metadata/__init__.py index ae23dbe4..67764795 100644 --- a/cmflib/commands/metadata/__init__.py +++ b/cmflib/commands/metadata/__init__.py @@ -16,19 +16,19 @@ import argparse -from cmflib.commands.metadata import push, pull +from cmflib.commands.metadata import push, pull, export from cmflib.cli.utils import * -SUB_COMMANDS = [push, pull] +SUB_COMMANDS = [push, pull, export] # This parser adds positional arguments to the main parser def add_parser(subparsers, parent_parser): - METADATA_HELP = "Command for metadata pull/push." + METADATA_HELP = "Command for metadata pull, push and export." metadata_parser = subparsers.add_parser( "metadata", parents=[parent_parser], - description="Command pulls or pushes metadata on to cmf-server.", + description="Command pulls or pushes metadata on to cmf-server and exports the local mlmd into json.", help=METADATA_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) diff --git a/cmflib/commands/metadata/export.py b/cmflib/commands/metadata/export.py new file mode 100644 index 00000000..b53096fa --- /dev/null +++ b/cmflib/commands/metadata/export.py @@ -0,0 +1,110 @@ +### +# Copyright (2022) Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +### + +#!/usr/bin/env python3 +import argparse +import json +import os +from cmflib import cmfquery +from cmflib.cli.command import CmdBase + + +# This class export local mlmd data to a json file +class CmdMetadataExport(CmdBase): + def run(self): + + current_directory = os.getcwd() + full_path_to_dump = "" + data = "" + mlmd_data = "" + + mlmd_file_name = "./mlmd" + + # checks if mlmd filepath is given + if self.args.file_name: + mlmd_file_name = self.args.file_name + current_directory = os.path.dirname(self.args.file_name) + + # checks if mlmd file is present in current directory or given directory + if not os.path.exists(mlmd_file_name): + return f"ERROR: {mlmd_file_name} doesn't exists in the {current_directory}." + + + # setting directory where mlmd file will be dumped + if self.args.json_file_name: + if not os.path.isdir(self.args.json_file_name): + temp = os.path.dirname(self.args.json_file_name) + if temp != "": + current_directory = temp + if os.path.exists(current_directory): + full_path_to_dump = self.args.json_file_name + else: + return f"{current_directory} doesn't exists." + else: + return "Provide path with file name." + else: + full_path_to_dump = os.getcwd() + f"/{self.args.pipeline_name}.json" + + # initialising cmfquery class + query = cmfquery.CmfQuery(mlmd_file_name) + + # check if pipeline exists in mlmd + pipeline = query.get_pipeline_id(self.args.pipeline_name) + + if pipeline > 0: + # pulling data from local mlmd file + json_payload = query.dumptojson(self.args.pipeline_name,None) + + # write metadata into json file + with open(full_path_to_dump, 'w') as f: + f.write(json.dumps(json.loads(json_payload),indent=2)) + return f"SUCCESS: metadata successfully exported in {full_path_to_dump}." + else: + return f"{self.args.pipeline_name} doesn't exists in {mlmd_file_name}!!" + + +def add_parser(subparsers, parent_parser): + PULL_HELP = "Exports local mlmd's metadata to a json file." + + parser = subparsers.add_parser( + "export", + parents=[parent_parser], + description="Export local mlmd's metadata in json format to a json file.", + help=PULL_HELP, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + required_arguments = parser.add_argument_group("required arguments") + + required_arguments.add_argument( + "-p", + "--pipeline_name", + required=True, + help="Specify Pipeline name.", + metavar="", + ) + + parser.add_argument( + "-j", + "--json_file_name", + help="Specify json file name with full path.", + metavar="", + ) + + parser.add_argument( + "-f", "--file_name", help="Specify mlmd file name.", metavar="" + ) + + parser.set_defaults(func=CmdMetadataExport) diff --git a/ui/src/pages/artifacts/index.jsx b/ui/src/pages/artifacts/index.jsx index 75edf1a7..4f80c051 100644 --- a/ui/src/pages/artifacts/index.jsx +++ b/ui/src/pages/artifacts/index.jsx @@ -147,7 +147,7 @@ const Artifacts = () => { )}
- {selectedPipeline !== null && selectedArtifactType !== null && ( + {selectedPipeline !== null && selectedArtifactType !== null && artifacts!== null && ( )}