SDK - DSL - Stopped adding mlpipeline artifacts to every compiled tem…

…plate Fixes kubeflow#1421 Fixes kubeflow#1422
Ark-kun · Sep 5, 2019 · bdd43a5 · bdd43a5
1 parent da8a9b0
commit bdd43a5
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 6 deletions.
diff --git a/sdk/python/kfp/compiler/_op_to_template.py b/sdk/python/kfp/compiler/_op_to_template.py
@@ -187,8 +187,6 @@ def _op_to_template(op: BaseOp):
     if isinstance(op, dsl.ContainerOp):
         # default output artifacts
         output_artifact_paths = OrderedDict(op.output_artifact_paths)
-        output_artifact_paths.setdefault('mlpipeline-ui-metadata', '/mlpipeline-ui-metadata.json')
-        output_artifact_paths.setdefault('mlpipeline-metrics', '/mlpipeline-metrics.json')
 
         output_artifacts = [
              K8sHelper.convert_k8s_obj_to_json(
@@ -200,10 +198,6 @@ def _op_to_template(op: BaseOp):
             for name, path in output_artifact_paths.items()
         ]
 
-        for output_artifact in output_artifacts:
-            if output_artifact['name'] in ['mlpipeline-ui-metadata', 'mlpipeline-metrics']:
-                output_artifact['optional'] = True
-
         # workflow template
         template = {
             'name': processed_op.name,

diff --git a/sdk/python/kfp/dsl/_container_op.py b/sdk/python/kfp/dsl/_container_op.py
@@ -1017,6 +1017,8 @@ def __init__(
 
         input_artifact_paths = {}
         artifact_arguments = {}
+        file_outputs = dict(file_outputs or {}) # Making a copy
+        output_artifact_paths = dict(output_artifact_paths or {}) # Making a copy
 
         def resolve_artifact_argument(artarg):
             from ..components._components import _generate_input_file_name
@@ -1076,6 +1078,14 @@ def _decorated(*args, **kwargs):
                 # only proxy public callables
                 setattr(self, attr_to_proxy, _proxy(attr_to_proxy))
 
+        # Special handling for the mlpipeline-ui-metadata and mlpipeline-metrics outputs that should always be saved as artifacts
+        # TODO: Remove when outputs are always saved as artifacts
+        for output_name, path in dict(file_outputs).items():
+            normalized_output_name = re.sub('[^a-zA-Z0-9]', '-', output_name.lower())
+            if normalized_output_name in ['mlpipeline-ui-metadata', 'mlpipeline-metrics']:
+                output_artifact_paths[normalized_output_name] = path
+                del file_outputs[output_name]
+
         # attributes specific to `ContainerOp`
         self.input_artifact_paths = input_artifact_paths
         self.artifact_arguments = artifact_arguments