From 5360f3fcabff5b3e87329e69dda95e320eb90c5f Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Thu, 5 Sep 2019 17:56:57 -0700 Subject: [PATCH] SDK - Compiler - Stopped adding mlpipeline artifacts to every compiled template (#2046) * Explicitly added mlpipeline outputs to the components that actually produce them * Updated samples * SDK - DSL - Stopped adding mlpipeline artifacts to every compiled template Fixes https://github.com/kubeflow/pipelines/issues/1421 Fixes https://github.com/kubeflow/pipelines/issues/1422 * Updated the Lighweight sample * Updated the compiler tests * Fixed the lightweight sample * Reverted the change to one contrib/samples/openvino The sample will still work fine as it is now. I'll add the change to that file as a separate PR. --- components/arena/docker/arena_launcher.py | 4 -- components/dataflow/predict/component.yaml | 2 + components/dataflow/tfma/component.yaml | 2 + components/gcp/bigquery/query/component.yaml | 3 ++ .../gcp/dataflow/launch_python/component.yaml | 5 ++- .../dataflow/launch_template/component.yaml | 5 ++- .../dataproc/create_cluster/component.yaml | 5 ++- .../dataproc/submit_hadoop_job/component.yaml | 5 ++- .../dataproc/submit_hive_job/component.yaml | 5 ++- .../dataproc/submit_pig_job/component.yaml | 5 ++- .../submit_pyspark_job/component.yaml | 5 ++- .../dataproc/submit_spark_job/component.yaml | 5 ++- .../submit_sparksql_job/component.yaml | 5 ++- .../ml_engine/batch_predict/component.yaml | 5 ++- .../gcp/ml_engine/deploy/component.yaml | 3 ++ components/gcp/ml_engine/train/component.yaml | 3 ++ components/kubeflow/dnntrainer/component.yaml | 2 + .../launcher/kubeflow_tfjob_launcher_op.py | 5 ++- .../local/confusion_matrix/component.yaml | 13 +++--- components/local/roc/component.yaml | 12 +++--- .../Local Development Quickstart.ipynb | 3 +- ...ghtweight Python components - basics.ipynb | 13 ++---- .../tfx_cab_classification.py | 18 ++++---- .../xgboost_training_cm.py | 11 +++-- sdk/python/kfp/compiler/_op_to_template.py | 11 ++--- sdk/python/kfp/dsl/_container_op.py | 10 +++++ sdk/python/tests/compiler/compiler_tests.py | 11 +---- .../tests/compiler/testdata/add_pod_env.yaml | 8 ---- .../compiler/testdata/artifact_location.py | 11 ++++- .../compiler/testdata/artifact_location.yaml | 43 +++---------------- sdk/python/tests/compiler/testdata/basic.yaml | 23 ---------- sdk/python/tests/compiler/testdata/coin.yaml | 30 ------------- .../tests/compiler/testdata/compose.yaml | 22 ---------- .../compiler/testdata/default_value.yaml | 15 ------- .../compiler/testdata/imagepullsecrets.yaml | 7 --- .../testdata/input_artifact_raw_value.yaml | 32 -------------- .../compiler/testdata/param_op_transform.yaml | 8 ---- .../testdata/param_substitutions.yaml | 8 ---- .../compiler/testdata/pipelineparams.yaml | 15 ------- .../testdata/preemptible_tpu_gpu.yaml | 7 --- .../compiler/testdata/recursive_do_while.yaml | 37 ---------------- .../compiler/testdata/recursive_while.yaml | 39 +---------------- .../compiler/testdata/resourceop_basic.yaml | 8 ---- .../tests/compiler/testdata/sidecar.yaml | 17 +------- .../tests/compiler/testdata/timeout.yaml | 16 ------- .../tests/compiler/testdata/tolerations.yaml | 7 --- .../tests/compiler/testdata/volume.yaml | 15 ------- .../testdata/volume_snapshotop_rokurl.yaml | 24 ----------- .../volume_snapshotop_sequential.yaml | 32 -------------- .../compiler/testdata/volumeop_basic.yaml | 8 ---- .../tests/compiler/testdata/volumeop_dag.yaml | 24 ----------- .../compiler/testdata/volumeop_parallel.yaml | 24 ----------- .../testdata/volumeop_sequential.yaml | 24 ----------- .../compiler/testdata/withitem_basic.yaml | 24 ----------- .../compiler/testdata/withitem_nested.yaml | 32 -------------- 55 files changed, 126 insertions(+), 610 deletions(-) diff --git a/components/arena/docker/arena_launcher.py b/components/arena/docker/arena_launcher.py index 400ede554c8..d127e915fd7 100644 --- a/components/arena/docker/arena_launcher.py +++ b/components/arena/docker/arena_launcher.py @@ -108,10 +108,6 @@ def main(argv=None): submit_job(command) - #with open('/mlpipeline-ui-metadata.json', 'w') as f: - # json.dump(metadata, f) - - succ = True # wait for job done diff --git a/components/dataflow/predict/component.yaml b/components/dataflow/predict/component.yaml index 6fae9083fb3..6e829bcb893 100644 --- a/components/dataflow/predict/component.yaml +++ b/components/dataflow/predict/component.yaml @@ -13,6 +13,7 @@ inputs: - {name: Predictions dir, type: GCSPath, description: 'GCS or local directory.'} #Will contain prediction_results-* and schema.json files; TODO: Split outputs and replace dir with single file # type: {GCSPath: {path_type: Directory}} outputs: - {name: Predictions dir, type: GCSPath, description: 'GCS or local directory.'} #Will contain prediction_results-* and schema.json files; TODO: Split outputs and replace dir with single file # type: {GCSPath: {path_type: Directory}} + - {name: MLPipeline UI metadata, type: UI metadata} implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tf-predict:151c5349f13bea9d626c988563c04c0a86210c21 @@ -29,3 +30,4 @@ implementation: ] fileOutputs: Predictions dir: /output.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/dataflow/tfma/component.yaml b/components/dataflow/tfma/component.yaml index bc865d6d4c0..978e97c7e2e 100644 --- a/components/dataflow/tfma/component.yaml +++ b/components/dataflow/tfma/component.yaml @@ -15,6 +15,7 @@ inputs: - {name: Analysis results dir, type: GCSPath, description: GCS or local directory where the analysis results should be written.} # type: {GCSPath: {path_type: Directory}} outputs: - {name: Analysis results dir, type: GCSPath, description: GCS or local directory where the analysis results should were written.} # type: {GCSPath: {path_type: Directory}} +- {name: MLPipeline UI metadata, type: UI metadata} implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-dataflow-tfma:151c5349f13bea9d626c988563c04c0a86210c21 @@ -30,3 +31,4 @@ implementation: ] fileOutputs: Analysis results dir: /output.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/bigquery/query/component.yaml b/components/gcp/bigquery/query/component.yaml index 07f29549b40..d1945e73fbe 100644 --- a/components/gcp/bigquery/query/component.yaml +++ b/components/gcp/bigquery/query/component.yaml @@ -55,6 +55,8 @@ outputs: - name: output_gcs_path description: 'The path to the Cloud Storage bucket containing the query output in CSV format.' type: GCSPath + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -72,3 +74,4 @@ implementation: KFP_POD_NAME: "{{pod.name}}" fileOutputs: output_gcs_path: /tmp/kfp/output/bigquery/query-output-path.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataflow/launch_python/component.yaml b/components/gcp/dataflow/launch_python/component.yaml index 920408aa48d..2c4b90fa468 100644 --- a/components/gcp/dataflow/launch_python/component.yaml +++ b/components/gcp/dataflow/launch_python/component.yaml @@ -49,6 +49,8 @@ outputs: - name: job_id description: 'The id of the created dataflow job.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -64,4 +66,5 @@ implementation: env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: - job_id: /tmp/kfp/output/dataflow/job_id.txt \ No newline at end of file + job_id: /tmp/kfp/output/dataflow/job_id.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataflow/launch_template/component.yaml b/components/gcp/dataflow/launch_template/component.yaml index 09f29d0fab9..75ad52be9e7 100644 --- a/components/gcp/dataflow/launch_template/component.yaml +++ b/components/gcp/dataflow/launch_template/component.yaml @@ -59,6 +59,8 @@ outputs: - name: job_id description: 'The id of the created dataflow job.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -75,4 +77,5 @@ implementation: env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: - job_id: /tmp/kfp/output/dataflow/job_id.txt \ No newline at end of file + job_id: /tmp/kfp/output/dataflow/job_id.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/create_cluster/component.yaml b/components/gcp/dataproc/create_cluster/component.yaml index c5cad047204..d5383dcbf5c 100644 --- a/components/gcp/dataproc/create_cluster/component.yaml +++ b/components/gcp/dataproc/create_cluster/component.yaml @@ -66,6 +66,8 @@ outputs: - name: cluster_name description: 'The cluster name of the created cluster.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -84,4 +86,5 @@ implementation: env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: - cluster_name: /tmp/kfp/output/dataproc/cluster_name.txt \ No newline at end of file + cluster_name: /tmp/kfp/output/dataproc/cluster_name.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_hadoop_job/component.yaml b/components/gcp/dataproc/submit_hadoop_job/component.yaml index 8e73c7a4d7a..8dfcc3d56d8 100644 --- a/components/gcp/dataproc/submit_hadoop_job/component.yaml +++ b/components/gcp/dataproc/submit_hadoop_job/component.yaml @@ -76,6 +76,8 @@ outputs: - name: job_id description: 'The ID of the created job.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -94,4 +96,5 @@ implementation: env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: - job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file + job_id: /tmp/kfp/output/dataproc/job_id.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_hive_job/component.yaml b/components/gcp/dataproc/submit_hive_job/component.yaml index 3140de61005..c0ef297fc7e 100644 --- a/components/gcp/dataproc/submit_hive_job/component.yaml +++ b/components/gcp/dataproc/submit_hive_job/component.yaml @@ -71,6 +71,8 @@ outputs: - name: job_id description: 'The ID of the created job.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -89,4 +91,5 @@ implementation: env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: - job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file + job_id: /tmp/kfp/output/dataproc/job_id.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_pig_job/component.yaml b/components/gcp/dataproc/submit_pig_job/component.yaml index 52e314afe1c..76ac296156f 100644 --- a/components/gcp/dataproc/submit_pig_job/component.yaml +++ b/components/gcp/dataproc/submit_pig_job/component.yaml @@ -71,6 +71,8 @@ outputs: - name: job_id description: 'The ID of the created job.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -89,4 +91,5 @@ implementation: env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: - job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file + job_id: /tmp/kfp/output/dataproc/job_id.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_pyspark_job/component.yaml b/components/gcp/dataproc/submit_pyspark_job/component.yaml index 30036ea18c0..995693ba1f7 100644 --- a/components/gcp/dataproc/submit_pyspark_job/component.yaml +++ b/components/gcp/dataproc/submit_pyspark_job/component.yaml @@ -65,6 +65,8 @@ outputs: - name: job_id description: 'The ID of the created job.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -82,4 +84,5 @@ implementation: env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: - job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file + job_id: /tmp/kfp/output/dataproc/job_id.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_spark_job/component.yaml b/components/gcp/dataproc/submit_spark_job/component.yaml index d585b97e0af..ea94cbffdc1 100644 --- a/components/gcp/dataproc/submit_spark_job/component.yaml +++ b/components/gcp/dataproc/submit_spark_job/component.yaml @@ -72,6 +72,8 @@ outputs: - name: job_id description: 'The ID of the created job.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -90,4 +92,5 @@ implementation: env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: - job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file + job_id: /tmp/kfp/output/dataproc/job_id.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/dataproc/submit_sparksql_job/component.yaml b/components/gcp/dataproc/submit_sparksql_job/component.yaml index 119e8592d35..f91151ef75c 100644 --- a/components/gcp/dataproc/submit_sparksql_job/component.yaml +++ b/components/gcp/dataproc/submit_sparksql_job/component.yaml @@ -71,6 +71,8 @@ outputs: - name: job_id description: 'The ID of the created job.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -89,4 +91,5 @@ implementation: env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: - job_id: /tmp/kfp/output/dataproc/job_id.txt \ No newline at end of file + job_id: /tmp/kfp/output/dataproc/job_id.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/ml_engine/batch_predict/component.yaml b/components/gcp/ml_engine/batch_predict/component.yaml index 0a3bbf1fb36..de3ca6f166f 100644 --- a/components/gcp/ml_engine/batch_predict/component.yaml +++ b/components/gcp/ml_engine/batch_predict/component.yaml @@ -65,6 +65,8 @@ outputs: - name: job_id description: 'The ID of the created job.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -84,4 +86,5 @@ implementation: env: KFP_POD_NAME: "{{pod.name}}" fileOutputs: - job_id: /tmp/kfp/output/ml_engine/job_id.txt \ No newline at end of file + job_id: /tmp/kfp/output/ml_engine/job_id.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/ml_engine/deploy/component.yaml b/components/gcp/ml_engine/deploy/component.yaml index 5b4280cefdb..e6d07f943f8 100644 --- a/components/gcp/ml_engine/deploy/component.yaml +++ b/components/gcp/ml_engine/deploy/component.yaml @@ -91,6 +91,8 @@ outputs: - name: version_name description: 'The name of the deployed version.' type: String + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -112,3 +114,4 @@ implementation: model_uri: /tmp/kfp/output/ml_engine/model_uri.txt model_name: /tmp/kfp/output/ml_engine/model_name.txt version_name: /tmp/kfp/output/ml_engine/version_name.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/gcp/ml_engine/train/component.yaml b/components/gcp/ml_engine/train/component.yaml index 0f6dd63adcc..bcdca54ab5b 100644 --- a/components/gcp/ml_engine/train/component.yaml +++ b/components/gcp/ml_engine/train/component.yaml @@ -99,6 +99,8 @@ outputs: The output path in Cloud Storage of the trainning job, which contains the trained model files. type: GCSPath + - name: MLPipeline UI metadata + type: UI metadata implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-gcp:151c5349f13bea9d626c988563c04c0a86210c21 @@ -123,3 +125,4 @@ implementation: fileOutputs: job_id: /tmp/kfp/output/ml_engine/job_id.txt job_dir: /tmp/kfp/output/ml_engine/job_dir.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/kubeflow/dnntrainer/component.yaml b/components/kubeflow/dnntrainer/component.yaml index 70b5a25a9ee..53dbd3f38e9 100644 --- a/components/kubeflow/dnntrainer/component.yaml +++ b/components/kubeflow/dnntrainer/component.yaml @@ -13,6 +13,7 @@ inputs: - {name: Training output dir, type: GCSPath, description: 'GCS or local directory.'} # type: {GCSPath: {path_type: Directory}} outputs: - {name: Training output dir, type: GCSPath, description: 'GCS or local directory.'} # type: {GCSPath: {path_type: Directory}} + - {name: MLPipeline UI metadata, type: UI metadata} implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-kubeflow-tf-trainer:151c5349f13bea9d626c988563c04c0a86210c21 @@ -31,3 +32,4 @@ implementation: ] fileOutputs: Training output dir: /output.txt + MLPipeline UI metadata: /mlpipeline-ui-metadata.json diff --git a/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py b/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py index be10cdc8115..512fe769480 100644 --- a/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py +++ b/components/kubeflow/launcher/kubeflow_tfjob_launcher_op.py @@ -27,5 +27,8 @@ def kubeflow_tfjob_launcher_op(container_image, command, number_of_workers: int, '--ui-metadata-type', 'tensorboard', '--', ] + command, - file_outputs = {'train': '/output.txt'} + file_outputs = {'train': '/output.txt'}, + output_artifact_paths={ + 'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json', + }, ) diff --git a/components/local/confusion_matrix/component.yaml b/components/local/confusion_matrix/component.yaml index 12d21d15e10..2de25cf5a29 100644 --- a/components/local/confusion_matrix/component.yaml +++ b/components/local/confusion_matrix/component.yaml @@ -4,9 +4,9 @@ inputs: - {name: Predictions, type: GCSPath, description: 'GCS path of prediction file pattern.'} # type: {GCSPath: {data_type: CSV}} - {name: Target lambda, type: String, default: '', description: 'Text of Python lambda function which computes target value. For example, "lambda x: x[''a''] + x[''b'']". If not set, the input must include a "target" column.'} - {name: Output dir, type: GCSPath, description: 'GCS path of the output directory.'} # type: {GCSPath: {path_type: Directory}} -#outputs: -# - {name: UI metadata, type: UI metadata} -# - {name: Metrics, type: Metrics} +outputs: + - {name: MLPipeline UI metadata, type: UI metadata} + - {name: MLPipeline Metrics, type: Metrics} implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:151c5349f13bea9d626c988563c04c0a86210c21 @@ -16,7 +16,6 @@ implementation: --target_lambda, {inputValue: Target lambda}, --output, {inputValue: Output dir}, ] -#Argo deletes the source files as soon as it uploads them to the artifact store. Trying to output the same files as parameter outputs fails since the source files are already deleted. -# fileOutputs: -# UI metadata: /mlpipeline-ui-metadata.json -# Metrics: /mlpipeline-metrics.json + fileOutputs: + MLPipeline UI metadata: /mlpipeline-ui-metadata.json + MLPipeline Metrics: /mlpipeline-metrics.json diff --git a/components/local/roc/component.yaml b/components/local/roc/component.yaml index 4110ca34ee0..e7bfacf505b 100644 --- a/components/local/roc/component.yaml +++ b/components/local/roc/component.yaml @@ -6,9 +6,9 @@ inputs: - {name: True score column, type: String, default: 'true', description: 'The name of the column for positive probability.'} - {name: Target lambda, type: String, default: '', description: 'Text of Python lambda function which returns boolean value indicating whether the classification result is correct.\nFor example, "lambda x: x[''a''] and x[''b'']". If missing, input must have a "target" column.'} - {name: Output dir, type: GCSPath, description: 'GCS path of the output directory.'} #TODO: Replace dir with single file # type: {GCSPath: {path_type: Directory}} -#outputs: -# - {name: UI metadata, type: UI metadata} -# - {name: Metrics, type: Metrics} +outputs: + - {name: MLPipeline UI metadata, type: UI metadata} + - {name: MLPipeline Metrics, type: Metrics} implementation: container: image: gcr.io/ml-pipeline/ml-pipeline-local-confusion-matrix:151c5349f13bea9d626c988563c04c0a86210c21 @@ -20,6 +20,6 @@ implementation: --target_lambda, {inputValue: Target lambda}, --output, {inputValue: Output dir}, ] -# fileOutputs: -# UI metadata: /mlpipeline-ui-metadata.json -# Metrics: /mlpipeline-metrics.json + fileOutputs: + MLPipeline UI metadata: /mlpipeline-ui-metadata.json + MLPipeline Metrics: /mlpipeline-metrics.json diff --git a/samples/contrib/local_development_quickstart/Local Development Quickstart.ipynb b/samples/contrib/local_development_quickstart/Local Development Quickstart.ipynb index 1dcf5ed5837..bf04f7ef1bf 100644 --- a/samples/contrib/local_development_quickstart/Local Development Quickstart.ipynb +++ b/samples/contrib/local_development_quickstart/Local Development Quickstart.ipynb @@ -508,8 +508,9 @@ " name=name,\n", " image='gcr.io/{}/listgcsblobs:latest'.format(PROJECT_ID),\n", " command=['python', '/app/app.py'],\n", + " arguments=['--bucket', bucket],\n", " file_outputs={'blobs': '/blobs.txt'},\n", - " arguments=['--bucket', bucket]\n", + " output_artifact_paths={'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json'},\n", " )\n", "\n", "def view_input_op(name, blobs):\n", diff --git a/samples/core/lightweight_component/Lightweight Python components - basics.ipynb b/samples/core/lightweight_component/Lightweight Python components - basics.ipynb index a4d8572dad0..a932d19aeab 100644 --- a/samples/core/lightweight_component/Lightweight Python components - basics.ipynb +++ b/samples/core/lightweight_component/Lightweight Python components - basics.ipynb @@ -102,7 +102,7 @@ "#Advanced function\n", "#Demonstrates imports, helper functions and multiple outputs\n", "from typing import NamedTuple\n", - "def my_divmod(dividend: float, divisor:float, output_dir:str = './') -> NamedTuple('MyDivmodOutput', [('quotient', float), ('remainder', float)]):\n", + "def my_divmod(dividend: float, divisor:float) -> NamedTuple('MyDivmodOutput', [('quotient', float), ('remainder', float), ('mlpipeline_ui_metadata', 'UI_metadata'), ('mlpipeline_metrics', 'Metrics')]):\n", " '''Divides two numbers and calculate the quotient and remainder'''\n", " #Pip installs inside a component function.\n", " #NOTE: installs should be placed right at the beginning to avoid upgrading a package\n", @@ -129,8 +129,6 @@ " 'source': 'gs://ml-pipeline-dataset/tensorboard-train',\n", " }]\n", " }\n", - " with open(output_dir + 'mlpipeline-ui-metadata.json', 'w') as f:\n", - " json.dump(metadata, f)\n", "\n", " # Exports two sample metrics:\n", " metrics = {\n", @@ -142,12 +140,9 @@ " 'numberValue': float(remainder),\n", " }]}\n", "\n", - " with file_io.FileIO(output_dir + 'mlpipeline-metrics.json', 'w') as f:\n", - " json.dump(metrics, f)\n", - "\n", " from collections import namedtuple\n", - " divmod_output = namedtuple('MyDivmodOutput', ['quotient', 'remainder'])\n", - " return divmod_output(quotient, remainder)" + " divmod_output = namedtuple('MyDivmodOutput', ['quotient', 'remainder', 'mlpipeline_ui_metadata', 'mlpipeline_metrics'])\n", + " return divmod_output(quotient, remainder, json.dumps(metadata), json.dumps(metrics))" ] }, { @@ -224,7 +219,7 @@ " \n", " #Passing a task output reference as operation arguments\n", " #For an operation with a single return value, the output reference can be accessed using `task.output` or `task.outputs['output_name']` syntax\n", - " divmod_task = divmod_op(add_task.output, b, '/')\n", + " divmod_task = divmod_op(add_task.output, b)\n", "\n", " #For an operation with a multiple return values, the output references can be accessed using `task.outputs['output_name']` syntax\n", " result_task = add_op(divmod_task.outputs['quotient'], c)" diff --git a/samples/core/tfx_cab_classification/tfx_cab_classification.py b/samples/core/tfx_cab_classification/tfx_cab_classification.py index db47320b5fb..3dd905f2245 100755 --- a/samples/core/tfx_cab_classification/tfx_cab_classification.py +++ b/samples/core/tfx_cab_classification/tfx_cab_classification.py @@ -24,12 +24,12 @@ dataflow_tf_data_validation_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/a97f1d0ad0e7b92203f35c5b0b9af3a314952e05/components/dataflow/tfdv/component.yaml') dataflow_tf_transform_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/a97f1d0ad0e7b92203f35c5b0b9af3a314952e05/components/dataflow/tft/component.yaml') -tf_train_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/a97f1d0ad0e7b92203f35c5b0b9af3a314952e05/components/kubeflow/dnntrainer/component.yaml') -dataflow_tf_model_analyze_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/a97f1d0ad0e7b92203f35c5b0b9af3a314952e05/components/dataflow/tfma/component.yaml') -dataflow_tf_predict_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/a97f1d0ad0e7b92203f35c5b0b9af3a314952e05/components/dataflow/predict/component.yaml') +tf_train_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b705631e892bd8181cabcd704e6e6385b16daf90/components/kubeflow/dnntrainer/component.yaml') +dataflow_tf_model_analyze_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b705631e892bd8181cabcd704e6e6385b16daf90/components/dataflow/tfma/component.yaml') +dataflow_tf_predict_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b705631e892bd8181cabcd704e6e6385b16daf90/components/dataflow/predict/component.yaml') -confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/a97f1d0ad0e7b92203f35c5b0b9af3a314952e05/components/local/confusion_matrix/component.yaml') -roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/a97f1d0ad0e7b92203f35c5b0b9af3a314952e05/components/local/roc/component.yaml') +confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b705631e892bd8181cabcd704e6e6385b16daf90/components/local/confusion_matrix/component.yaml') +roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b705631e892bd8181cabcd704e6e6385b16daf90/components/local/roc/component.yaml') kubeflow_deploy_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/a97f1d0ad0e7b92203f35c5b0b9af3a314952e05/components/kubeflow/deployer/component.yaml') @@ -106,7 +106,7 @@ def taxi_cab_classification( ) analysis = dataflow_tf_model_analyze_op( - model=training.output, + model=training.outputs['training_output_dir'], evaluation_data=evaluation, schema=validation.outputs['schema'], gcp_project=project, @@ -119,20 +119,20 @@ def taxi_cab_classification( data_file_pattern=evaluation, schema=validation.outputs['schema'], target_column='tips', - model=training.output, + model=training.outputs['training_output_dir'], run_mode=mode, gcp_project=project, predictions_dir=output_template ) cm = confusion_matrix_op( - predictions=prediction.output, + predictions=prediction.outputs['predictions_dir'], target_lambda=target_lambda, output_dir=output_template ) roc = roc_op( - predictions_dir=prediction.output, + predictions_dir=prediction.outputs['predictions_dir'], target_lambda=target_class_lambda, output_dir=output_template ) diff --git a/samples/core/xgboost_training_cm/xgboost_training_cm.py b/samples/core/xgboost_training_cm/xgboost_training_cm.py index f581eae7623..fd4796db987 100755 --- a/samples/core/xgboost_training_cm/xgboost_training_cm.py +++ b/samples/core/xgboost_training_cm/xgboost_training_cm.py @@ -20,8 +20,8 @@ from kfp import dsl from kfp import gcp -confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/a97f1d0ad0e7b92203f35c5b0b9af3a314952e05/components/local/confusion_matrix/component.yaml') -roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/a97f1d0ad0e7b92203f35c5b0b9af3a314952e05/components/local/roc/component.yaml') +confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b705631e892bd8181cabcd704e6e6385b16daf90/components/local/confusion_matrix/component.yaml') +roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b705631e892bd8181cabcd704e6e6385b16daf90/components/local/roc/component.yaml') # ! Please do not forget to enable the Dataproc API in your cluster https://console.developers.google.com/apis/api/dataproc.googleapis.com/overview @@ -188,7 +188,10 @@ def dataproc_predict_op( ], file_outputs={ 'output': '/output.txt', - } + }, + output_artifact_paths={ + 'mlpipeline-ui-metadata': '/mlpipeline-ui-metadata.json', + }, ) # ======================================================================= @@ -280,4 +283,4 @@ def xgb_train_pipeline( ).apply(gcp.use_gcp_secret('user-gcp-sa')) if __name__ == '__main__': - kfp.compiler.Compiler().compile(xgb_train_pipeline, __file__ + '.zip') + kfp.compiler.Compiler().compile(xgb_train_pipeline, __file__ + '.yaml') diff --git a/sdk/python/kfp/compiler/_op_to_template.py b/sdk/python/kfp/compiler/_op_to_template.py index ef2aaed1beb..7476e148870 100644 --- a/sdk/python/kfp/compiler/_op_to_template.py +++ b/sdk/python/kfp/compiler/_op_to_template.py @@ -190,8 +190,6 @@ def _op_to_template(op: BaseOp): if isinstance(op, dsl.ContainerOp): # default output artifacts output_artifact_paths = OrderedDict(op.output_artifact_paths) - output_artifact_paths.setdefault('mlpipeline-ui-metadata', '/mlpipeline-ui-metadata.json') - output_artifact_paths.setdefault('mlpipeline-metrics', '/mlpipeline-metrics.json') output_artifacts = [ K8sHelper.convert_k8s_obj_to_json( @@ -203,10 +201,6 @@ def _op_to_template(op: BaseOp): for name, path in output_artifact_paths.items() ] - for output_artifact in output_artifacts: - if output_artifact['name'] in ['mlpipeline-ui-metadata', 'mlpipeline-metrics']: - output_artifact['optional'] = True - # workflow template template = { 'name': processed_op.name, @@ -242,8 +236,9 @@ def _op_to_template(op: BaseOp): param_outputs = processed_op.file_outputs elif isinstance(op, dsl.ResourceOp): param_outputs = processed_op.attribute_outputs - template['outputs'] = _outputs_to_json(op, processed_op.outputs, - param_outputs, output_artifacts) + outputs_dict = _outputs_to_json(op, processed_op.outputs, param_outputs, output_artifacts) + if outputs_dict: + template['outputs'] = outputs_dict # node selector if processed_op.node_selector: diff --git a/sdk/python/kfp/dsl/_container_op.py b/sdk/python/kfp/dsl/_container_op.py index 6c636be6999..86d5b9c3778 100644 --- a/sdk/python/kfp/dsl/_container_op.py +++ b/sdk/python/kfp/dsl/_container_op.py @@ -1017,6 +1017,8 @@ def __init__( input_artifact_paths = {} artifact_arguments = {} + file_outputs = dict(file_outputs or {}) # Making a copy + output_artifact_paths = dict(output_artifact_paths or {}) # Making a copy def resolve_artifact_argument(artarg): from ..components._components import _generate_input_file_name @@ -1073,6 +1075,14 @@ def _decorated(*args, **kwargs): # only proxy public callables setattr(self, attr_to_proxy, _proxy(attr_to_proxy)) + # Special handling for the mlpipeline-ui-metadata and mlpipeline-metrics outputs that should always be saved as artifacts + # TODO: Remove when outputs are always saved as artifacts + for output_name, path in dict(file_outputs).items(): + normalized_output_name = re.sub('[^a-zA-Z0-9]', '-', output_name.lower()) + if normalized_output_name in ['mlpipeline-ui-metadata', 'mlpipeline-metrics']: + output_artifact_paths[normalized_output_name] = path + del file_outputs[output_name] + # attributes specific to `ContainerOp` self.input_artifact_paths = input_artifact_paths self.artifact_arguments = artifact_arguments diff --git a/sdk/python/tests/compiler/compiler_tests.py b/sdk/python/tests/compiler/compiler_tests.py index a4e9cee628e..da4fe253d1f 100644 --- a/sdk/python/tests/compiler/compiler_tests.py +++ b/sdk/python/tests/compiler/compiler_tests.py @@ -102,15 +102,6 @@ def my_pipeline(msg1, json, kind, msg2='value2'): {'name': 'echo-merged', 'valueFrom': {'path': '/tmp/message.txt'} }], - 'artifacts': [{ - 'name': 'mlpipeline-ui-metadata', - 'path': '/mlpipeline-ui-metadata.json', - 'optional': True, - },{ - 'name': 'mlpipeline-metrics', - 'path': '/mlpipeline-metrics.json', - 'optional': True, - }] } } res_output = { @@ -699,7 +690,7 @@ def some_pipeline(): # See https://github.com/argoproj/argo/blob/5331fc02e257266a4a5887dfe6277e5a0b42e7fc/cmd/argoexec/commands/resource.go#L30 self.assertIsNone(delete_op_template.get("successCondition")) self.assertIsNone(delete_op_template.get("failureCondition")) - self.assertDictEqual(delete_op_template.get("outputs"), {}) + self.assertDictEqual(delete_op_template.get("outputs", {}), {}) def test_py_input_artifact_raw_value(self): """Test pipeline input_artifact_raw_value.""" diff --git a/sdk/python/tests/compiler/testdata/add_pod_env.yaml b/sdk/python/tests/compiler/testdata/add_pod_env.yaml index 990bd19d4f4..a45e831445a 100644 --- a/sdk/python/tests/compiler/testdata/add_pod_env.yaml +++ b/sdk/python/tests/compiler/testdata/add_pod_env.yaml @@ -30,14 +30,6 @@ spec: labels: add-pod-env: 'true' name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - dag: tasks: - name: echo diff --git a/sdk/python/tests/compiler/testdata/artifact_location.py b/sdk/python/tests/compiler/testdata/artifact_location.py index f3323fa7a48..61127544f67 100644 --- a/sdk/python/tests/compiler/testdata/artifact_location.py +++ b/sdk/python/tests/compiler/testdata/artifact_location.py @@ -38,10 +38,19 @@ def foo_pipeline(tag: str, namespace: str = "kubeflow", bucket: str = "foobar"): dsl.get_pipeline_conf().set_artifact_location(pipeline_artifact_location) # pipeline level artifact location (to minio) - op1 = dsl.ContainerOp(name='foo', image='busybox:%s' % tag) + op1 = dsl.ContainerOp( + name='foo', + image='busybox:%s' % tag, + output_artifact_paths={ + 'out_art': '/tmp/out_art.txt', + }, + ) # op level artifact location (to s3 bucket) op2 = dsl.ContainerOp(name='foo', image='busybox:%s' % tag, + output_artifact_paths={ + 'out_art': '/tmp/out_art.txt', + }, # configures artifact location artifact_location=artifact_location) diff --git a/sdk/python/tests/compiler/testdata/artifact_location.yaml b/sdk/python/tests/compiler/testdata/artifact_location.yaml index 467032ec53a..fa0bf9fa214 100644 --- a/sdk/python/tests/compiler/testdata/artifact_location.yaml +++ b/sdk/python/tests/compiler/testdata/artifact_location.yaml @@ -64,9 +64,8 @@ spec: name: foo outputs: artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json + - name: out_art + path: /tmp/out_art.txt s3: accessKeySecret: key: 'accesskey' @@ -74,21 +73,7 @@ spec: bucket: '{{inputs.parameters.bucket}}' endpoint: minio-service.{{inputs.parameters.namespace}}:9000 insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - secretKeySecret: - key: 'secretkey' - name: 'minio' - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - s3: - accessKeySecret: - key: 'accesskey' - name: 'minio' - bucket: '{{inputs.parameters.bucket}}' - endpoint: minio-service.{{inputs.parameters.namespace}}:9000 - insecure: true - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + key: runs/{{workflow.uid}}/{{pod.name}}/out_art.tgz secretKeySecret: key: 'secretkey' name: 'minio' @@ -101,24 +86,8 @@ spec: name: foo-2 outputs: artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - s3: - accessKeySecret: - key: '' - optional: true - bucket: '{{inputs.parameters.bucket}}' - endpoint: s3.amazonaws.com - insecure: false - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-ui-metadata.tgz - region: ap-southeast-1 - secretKeySecret: - key: '' - optional: true - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json + - name: out_art + path: /tmp/out_art.txt s3: accessKeySecret: key: '' @@ -126,7 +95,7 @@ spec: bucket: '{{inputs.parameters.bucket}}' endpoint: s3.amazonaws.com insecure: false - key: runs/{{workflow.uid}}/{{pod.name}}/mlpipeline-metrics.tgz + key: runs/{{workflow.uid}}/{{pod.name}}/out_art.tgz region: ap-southeast-1 secretKeySecret: key: '' diff --git a/sdk/python/tests/compiler/testdata/basic.yaml b/sdk/python/tests/compiler/testdata/basic.yaml index f94cdbfa43e..e123d601af7 100644 --- a/sdk/python/tests/compiler/testdata/basic.yaml +++ b/sdk/python/tests/compiler/testdata/basic.yaml @@ -57,14 +57,6 @@ spec: - -c image: python:3.5-jessie name: exiting - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: args: - python -c "from collections import Counter; words = Counter('{{inputs.parameters.message}}'.split()); @@ -81,13 +73,6 @@ spec: - name: message name: get-frequent outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: get-frequent-word valueFrom: @@ -115,14 +100,6 @@ spec: name: save nodeSelector: cloud.google.com/gke-accelerator: nvidia-tesla-k80 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - dag: tasks: - arguments: diff --git a/sdk/python/tests/compiler/testdata/coin.yaml b/sdk/python/tests/compiler/testdata/coin.yaml index 252357345a0..6e9d5b13a73 100644 --- a/sdk/python/tests/compiler/testdata/coin.yaml +++ b/sdk/python/tests/compiler/testdata/coin.yaml @@ -91,13 +91,6 @@ spec: image: python:alpine3.6 name: flip outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: flip-output valueFrom: @@ -112,13 +105,6 @@ spec: image: python:alpine3.6 name: flip-again outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: flip-again-output valueFrom: @@ -158,14 +144,6 @@ spec: parameters: - name: flip-again-output name: print1 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: command: - echo @@ -175,11 +153,3 @@ spec: parameters: - name: flip-again-output name: print2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true diff --git a/sdk/python/tests/compiler/testdata/compose.yaml b/sdk/python/tests/compiler/testdata/compose.yaml index b0520316d05..00c49a00878 100644 --- a/sdk/python/tests/compiler/testdata/compose.yaml +++ b/sdk/python/tests/compiler/testdata/compose.yaml @@ -37,13 +37,6 @@ spec: - name: url name: download outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: download-downloaded valueFrom: @@ -92,13 +85,6 @@ spec: - name: download-downloaded name: get-frequent outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: get-frequent-word valueFrom: @@ -116,11 +102,3 @@ spec: - name: get-frequent-word - name: outputpath name: save - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true diff --git a/sdk/python/tests/compiler/testdata/default_value.yaml b/sdk/python/tests/compiler/testdata/default_value.yaml index eed2408cf51..afbc8bdee6a 100644 --- a/sdk/python/tests/compiler/testdata/default_value.yaml +++ b/sdk/python/tests/compiler/testdata/default_value.yaml @@ -57,13 +57,6 @@ spec: - name: url name: download outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: download-downloaded valueFrom: @@ -79,11 +72,3 @@ spec: parameters: - name: download-downloaded name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true diff --git a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml index 571dedd070e..63185e81a56 100644 --- a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml +++ b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml @@ -26,13 +26,6 @@ spec: - name: message name: get-frequent outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: get-frequent-word valueFrom: diff --git a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml index e1e44ad09c2..52c3df06063 100644 --- a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml +++ b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml @@ -22,14 +22,6 @@ spec: raw: data: Constant artifact value name: component-with-inline-input-artifact - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - container: command: - cat @@ -42,14 +34,6 @@ spec: raw: data: Constant artifact value name: component-with-input-artifact - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - container: command: - cat @@ -62,14 +46,6 @@ spec: raw: data: hard-coded artifact value name: component-with-input-artifact-2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - container: command: - cat @@ -82,14 +58,6 @@ spec: raw: data: Text from a file with hard-coded artifact value name: component-with-input-artifact-3 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - dag: tasks: - name: component-with-inline-input-artifact diff --git a/sdk/python/tests/compiler/testdata/param_op_transform.yaml b/sdk/python/tests/compiler/testdata/param_op_transform.yaml index a555191abfb..b0da3751f66 100644 --- a/sdk/python/tests/compiler/testdata/param_op_transform.yaml +++ b/sdk/python/tests/compiler/testdata/param_op_transform.yaml @@ -23,14 +23,6 @@ spec: labels: param: '{{inputs.parameters.param}}' name: cop - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - dag: tasks: - arguments: diff --git a/sdk/python/tests/compiler/testdata/param_substitutions.yaml b/sdk/python/tests/compiler/testdata/param_substitutions.yaml index 3062dbc8f5a..9ae0f45f24d 100644 --- a/sdk/python/tests/compiler/testdata/param_substitutions.yaml +++ b/sdk/python/tests/compiler/testdata/param_substitutions.yaml @@ -23,14 +23,6 @@ spec: parameters: - name: create-volume-name name: cop - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - name: create-volume outputs: parameters: diff --git a/sdk/python/tests/compiler/testdata/pipelineparams.yaml b/sdk/python/tests/compiler/testdata/pipelineparams.yaml index 09d1aba9b06..ece877a7ff3 100644 --- a/sdk/python/tests/compiler/testdata/pipelineparams.yaml +++ b/sdk/python/tests/compiler/testdata/pipelineparams.yaml @@ -41,13 +41,6 @@ spec: - sh - "-c" outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: "/mlpipeline-ui-metadata.json" - optional: true - - name: mlpipeline-metrics - path: "/mlpipeline-metrics.json" - optional: true parameters: - name: download-downloaded valueFrom: @@ -71,14 +64,6 @@ spec: env: - name: MSG value: 'pipelineParams: ' - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: "/mlpipeline-ui-metadata.json" - optional: true - - name: mlpipeline-metrics - path: "/mlpipeline-metrics.json" - optional: true - name: pipelineparams inputs: parameters: diff --git a/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml b/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml index 4d2ee3ce61c..e70079ce5cd 100644 --- a/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml +++ b/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml @@ -25,13 +25,6 @@ spec: nodeSelector: cloud.google.com/gke-preemptible: 'true' outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json parameters: - name: flip-output valueFrom: diff --git a/sdk/python/tests/compiler/testdata/recursive_do_while.yaml b/sdk/python/tests/compiler/testdata/recursive_do_while.yaml index 8aaf35edaaf..1c98485fa47 100644 --- a/sdk/python/tests/compiler/testdata/recursive_do_while.yaml +++ b/sdk/python/tests/compiler/testdata/recursive_do_while.yaml @@ -33,13 +33,6 @@ spec: image: python:alpine3.6 name: flip outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: flip-output valueFrom: @@ -54,13 +47,6 @@ spec: image: python:alpine3.6 name: flip-2 outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: flip-2-output valueFrom: @@ -75,13 +61,6 @@ spec: image: python:alpine3.6 name: flip-3 outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: flip-3-output valueFrom: @@ -145,14 +124,6 @@ spec: parameters: - name: flip-output name: print - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: command: - echo @@ -162,11 +133,3 @@ spec: parameters: - name: flip-output name: print-2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true diff --git a/sdk/python/tests/compiler/testdata/recursive_while.yaml b/sdk/python/tests/compiler/testdata/recursive_while.yaml index 39d89318588..d7120b53f5b 100644 --- a/sdk/python/tests/compiler/testdata/recursive_while.yaml +++ b/sdk/python/tests/compiler/testdata/recursive_while.yaml @@ -55,13 +55,6 @@ spec: image: python:alpine3.6 name: flip outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json parameters: - name: flip-output valueFrom: @@ -76,13 +69,6 @@ spec: image: python:alpine3.6 name: flip-2 outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json parameters: - name: flip-2-output valueFrom: @@ -97,13 +83,6 @@ spec: image: python:alpine3.6 name: flip-3 outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json parameters: - name: flip-3-output valueFrom: @@ -163,14 +142,7 @@ spec: parameters: - name: flip-output name: print - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json + - container: command: - echo @@ -180,11 +152,4 @@ spec: parameters: - name: flip-output name: print-2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json + diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml index ed52222d57f..832b449b807 100644 --- a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml +++ b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml @@ -27,14 +27,6 @@ spec: parameters: - name: create-my-secret-name name: cop - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - inputs: parameters: - name: password diff --git a/sdk/python/tests/compiler/testdata/sidecar.yaml b/sdk/python/tests/compiler/testdata/sidecar.yaml index a614655fc0c..19b48aab8e5 100644 --- a/sdk/python/tests/compiler/testdata/sidecar.yaml +++ b/sdk/python/tests/compiler/testdata/sidecar.yaml @@ -23,13 +23,6 @@ spec: parameters: [] templates: - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: "/mlpipeline-ui-metadata.json" - optional: true - - name: mlpipeline-metrics - path: "/mlpipeline-metrics.json" - optional: true parameters: - name: download-downloaded valueFrom: @@ -47,15 +40,7 @@ spec: command: - sh - "-c" - - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: "/mlpipeline-ui-metadata.json" - optional: true - - name: mlpipeline-metrics - path: "/mlpipeline-metrics.json" - optional: true - name: echo + - name: echo inputs: parameters: - name: download-downloaded diff --git a/sdk/python/tests/compiler/testdata/timeout.yaml b/sdk/python/tests/compiler/testdata/timeout.yaml index 9fa6ecc5685..95895f1671d 100644 --- a/sdk/python/tests/compiler/testdata/timeout.yaml +++ b/sdk/python/tests/compiler/testdata/timeout.yaml @@ -30,14 +30,6 @@ spec: - -c image: python:alpine3.6 name: random-failure - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - container: args: - import random; import sys; exit_code = random.choice([0,1]); print(exit_code); @@ -47,11 +39,3 @@ spec: - -c image: python:alpine3.6 name: random-failure-2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json diff --git a/sdk/python/tests/compiler/testdata/tolerations.yaml b/sdk/python/tests/compiler/testdata/tolerations.yaml index c6437e088f5..c5fe2b9c23d 100644 --- a/sdk/python/tests/compiler/testdata/tolerations.yaml +++ b/sdk/python/tests/compiler/testdata/tolerations.yaml @@ -20,13 +20,6 @@ spec: parameters: [] templates: - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: "/mlpipeline-ui-metadata.json" - optional: true - - name: mlpipeline-metrics - path: "/mlpipeline-metrics.json" - optional: true parameters: - name: download-downloaded valueFrom: diff --git a/sdk/python/tests/compiler/testdata/volume.yaml b/sdk/python/tests/compiler/testdata/volume.yaml index b702f9110f7..b2703ccf4e2 100644 --- a/sdk/python/tests/compiler/testdata/volume.yaml +++ b/sdk/python/tests/compiler/testdata/volume.yaml @@ -41,13 +41,6 @@ spec: name: gcp-credentials name: download outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true parameters: - name: download-downloaded valueFrom: @@ -63,14 +56,6 @@ spec: parameters: - name: download-downloaded name: echo - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - dag: tasks: - name: download diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml index f07434d9249..3ca258e30e3 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml @@ -133,14 +133,6 @@ spec: parameters: - name: create-volume-1-name name: step1-concat - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: command: - gunzip @@ -154,14 +146,6 @@ spec: parameters: - name: create-volume-2-name name: step2-gunzip - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: command: - cat @@ -174,14 +158,6 @@ spec: parameters: - name: create-volume-3-name name: step3-output - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - dag: tasks: - arguments: diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml index 28236c284c2..b6ae8ef41e5 100644 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml +++ b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml @@ -44,14 +44,6 @@ spec: - name: create-volume-name - name: url name: step1-ingest - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - inputs: parameters: - name: create-volume-name @@ -87,14 +79,6 @@ spec: parameters: - name: create-volume-name name: step2-gunzip - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - inputs: parameters: - name: create-volume-name @@ -130,14 +114,6 @@ spec: parameters: - name: create-volume-name name: step3-copy - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - inputs: parameters: - name: create-volume-name @@ -172,14 +148,6 @@ spec: parameters: - name: create-volume-name name: step4-output - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - dag: tasks: - name: create-volume diff --git a/sdk/python/tests/compiler/testdata/volumeop_basic.yaml b/sdk/python/tests/compiler/testdata/volumeop_basic.yaml index 00623c873a3..6c0d2d521b0 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_basic.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_basic.yaml @@ -26,14 +26,6 @@ spec: parameters: - name: create-pvc-name name: cop - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - inputs: parameters: - name: size diff --git a/sdk/python/tests/compiler/testdata/volumeop_dag.yaml b/sdk/python/tests/compiler/testdata/volumeop_dag.yaml index 7426ae873a7..05a8ba2035a 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_dag.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_dag.yaml @@ -42,14 +42,6 @@ spec: parameters: - name: create-pvc-name name: step1 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: args: - echo 2 | tee /mnt2/file2 @@ -64,14 +56,6 @@ spec: parameters: - name: create-pvc-name name: step2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: args: - cat /mnt/file1 /mnt/file2 @@ -86,14 +70,6 @@ spec: parameters: - name: create-pvc-name name: step3 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - dag: tasks: - name: create-pvc diff --git a/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml b/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml index 890e0bd962b..8a9341a75b0 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml @@ -42,14 +42,6 @@ spec: parameters: - name: create-pvc-name name: step1 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: args: - echo 2 | tee /common/file2 @@ -64,14 +56,6 @@ spec: parameters: - name: create-pvc-name name: step2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: args: - echo 3 | tee /mnt3/file3 @@ -86,14 +70,6 @@ spec: parameters: - name: create-pvc-name name: step3 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - dag: tasks: - name: create-pvc diff --git a/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml b/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml index dbadc467aa5..f1cf8bbbad9 100644 --- a/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml +++ b/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml @@ -42,14 +42,6 @@ spec: parameters: - name: mypvc-name name: step1 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: args: - cp /data/file1 /data/file2 @@ -64,14 +56,6 @@ spec: parameters: - name: mypvc-name name: step2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - container: command: - cat @@ -85,14 +69,6 @@ spec: parameters: - name: mypvc-name name: step3 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - path: /mlpipeline-ui-metadata.json - optional: true - - name: mlpipeline-metrics - path: /mlpipeline-metrics.json - optional: true - dag: tasks: - name: mypvc diff --git a/sdk/python/tests/compiler/testdata/withitem_basic.yaml b/sdk/python/tests/compiler/testdata/withitem_basic.yaml index c4bd2fc514f..5cd187eaa45 100644 --- a/sdk/python/tests/compiler/testdata/withitem_basic.yaml +++ b/sdk/python/tests/compiler/testdata/withitem_basic.yaml @@ -47,14 +47,6 @@ spec: - name: loop-item-param-00000001-subvar-a - name: my-pipe-param name: my-in-coop1 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - container: args: - echo op2 {{inputs.parameters.loop-item-param-00000001-subvar-b}} @@ -66,14 +58,6 @@ spec: parameters: - name: loop-item-param-00000001-subvar-b name: my-in-coop2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - container: args: - echo {{inputs.parameters.my-pipe-param}} @@ -85,14 +69,6 @@ spec: parameters: - name: my-pipe-param name: my-out-cop - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - dag: tasks: - arguments: diff --git a/sdk/python/tests/compiler/testdata/withitem_nested.yaml b/sdk/python/tests/compiler/testdata/withitem_nested.yaml index c3feed786ca..70c2d740298 100644 --- a/sdk/python/tests/compiler/testdata/withitem_nested.yaml +++ b/sdk/python/tests/compiler/testdata/withitem_nested.yaml @@ -79,14 +79,6 @@ spec: - name: loop-item-param-00000001-subvar-a - name: my-pipe-param name: my-in-coop1 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - container: args: - echo op2 {{inputs.parameters.loop-item-param-00000001-subvar-b}} @@ -98,14 +90,6 @@ spec: parameters: - name: loop-item-param-00000001-subvar-b name: my-in-coop2 - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - container: args: - echo op1 {{inputs.parameters.loop-item-param-00000001-subvar-a}} {{inputs.parameters.loop-item-param-00000002}} @@ -120,14 +104,6 @@ spec: - name: loop-item-param-00000002 - name: my-pipe-param name: my-inner-inner-coop - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - container: args: - echo {{inputs.parameters.my-pipe-param}} @@ -139,14 +115,6 @@ spec: parameters: - name: my-pipe-param name: my-out-cop - outputs: - artifacts: - - name: mlpipeline-ui-metadata - optional: true - path: /mlpipeline-ui-metadata.json - - name: mlpipeline-metrics - optional: true - path: /mlpipeline-metrics.json - dag: tasks: - arguments: