From 96bac98783df9e1d75e4eeb976cbbf7a83f8dc44 Mon Sep 17 00:00:00 2001 From: Chen Sun Date: Fri, 8 Nov 2024 18:47:34 +0000 Subject: [PATCH] chore(sdk): remove kfp.deprecated and legacy samples and tests Signed-off-by: Chen Sun --- .gitignore | 4 - pytest.ini | 1 - ...- Retail product stockout prediction.ipynb | 164 -- samples/core/XGBoost/xgboost_sample_test.py | 10 +- samples/core/caching/caching_test.py | 6 +- samples/core/condition/condition.py | 51 - samples/core/condition/condition_test.py | 22 +- .../core/condition/nested_condition_test.py | 10 +- .../continue_training_from_prod.py | 167 -- samples/core/dataflow/dataflow.ipynb | 462 ----- samples/core/dataflow/dataflow_test.py | 29 - samples/core/dns_config/dns_config.py | 50 - .../dsl_static_type_checking.ipynb | 814 --------- .../dsl_static_type_checking_test.py | 24 - .../execution_order/execution_order_test.py | 10 +- .../core/exit_handler/exit_handler_test.py | 12 +- samples/core/helloworld/hello_world.py | 31 - .../core/imagepullsecrets/imagepullsecrets.py | 58 - .../imagepullsecrets/imagepullsecrets_test.py | 23 - .../kubeflow_tf_serving.ipynb | 350 ---- .../lightweight_component.ipynb | 273 --- .../lightweight_component_test.py | 23 - samples/core/loop_output/loop_output_test.py | 8 +- .../loop_parallelism/loop_parallelism_test.py | 10 +- .../loop_parameter/loop_parameter_test.py | 12 +- samples/core/loop_static/loop_static_test.py | 10 +- .../multiple_outputs/multiple_outputs_test.py | 10 +- .../output_a_directory_test.py | 10 +- .../parallelism_sub_dag.py | 35 - .../parallelism_sub_dag_test.py | 28 - .../parallelism_sub_dag_with_op_output.py | 39 - samples/core/parameterized_tfx_oss/README.md | 57 - .../check_permission.png | Bin 130803 -> 0 bytes .../parameterized_tfx_oss.py | 185 -- .../parameterized_tfx_oss_test.py | 45 - .../taxi_pipeline_notebook.ipynb | 314 ---- .../pipeline_parallelism_limits.py | 36 - .../pipeline_transformers.py | 42 - .../pipeline_transformers_test.py | 23 - .../preemptible_tpu_gpu.py | 43 - .../preemptible_tpu_gpu_test.py | 24 - samples/core/resource_ops/resource_ops.py | 66 - .../core/resource_ops/resource_ops_test.py | 23 - .../core/resource_spec/resource_spec_test.py | 10 +- .../runtime_resource_request_test.py | 9 +- samples/core/retry/retry_test.py | 9 +- samples/core/secret/secret_test.py | 10 +- samples/core/sidecar/sidecar.py | 46 - samples/core/sidecar/sidecar_test.py | 23 - samples/core/use_run_info/use_run_id.py | 50 - samples/core/use_run_info/use_run_id_test.py | 25 - .../core/visualization/confusion_matrix.csv | 9 - .../core/visualization/confusion_matrix.py | 55 - .../visualization/confusion_matrix_test.py | 48 - samples/core/visualization/hello-world.html | 52 - samples/core/visualization/html.py | 56 - samples/core/visualization/html_test.py | 23 - samples/core/visualization/markdown.py | 60 - samples/core/visualization/markdown_test.py | 24 - samples/core/visualization/roc.csv | 293 --- samples/core/visualization/roc.py | 55 - samples/core/visualization/roc_test.py | 48 - samples/core/visualization/table.csv | 7 - samples/core/visualization/table.py | 53 - samples/core/visualization/table_test.py | 49 - samples/core/visualization/tensorboard_gcs.py | 73 - .../core/visualization/tensorboard_minio.py | 179 -- .../visualization/tensorboard_minio_test.py | 101 -- samples/core/volume_snapshot_ops/README.md | 44 - .../volume_snapshot_ops.py | 83 - .../volume_snapshot_ops_test.py | 24 - samples/core/xgboost_training_cm/README.md | 79 - .../xgboost_training_cm.py | 312 ---- samples/test/after_test.py | 10 +- samples/test/cache_v2_compatible_test.py | 145 -- samples/test/fail.py | 31 - samples/test/fail_parameter_value_missing.py | 36 - .../test/fail_parameter_value_missing_test.py | 31 - samples/test/fail_test.py | 20 +- samples/test/legacy_data_passing.py | 151 -- samples/test/legacy_data_passing_test.py | 10 - samples/test/legacy_exit_handler.py | 57 - samples/test/legacy_exit_handler_test.py | 25 - ...eight_python_functions_v2_pipeline_test.py | 13 +- ...t_python_functions_v2_with_outputs_test.py | 13 +- samples/test/metrics_visualization_v1.py | 32 - samples/test/metrics_visualization_v1_test.py | 26 - samples/test/metrics_visualization_v2_test.py | 12 +- samples/test/parameter_with_format.py | 28 - samples/test/parameter_with_format_test.py | 38 - samples/test/placeholder_concat_test.py | 7 +- samples/test/placeholder_if.py | 66 - samples/test/placeholder_if_test.py | 25 +- samples/test/reused_component.py | 54 - samples/test/reused_component_test.py | 46 - samples/test/two_step.py | 58 - samples/test/two_step_test.py | 149 -- .../two_step_with_uri_placeholder_test.py | 12 +- samples/test/utils/kfp/samples/test/utils.py | 189 +- samples/v2/cache_test.py | 13 +- .../v2/component_with_optional_inputs_test.py | 4 +- samples/v2/hello_world_test.py | 9 +- .../v2/pipeline_container_no_input_test.py | 9 +- samples/v2/pipeline_with_env_test.py | 24 +- samples/v2/pipeline_with_importer_test.py | 3 +- .../v2/pipeline_with_secret_as_env_test.py | 9 +- .../v2/pipeline_with_secret_as_volume_test.py | 9 +- samples/v2/pipeline_with_volume_test.py | 10 +- samples/v2/producer_consumer_param_test.py | 15 +- .../two_step_pipeline_containerized_test.py | 3 +- sdk/CONTRIBUTING.md | 2 +- sdk/python/kfp/deprecated/__init__.py | 24 - sdk/python/kfp/deprecated/__main__.py | 22 - sdk/python/kfp/deprecated/_auth.py | 223 --- sdk/python/kfp/deprecated/_client.py | 1529 ---------------- sdk/python/kfp/deprecated/_config.py | 18 - sdk/python/kfp/deprecated/_local_client.py | 546 ------ sdk/python/kfp/deprecated/_runners.py | 95 - sdk/python/kfp/deprecated/auth/__init__.py | 19 - .../deprecated/auth/_satvolumecredentials.py | 69 - .../deprecated/auth/_tokencredentialsbase.py | 47 - sdk/python/kfp/deprecated/aws.py | 73 - sdk/python/kfp/deprecated/azure.py | 53 - sdk/python/kfp/deprecated/cli/__init__.py | 13 - sdk/python/kfp/deprecated/cli/cli.py | 83 - sdk/python/kfp/deprecated/cli/components.py | 408 ----- .../kfp/deprecated/cli/components_test.py | 492 ----- .../deprecated/cli/diagnose_me/__init__.py | 13 - .../kfp/deprecated/cli/diagnose_me/dev_env.py | 71 - .../cli/diagnose_me/dev_env_test.py | 63 - .../kfp/deprecated/cli/diagnose_me/gcp.py | 152 -- .../deprecated/cli/diagnose_me/gcp_test.py | 87 - .../cli/diagnose_me/kubernetes_cluster.py | 124 -- .../diagnose_me/kubernetes_cluster_test.py | 76 - .../kfp/deprecated/cli/diagnose_me/utility.py | 91 - .../cli/diagnose_me/utility_test.py | 43 - .../kfp/deprecated/cli/diagnose_me_cli.py | 106 -- sdk/python/kfp/deprecated/cli/experiment.py | 143 -- sdk/python/kfp/deprecated/cli/output.py | 63 - sdk/python/kfp/deprecated/cli/pipeline.py | 263 --- .../kfp/deprecated/cli/recurring_run.py | 214 --- sdk/python/kfp/deprecated/cli/run.py | 227 --- .../kfp/deprecated/compiler/__init__.py | 16 - .../compiler/_data_passing_rewriter.py | 475 ----- .../compiler/_data_passing_using_volume.py | 256 --- .../compiler/_default_transformers.py | 92 - .../kfp/deprecated/compiler/_k8s_helper.py | 90 - .../deprecated/compiler/_op_to_template.py | 364 ---- .../kfp/deprecated/compiler/compiler.py | 1271 ------------- sdk/python/kfp/deprecated/compiler/main.py | 148 -- .../v2_compatible_two_step_pipeline.yaml | 254 --- ...wo_step_pipeline_with_custom_launcher.yaml | 254 --- .../kfp/deprecated/compiler/v2_compat.py | 192 -- .../compiler/v2_compatible_compiler_test.py | 179 -- .../kfp/deprecated/components/__init__.py | 19 - .../kfp/deprecated/components/_airflow_op.py | 145 -- .../deprecated/components/_component_store.py | 395 ---- .../kfp/deprecated/components/_components.py | 676 ------- .../deprecated/components/_data_passing.py | 252 --- .../kfp/deprecated/components/_dynamic.py | 96 - .../deprecated/components/_key_value_store.py | 68 - .../kfp/deprecated/components/_naming.py | 121 -- .../kfp/deprecated/components/_python_op.py | 1123 ------------ .../components/_python_to_graph_component.py | 213 --- .../kfp/deprecated/components/_structures.py | 896 --------- .../kfp/deprecated/components/_yaml_utils.py | 71 - .../kfp/deprecated/components/modelbase.py | 394 ---- .../components/structures/__init__.py | 1 - .../structures/components.json_schema.json | 375 ---- .../components.json_schema.outline.yaml | 69 - .../components/structures/components.proto | 216 --- .../structures/generate_proto_code.sh | 20 - .../components/type_annotation_utils.py | 65 - .../components/type_annotation_utils_test.py | 87 - .../deprecated/components_tests/__init__.py | 0 .../components_tests/test_components.py | 1114 ------------ ...with_0_inputs_and_2_outputs.component.yaml | 15 - ...with_2_inputs_and_0_outputs.component.yaml | 15 - ...with_2_inputs_and_2_outputs.component.yaml | 22 - .../components_tests/test_data/module1.py | 15 - .../module2_which_depends_on_module1.py | 5 - .../test_data/python_add.component.yaml | 35 - .../test_data/python_add.component.zip | Bin 547 -> 0 bytes ...tockout_prediction_pipeline.component.yaml | 134 -- ...il_product_stockout_prediction_pipeline.py | 77 - .../components_tests/test_data_passing.py | 34 - .../components_tests/test_graph_components.py | 336 ---- .../components_tests/test_python_op.py | 1244 ------------- ...test_python_pipeline_to_graph_component.py | 82 - .../test_structure_model_base.py | 303 ---- .../kfp/deprecated/containers/__init__.py | 15 - .../deprecated/containers/_build_image_api.py | 146 -- .../kfp/deprecated/containers/_cache.py | 73 - .../containers/_component_builder.py | 429 ----- .../containers/_container_builder.py | 230 --- .../kfp/deprecated/containers/_gcs_helper.py | 115 -- .../deprecated/containers/_k8s_job_helper.py | 163 -- .../kfp/deprecated/containers/entrypoint.py | 227 --- .../deprecated/containers/entrypoint_utils.py | 190 -- .../deprecated/containers_tests/__init__.py | 0 .../component_builder_test.py | 66 - .../containers_tests/test_build_image_api.py | 167 -- .../containers_tests/testdata/__init__.py | 13 - .../testdata/executor_output.json | 29 - .../testdata/expected_component.yaml | 20 - .../containers_tests/testdata/main.py | 51 - .../testdata/pipeline_source.py | 18 - sdk/python/kfp/deprecated/dsl/__init__.py | 39 - sdk/python/kfp/deprecated/dsl/_component.py | 167 -- .../kfp/deprecated/dsl/_component_bridge.py | 691 ------- .../kfp/deprecated/dsl/_container_op.py | 1616 ----------------- .../kfp/deprecated/dsl/_container_op_test.py | 57 - sdk/python/kfp/deprecated/dsl/_for_loop.py | 257 --- sdk/python/kfp/deprecated/dsl/_metadata.py | 96 - sdk/python/kfp/deprecated/dsl/_ops_group.py | 235 --- sdk/python/kfp/deprecated/dsl/_pipeline.py | 372 ---- .../kfp/deprecated/dsl/_pipeline_param.py | 253 --- .../kfp/deprecated/dsl/_pipeline_volume.py | 125 -- sdk/python/kfp/deprecated/dsl/_resource_op.py | 230 --- sdk/python/kfp/deprecated/dsl/_volume_op.py | 137 -- .../kfp/deprecated/dsl/_volume_snapshot_op.py | 117 -- sdk/python/kfp/deprecated/dsl/artifact.py | 215 --- .../kfp/deprecated/dsl/artifact_utils.py | 111 -- .../kfp/deprecated/dsl/component_spec.py | 431 ----- .../kfp/deprecated/dsl/component_spec_test.py | 631 ------- .../deprecated/dsl/data_passing_methods.py | 27 - sdk/python/kfp/deprecated/dsl/dsl_utils.py | 137 -- .../kfp/deprecated/dsl/dsl_utils_test.py | 58 - .../kfp/deprecated/dsl/extensions/__init__.py | 0 .../deprecated/dsl/extensions/kubernetes.py | 79 - sdk/python/kfp/deprecated/dsl/io_types.py | 34 - .../kfp/deprecated/dsl/metrics_utils.py | 176 -- .../kfp/deprecated/dsl/metrics_utils_test.py | 63 - .../kfp/deprecated/dsl/serialization_utils.py | 41 - .../dsl/serialization_utils_test.py | 48 - ...expected_bulk_loaded_confusion_matrix.json | 10 - .../test_data/expected_confusion_matrix.json | 10 - .../type_schemas/classification_metrics.yaml | 81 - .../dsl/type_schemas/confidence_metrics.yaml | 45 - .../dsl/type_schemas/confusion_matrix.yaml | 23 - .../deprecated/dsl/type_schemas/dataset.yaml | 7 - .../deprecated/dsl/type_schemas/metrics.yaml | 15 - .../deprecated/dsl/type_schemas/model.yaml | 7 - .../sliced_classification_metrics.yaml | 94 - sdk/python/kfp/deprecated/dsl/type_utils.py | 95 - sdk/python/kfp/deprecated/dsl/types.py | 244 --- sdk/python/kfp/deprecated/gcp.py | 146 -- .../kfp/deprecated/notebook/__init__.py | 15 - sdk/python/kfp/deprecated/notebook/_magic.py | 13 - sdk/python/kfp/deprecated/onprem.py | 133 -- sdk/python/setup.py | 1 - sdk/python/tests/__init__.py | 0 sdk/python/tests/compiler/__init__.py | 0 .../tests/compiler/component_builder_test.py | 194 -- .../tests/compiler/container_builder_test.py | 103 -- sdk/python/tests/compiler/k8s_helper_tests.py | 36 - sdk/python/tests/compiler/main.py | 35 - sdk/python/tests/compiler/testdata/README.md | 3 - .../tests/compiler/testdata/add_pod_env.py | 29 - .../tests/compiler/testdata/add_pod_env.yaml | 53 - .../testdata/artifact_passing_using_volume.py | 82 - .../artifact_passing_using_volume.yaml | 234 --- sdk/python/tests/compiler/testdata/basic.py | 94 - sdk/python/tests/compiler/testdata/basic.yaml | 120 -- .../compiler/testdata/basic_no_decorator.py | 92 - .../compiler/testdata/basic_no_decorator.yaml | 121 -- sdk/python/tests/compiler/testdata/coin.py | 51 - sdk/python/tests/compiler/testdata/coin.yaml | 140 -- sdk/python/tests/compiler/testdata/compose.py | 101 -- .../tests/compiler/testdata/compose.yaml | 110 -- .../tests/compiler/testdata/default_value.py | 35 - .../compiler/testdata/default_value.yaml | 77 - .../compiler/testdata/imagepullsecrets.yaml | 43 - .../testdata/input_artifact_raw_value.py | 77 - .../testdata/input_artifact_raw_value.txt | 1 - .../testdata/input_artifact_raw_value.yaml | 71 - .../tests/compiler/testdata/kaniko.basic.yaml | 34 - .../compiler/testdata/kaniko.kubeflow.yaml | 34 - .../testdata/loop_over_lightweight_output.py | 52 - .../loop_over_lightweight_output.yaml | 123 -- .../tests/compiler/testdata/opsgroups.py | 40 - .../tests/compiler/testdata/opsgroups.yaml | 73 - .../parallelfor_item_argument_resolving.py | 75 - .../parallelfor_item_argument_resolving.yaml | 788 -------- ...elfor_pipeline_param_in_items_resolving.py | 66 - ...for_pipeline_param_in_items_resolving.yaml | 455 ----- .../compiler/testdata/param_op_transform.py | 27 - .../compiler/testdata/param_op_transform.yaml | 37 - .../compiler/testdata/param_substitutions.py | 35 - .../testdata/param_substitutions.yaml | 59 - .../tests/compiler/testdata/pipelineparams.py | 47 - .../compiler/testdata/pipelineparams.yaml | 94 - .../testdata/preemptible_tpu_gpu.yaml | 50 - .../compiler/testdata/recursive_do_while.py | 65 - .../compiler/testdata/recursive_do_while.yaml | 140 -- .../compiler/testdata/recursive_while.py | 64 - .../compiler/testdata/recursive_while.yaml | 142 -- .../compiler/testdata/resourceop_basic.py | 55 - .../compiler/testdata/resourceop_basic.yaml | 74 - sdk/python/tests/compiler/testdata/sidecar.py | 36 - .../tests/compiler/testdata/sidecar.yaml | 71 - .../test_data/consume_2.component.yaml | 16 - .../test_data/process_2_2.component.yaml | 23 - .../test_data/produce_2.component.yaml | 18 - .../testpackage/mypipeline/__init__.py | 15 - .../testpackage/mypipeline/compose.py | 101 -- .../testpackage/mypipeline/kaniko.basic.yaml | 30 - .../compiler/testdata/testpackage/setup.py | 23 - sdk/python/tests/compiler/testdata/timeout.py | 44 - .../tests/compiler/testdata/timeout.yaml | 41 - .../tests/compiler/testdata/tolerations.yaml | 43 - .../tests/compiler/testdata/two_step.py | 54 - .../tests/compiler/testdata/uri_artifacts.py | 107 -- .../compiler/testdata/uri_artifacts.yaml | 447 ----- sdk/python/tests/compiler/testdata/volume.py | 40 - .../tests/compiler/testdata/volume.yaml | 78 - .../testdata/volume_snapshotop_rokurl.py | 76 - .../testdata/volume_snapshotop_rokurl.yaml | 246 --- .../testdata/volume_snapshotop_sequential.py | 77 - .../volume_snapshotop_sequential.yaml | 238 --- .../tests/compiler/testdata/volumeop_basic.py | 37 - .../compiler/testdata/volumeop_basic.yaml | 72 - .../tests/compiler/testdata/volumeop_dag.py | 51 - .../tests/compiler/testdata/volumeop_dag.yaml | 115 -- .../compiler/testdata/volumeop_parallel.py | 52 - .../compiler/testdata/volumeop_parallel.yaml | 113 -- .../compiler/testdata/volumeop_sequential.py | 51 - .../testdata/volumeop_sequential.yaml | 114 -- .../tests/compiler/testdata/withitem_basic.py | 59 - .../compiler/testdata/withitem_basic.yaml | 98 - .../compiler/testdata/withitem_nested.py | 70 - .../compiler/testdata/withitem_nested.yaml | 144 -- .../compiler/testdata/withparam_global.py | 62 - .../compiler/testdata/withparam_global.yaml | 88 - .../testdata/withparam_global_dict.py | 62 - .../testdata/withparam_global_dict.yaml | 88 - .../compiler/testdata/withparam_output.py | 62 - .../compiler/testdata/withparam_output.yaml | 85 - .../testdata/withparam_output_dict.py | 62 - .../testdata/withparam_output_dict.yaml | 85 - sdk/python/tests/dsl/__init__.py | 0 sdk/python/tests/dsl/aws_extensions_tests.py | 68 - .../tests/dsl/component_bridge_tests.py | 414 ----- sdk/python/tests/dsl/component_tests.py | 49 - sdk/python/tests/dsl/extensions/__init__.py | 0 .../tests/dsl/extensions/test_kubernetes.py | 70 - sdk/python/tests/dsl/main.py | 65 - sdk/python/tests/dsl/metadata_tests.py | 105 -- sdk/python/tests/dsl/ops_group_tests.py | 133 -- sdk/python/tests/dsl/pipeline_param_tests.py | 110 -- sdk/python/tests/dsl/pipeline_tests.py | 97 - sdk/python/tests/dsl/test_azure_extensions.py | 48 - sdk/python/tests/dsl/type_tests.py | 52 - sdk/python/tests/local_runner_test.py | 240 --- sdk/python/tests/run_tests.sh | 17 - sdk/python/tests/test_kfp.py | 8 - test/presubmit-isort-sdk.sh | 4 +- test/presubmit-tests-sdk.sh | 3 +- test/presubmit-yapf-sdk.sh | 4 +- 359 files changed, 223 insertions(+), 43214 deletions(-) delete mode 100644 samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb delete mode 100644 samples/core/condition/condition.py delete mode 100644 samples/core/continue_training_from_prod/continue_training_from_prod.py delete mode 100644 samples/core/dataflow/dataflow.ipynb delete mode 100644 samples/core/dataflow/dataflow_test.py delete mode 100644 samples/core/dns_config/dns_config.py delete mode 100644 samples/core/dsl_static_type_checking/dsl_static_type_checking.ipynb delete mode 100644 samples/core/dsl_static_type_checking/dsl_static_type_checking_test.py delete mode 100644 samples/core/helloworld/hello_world.py delete mode 100644 samples/core/imagepullsecrets/imagepullsecrets.py delete mode 100644 samples/core/imagepullsecrets/imagepullsecrets_test.py delete mode 100644 samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb delete mode 100644 samples/core/lightweight_component/lightweight_component.ipynb delete mode 100644 samples/core/lightweight_component/lightweight_component_test.py delete mode 100644 samples/core/parallelism_sub_dag/parallelism_sub_dag.py delete mode 100644 samples/core/parallelism_sub_dag/parallelism_sub_dag_test.py delete mode 100644 samples/core/parallelism_sub_dag/parallelism_sub_dag_with_op_output.py delete mode 100644 samples/core/parameterized_tfx_oss/README.md delete mode 100644 samples/core/parameterized_tfx_oss/check_permission.png delete mode 100644 samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py delete mode 100644 samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py delete mode 100644 samples/core/parameterized_tfx_oss/taxi_pipeline_notebook.ipynb delete mode 100644 samples/core/pipeline_parallelism/pipeline_parallelism_limits.py delete mode 100644 samples/core/pipeline_transformers/pipeline_transformers.py delete mode 100644 samples/core/pipeline_transformers/pipeline_transformers_test.py delete mode 100644 samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu.py delete mode 100644 samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu_test.py delete mode 100644 samples/core/resource_ops/resource_ops.py delete mode 100644 samples/core/resource_ops/resource_ops_test.py delete mode 100644 samples/core/sidecar/sidecar.py delete mode 100644 samples/core/sidecar/sidecar_test.py delete mode 100644 samples/core/use_run_info/use_run_id.py delete mode 100644 samples/core/use_run_info/use_run_id_test.py delete mode 100644 samples/core/visualization/confusion_matrix.csv delete mode 100644 samples/core/visualization/confusion_matrix.py delete mode 100644 samples/core/visualization/confusion_matrix_test.py delete mode 100644 samples/core/visualization/hello-world.html delete mode 100644 samples/core/visualization/html.py delete mode 100644 samples/core/visualization/html_test.py delete mode 100644 samples/core/visualization/markdown.py delete mode 100644 samples/core/visualization/markdown_test.py delete mode 100644 samples/core/visualization/roc.csv delete mode 100644 samples/core/visualization/roc.py delete mode 100644 samples/core/visualization/roc_test.py delete mode 100644 samples/core/visualization/table.csv delete mode 100644 samples/core/visualization/table.py delete mode 100644 samples/core/visualization/table_test.py delete mode 100644 samples/core/visualization/tensorboard_gcs.py delete mode 100644 samples/core/visualization/tensorboard_minio.py delete mode 100644 samples/core/visualization/tensorboard_minio_test.py delete mode 100644 samples/core/volume_snapshot_ops/README.md delete mode 100644 samples/core/volume_snapshot_ops/volume_snapshot_ops.py delete mode 100644 samples/core/volume_snapshot_ops/volume_snapshot_ops_test.py delete mode 100644 samples/core/xgboost_training_cm/README.md delete mode 100644 samples/core/xgboost_training_cm/xgboost_training_cm.py delete mode 100644 samples/test/cache_v2_compatible_test.py delete mode 100644 samples/test/fail.py delete mode 100644 samples/test/fail_parameter_value_missing.py delete mode 100644 samples/test/fail_parameter_value_missing_test.py delete mode 100644 samples/test/legacy_data_passing.py delete mode 100644 samples/test/legacy_data_passing_test.py delete mode 100755 samples/test/legacy_exit_handler.py delete mode 100755 samples/test/legacy_exit_handler_test.py delete mode 100644 samples/test/metrics_visualization_v1.py delete mode 100644 samples/test/metrics_visualization_v1_test.py delete mode 100644 samples/test/parameter_with_format.py delete mode 100644 samples/test/parameter_with_format_test.py delete mode 100644 samples/test/placeholder_if.py delete mode 100644 samples/test/reused_component.py delete mode 100644 samples/test/reused_component_test.py delete mode 100644 samples/test/two_step.py delete mode 100644 samples/test/two_step_test.py delete mode 100644 sdk/python/kfp/deprecated/__init__.py delete mode 100644 sdk/python/kfp/deprecated/__main__.py delete mode 100644 sdk/python/kfp/deprecated/_auth.py delete mode 100644 sdk/python/kfp/deprecated/_client.py delete mode 100644 sdk/python/kfp/deprecated/_config.py delete mode 100644 sdk/python/kfp/deprecated/_local_client.py delete mode 100644 sdk/python/kfp/deprecated/_runners.py delete mode 100644 sdk/python/kfp/deprecated/auth/__init__.py delete mode 100644 sdk/python/kfp/deprecated/auth/_satvolumecredentials.py delete mode 100644 sdk/python/kfp/deprecated/auth/_tokencredentialsbase.py delete mode 100644 sdk/python/kfp/deprecated/aws.py delete mode 100644 sdk/python/kfp/deprecated/azure.py delete mode 100644 sdk/python/kfp/deprecated/cli/__init__.py delete mode 100644 sdk/python/kfp/deprecated/cli/cli.py delete mode 100644 sdk/python/kfp/deprecated/cli/components.py delete mode 100644 sdk/python/kfp/deprecated/cli/components_test.py delete mode 100644 sdk/python/kfp/deprecated/cli/diagnose_me/__init__.py delete mode 100644 sdk/python/kfp/deprecated/cli/diagnose_me/dev_env.py delete mode 100644 sdk/python/kfp/deprecated/cli/diagnose_me/dev_env_test.py delete mode 100644 sdk/python/kfp/deprecated/cli/diagnose_me/gcp.py delete mode 100644 sdk/python/kfp/deprecated/cli/diagnose_me/gcp_test.py delete mode 100644 sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster.py delete mode 100644 sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster_test.py delete mode 100644 sdk/python/kfp/deprecated/cli/diagnose_me/utility.py delete mode 100644 sdk/python/kfp/deprecated/cli/diagnose_me/utility_test.py delete mode 100644 sdk/python/kfp/deprecated/cli/diagnose_me_cli.py delete mode 100644 sdk/python/kfp/deprecated/cli/experiment.py delete mode 100644 sdk/python/kfp/deprecated/cli/output.py delete mode 100644 sdk/python/kfp/deprecated/cli/pipeline.py delete mode 100644 sdk/python/kfp/deprecated/cli/recurring_run.py delete mode 100644 sdk/python/kfp/deprecated/cli/run.py delete mode 100644 sdk/python/kfp/deprecated/compiler/__init__.py delete mode 100644 sdk/python/kfp/deprecated/compiler/_data_passing_rewriter.py delete mode 100644 sdk/python/kfp/deprecated/compiler/_data_passing_using_volume.py delete mode 100644 sdk/python/kfp/deprecated/compiler/_default_transformers.py delete mode 100644 sdk/python/kfp/deprecated/compiler/_k8s_helper.py delete mode 100644 sdk/python/kfp/deprecated/compiler/_op_to_template.py delete mode 100644 sdk/python/kfp/deprecated/compiler/compiler.py delete mode 100644 sdk/python/kfp/deprecated/compiler/main.py delete mode 100644 sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline.yaml delete mode 100644 sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml delete mode 100644 sdk/python/kfp/deprecated/compiler/v2_compat.py delete mode 100644 sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py delete mode 100644 sdk/python/kfp/deprecated/components/__init__.py delete mode 100644 sdk/python/kfp/deprecated/components/_airflow_op.py delete mode 100644 sdk/python/kfp/deprecated/components/_component_store.py delete mode 100644 sdk/python/kfp/deprecated/components/_components.py delete mode 100644 sdk/python/kfp/deprecated/components/_data_passing.py delete mode 100644 sdk/python/kfp/deprecated/components/_dynamic.py delete mode 100644 sdk/python/kfp/deprecated/components/_key_value_store.py delete mode 100644 sdk/python/kfp/deprecated/components/_naming.py delete mode 100644 sdk/python/kfp/deprecated/components/_python_op.py delete mode 100644 sdk/python/kfp/deprecated/components/_python_to_graph_component.py delete mode 100644 sdk/python/kfp/deprecated/components/_structures.py delete mode 100644 sdk/python/kfp/deprecated/components/_yaml_utils.py delete mode 100644 sdk/python/kfp/deprecated/components/modelbase.py delete mode 100644 sdk/python/kfp/deprecated/components/structures/__init__.py delete mode 100644 sdk/python/kfp/deprecated/components/structures/components.json_schema.json delete mode 100644 sdk/python/kfp/deprecated/components/structures/components.json_schema.outline.yaml delete mode 100644 sdk/python/kfp/deprecated/components/structures/components.proto delete mode 100755 sdk/python/kfp/deprecated/components/structures/generate_proto_code.sh delete mode 100644 sdk/python/kfp/deprecated/components/type_annotation_utils.py delete mode 100644 sdk/python/kfp/deprecated/components/type_annotation_utils_test.py delete mode 100644 sdk/python/kfp/deprecated/components_tests/__init__.py delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_components.py delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_data/component_with_0_inputs_and_2_outputs.component.yaml delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_0_outputs.component.yaml delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_2_outputs.component.yaml delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_data/module1.py delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_data/module2_which_depends_on_module1.py delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.yaml delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.zip delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_data/retail_product_stockout_prediction_pipeline.component.yaml delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_data/retail_product_stockout_prediction_pipeline.py delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_data_passing.py delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_graph_components.py delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_python_op.py delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_python_pipeline_to_graph_component.py delete mode 100644 sdk/python/kfp/deprecated/components_tests/test_structure_model_base.py delete mode 100644 sdk/python/kfp/deprecated/containers/__init__.py delete mode 100644 sdk/python/kfp/deprecated/containers/_build_image_api.py delete mode 100644 sdk/python/kfp/deprecated/containers/_cache.py delete mode 100644 sdk/python/kfp/deprecated/containers/_component_builder.py delete mode 100644 sdk/python/kfp/deprecated/containers/_container_builder.py delete mode 100644 sdk/python/kfp/deprecated/containers/_gcs_helper.py delete mode 100644 sdk/python/kfp/deprecated/containers/_k8s_job_helper.py delete mode 100644 sdk/python/kfp/deprecated/containers/entrypoint.py delete mode 100644 sdk/python/kfp/deprecated/containers/entrypoint_utils.py delete mode 100644 sdk/python/kfp/deprecated/containers_tests/__init__.py delete mode 100644 sdk/python/kfp/deprecated/containers_tests/component_builder_test.py delete mode 100644 sdk/python/kfp/deprecated/containers_tests/test_build_image_api.py delete mode 100644 sdk/python/kfp/deprecated/containers_tests/testdata/__init__.py delete mode 100644 sdk/python/kfp/deprecated/containers_tests/testdata/executor_output.json delete mode 100644 sdk/python/kfp/deprecated/containers_tests/testdata/expected_component.yaml delete mode 100644 sdk/python/kfp/deprecated/containers_tests/testdata/main.py delete mode 100644 sdk/python/kfp/deprecated/containers_tests/testdata/pipeline_source.py delete mode 100644 sdk/python/kfp/deprecated/dsl/__init__.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_component.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_component_bridge.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_container_op.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_container_op_test.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_for_loop.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_metadata.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_ops_group.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_pipeline.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_pipeline_param.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_pipeline_volume.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_resource_op.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_volume_op.py delete mode 100644 sdk/python/kfp/deprecated/dsl/_volume_snapshot_op.py delete mode 100644 sdk/python/kfp/deprecated/dsl/artifact.py delete mode 100644 sdk/python/kfp/deprecated/dsl/artifact_utils.py delete mode 100644 sdk/python/kfp/deprecated/dsl/component_spec.py delete mode 100644 sdk/python/kfp/deprecated/dsl/component_spec_test.py delete mode 100644 sdk/python/kfp/deprecated/dsl/data_passing_methods.py delete mode 100644 sdk/python/kfp/deprecated/dsl/dsl_utils.py delete mode 100644 sdk/python/kfp/deprecated/dsl/dsl_utils_test.py delete mode 100644 sdk/python/kfp/deprecated/dsl/extensions/__init__.py delete mode 100644 sdk/python/kfp/deprecated/dsl/extensions/kubernetes.py delete mode 100644 sdk/python/kfp/deprecated/dsl/io_types.py delete mode 100644 sdk/python/kfp/deprecated/dsl/metrics_utils.py delete mode 100644 sdk/python/kfp/deprecated/dsl/metrics_utils_test.py delete mode 100644 sdk/python/kfp/deprecated/dsl/serialization_utils.py delete mode 100644 sdk/python/kfp/deprecated/dsl/serialization_utils_test.py delete mode 100644 sdk/python/kfp/deprecated/dsl/test_data/expected_bulk_loaded_confusion_matrix.json delete mode 100644 sdk/python/kfp/deprecated/dsl/test_data/expected_confusion_matrix.json delete mode 100644 sdk/python/kfp/deprecated/dsl/type_schemas/classification_metrics.yaml delete mode 100644 sdk/python/kfp/deprecated/dsl/type_schemas/confidence_metrics.yaml delete mode 100644 sdk/python/kfp/deprecated/dsl/type_schemas/confusion_matrix.yaml delete mode 100644 sdk/python/kfp/deprecated/dsl/type_schemas/dataset.yaml delete mode 100644 sdk/python/kfp/deprecated/dsl/type_schemas/metrics.yaml delete mode 100644 sdk/python/kfp/deprecated/dsl/type_schemas/model.yaml delete mode 100644 sdk/python/kfp/deprecated/dsl/type_schemas/sliced_classification_metrics.yaml delete mode 100644 sdk/python/kfp/deprecated/dsl/type_utils.py delete mode 100644 sdk/python/kfp/deprecated/dsl/types.py delete mode 100644 sdk/python/kfp/deprecated/gcp.py delete mode 100644 sdk/python/kfp/deprecated/notebook/__init__.py delete mode 100644 sdk/python/kfp/deprecated/notebook/_magic.py delete mode 100644 sdk/python/kfp/deprecated/onprem.py delete mode 100644 sdk/python/tests/__init__.py delete mode 100644 sdk/python/tests/compiler/__init__.py delete mode 100644 sdk/python/tests/compiler/component_builder_test.py delete mode 100644 sdk/python/tests/compiler/container_builder_test.py delete mode 100644 sdk/python/tests/compiler/k8s_helper_tests.py delete mode 100644 sdk/python/tests/compiler/main.py delete mode 100644 sdk/python/tests/compiler/testdata/README.md delete mode 100644 sdk/python/tests/compiler/testdata/add_pod_env.py delete mode 100644 sdk/python/tests/compiler/testdata/add_pod_env.yaml delete mode 100644 sdk/python/tests/compiler/testdata/artifact_passing_using_volume.py delete mode 100644 sdk/python/tests/compiler/testdata/artifact_passing_using_volume.yaml delete mode 100644 sdk/python/tests/compiler/testdata/basic.py delete mode 100644 sdk/python/tests/compiler/testdata/basic.yaml delete mode 100644 sdk/python/tests/compiler/testdata/basic_no_decorator.py delete mode 100644 sdk/python/tests/compiler/testdata/basic_no_decorator.yaml delete mode 100644 sdk/python/tests/compiler/testdata/coin.py delete mode 100644 sdk/python/tests/compiler/testdata/coin.yaml delete mode 100644 sdk/python/tests/compiler/testdata/compose.py delete mode 100644 sdk/python/tests/compiler/testdata/compose.yaml delete mode 100644 sdk/python/tests/compiler/testdata/default_value.py delete mode 100644 sdk/python/tests/compiler/testdata/default_value.yaml delete mode 100644 sdk/python/tests/compiler/testdata/imagepullsecrets.yaml delete mode 100644 sdk/python/tests/compiler/testdata/input_artifact_raw_value.py delete mode 100644 sdk/python/tests/compiler/testdata/input_artifact_raw_value.txt delete mode 100644 sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml delete mode 100644 sdk/python/tests/compiler/testdata/kaniko.basic.yaml delete mode 100644 sdk/python/tests/compiler/testdata/kaniko.kubeflow.yaml delete mode 100644 sdk/python/tests/compiler/testdata/loop_over_lightweight_output.py delete mode 100644 sdk/python/tests/compiler/testdata/loop_over_lightweight_output.yaml delete mode 100644 sdk/python/tests/compiler/testdata/opsgroups.py delete mode 100644 sdk/python/tests/compiler/testdata/opsgroups.yaml delete mode 100644 sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.py delete mode 100644 sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.yaml delete mode 100644 sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.py delete mode 100644 sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.yaml delete mode 100644 sdk/python/tests/compiler/testdata/param_op_transform.py delete mode 100644 sdk/python/tests/compiler/testdata/param_op_transform.yaml delete mode 100644 sdk/python/tests/compiler/testdata/param_substitutions.py delete mode 100644 sdk/python/tests/compiler/testdata/param_substitutions.yaml delete mode 100644 sdk/python/tests/compiler/testdata/pipelineparams.py delete mode 100644 sdk/python/tests/compiler/testdata/pipelineparams.yaml delete mode 100644 sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml delete mode 100644 sdk/python/tests/compiler/testdata/recursive_do_while.py delete mode 100644 sdk/python/tests/compiler/testdata/recursive_do_while.yaml delete mode 100644 sdk/python/tests/compiler/testdata/recursive_while.py delete mode 100644 sdk/python/tests/compiler/testdata/recursive_while.yaml delete mode 100644 sdk/python/tests/compiler/testdata/resourceop_basic.py delete mode 100644 sdk/python/tests/compiler/testdata/resourceop_basic.yaml delete mode 100644 sdk/python/tests/compiler/testdata/sidecar.py delete mode 100644 sdk/python/tests/compiler/testdata/sidecar.yaml delete mode 100644 sdk/python/tests/compiler/testdata/test_data/consume_2.component.yaml delete mode 100644 sdk/python/tests/compiler/testdata/test_data/process_2_2.component.yaml delete mode 100644 sdk/python/tests/compiler/testdata/test_data/produce_2.component.yaml delete mode 100644 sdk/python/tests/compiler/testdata/testpackage/mypipeline/__init__.py delete mode 100644 sdk/python/tests/compiler/testdata/testpackage/mypipeline/compose.py delete mode 100644 sdk/python/tests/compiler/testdata/testpackage/mypipeline/kaniko.basic.yaml delete mode 100644 sdk/python/tests/compiler/testdata/testpackage/setup.py delete mode 100755 sdk/python/tests/compiler/testdata/timeout.py delete mode 100644 sdk/python/tests/compiler/testdata/timeout.yaml delete mode 100644 sdk/python/tests/compiler/testdata/tolerations.yaml delete mode 100644 sdk/python/tests/compiler/testdata/two_step.py delete mode 100644 sdk/python/tests/compiler/testdata/uri_artifacts.py delete mode 100644 sdk/python/tests/compiler/testdata/uri_artifacts.yaml delete mode 100644 sdk/python/tests/compiler/testdata/volume.py delete mode 100644 sdk/python/tests/compiler/testdata/volume.yaml delete mode 100644 sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py delete mode 100644 sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml delete mode 100644 sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py delete mode 100644 sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml delete mode 100644 sdk/python/tests/compiler/testdata/volumeop_basic.py delete mode 100644 sdk/python/tests/compiler/testdata/volumeop_basic.yaml delete mode 100644 sdk/python/tests/compiler/testdata/volumeop_dag.py delete mode 100644 sdk/python/tests/compiler/testdata/volumeop_dag.yaml delete mode 100644 sdk/python/tests/compiler/testdata/volumeop_parallel.py delete mode 100644 sdk/python/tests/compiler/testdata/volumeop_parallel.yaml delete mode 100644 sdk/python/tests/compiler/testdata/volumeop_sequential.py delete mode 100644 sdk/python/tests/compiler/testdata/volumeop_sequential.yaml delete mode 100644 sdk/python/tests/compiler/testdata/withitem_basic.py delete mode 100644 sdk/python/tests/compiler/testdata/withitem_basic.yaml delete mode 100644 sdk/python/tests/compiler/testdata/withitem_nested.py delete mode 100644 sdk/python/tests/compiler/testdata/withitem_nested.yaml delete mode 100644 sdk/python/tests/compiler/testdata/withparam_global.py delete mode 100644 sdk/python/tests/compiler/testdata/withparam_global.yaml delete mode 100644 sdk/python/tests/compiler/testdata/withparam_global_dict.py delete mode 100644 sdk/python/tests/compiler/testdata/withparam_global_dict.yaml delete mode 100644 sdk/python/tests/compiler/testdata/withparam_output.py delete mode 100644 sdk/python/tests/compiler/testdata/withparam_output.yaml delete mode 100644 sdk/python/tests/compiler/testdata/withparam_output_dict.py delete mode 100644 sdk/python/tests/compiler/testdata/withparam_output_dict.yaml delete mode 100644 sdk/python/tests/dsl/__init__.py delete mode 100644 sdk/python/tests/dsl/aws_extensions_tests.py delete mode 100644 sdk/python/tests/dsl/component_bridge_tests.py delete mode 100644 sdk/python/tests/dsl/component_tests.py delete mode 100644 sdk/python/tests/dsl/extensions/__init__.py delete mode 100644 sdk/python/tests/dsl/extensions/test_kubernetes.py delete mode 100644 sdk/python/tests/dsl/main.py delete mode 100644 sdk/python/tests/dsl/metadata_tests.py delete mode 100644 sdk/python/tests/dsl/ops_group_tests.py delete mode 100644 sdk/python/tests/dsl/pipeline_param_tests.py delete mode 100644 sdk/python/tests/dsl/pipeline_tests.py delete mode 100644 sdk/python/tests/dsl/test_azure_extensions.py delete mode 100644 sdk/python/tests/dsl/type_tests.py delete mode 100644 sdk/python/tests/local_runner_test.py delete mode 100755 sdk/python/tests/run_tests.sh delete mode 100644 sdk/python/tests/test_kfp.py diff --git a/.gitignore b/.gitignore index 62b57bdccef..4474e60c199 100644 --- a/.gitignore +++ b/.gitignore @@ -56,10 +56,6 @@ bazel-* # VSCode .vscode -# test yaml -sdk/python/tests/compiler/pipeline.yaml -sdk/python/tests/compiler/testdata/testpackage/pipeline.yaml - # Test temporary files _artifacts diff --git a/pytest.ini b/pytest.ini index a079fdd1c73..09ff20ca8f5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,2 @@ [pytest] -addopts = --ignore=sdk/python/kfp/deprecated --ignore=sdk/python/kfp/tests testpaths = sdk/python/kfp diff --git a/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb b/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb deleted file mode 100644 index d3ef3fc176f..00000000000 --- a/samples/core/AutoML tables/AutoML Tables - Retail product stockout prediction.ipynb +++ /dev/null @@ -1,164 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Kubeflow Pipelines - Retail Product Stockouts Prediction using AutoML Tables\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Configuration\n", - "\n", - "PROJECT_ID = \"\"\n", - "COMPUTE_REGION = \"us-central1\" # Currently \"us-central1\" is the only region supported by AutoML tables.\n", - "# The bucket must be Regional (not multi-regional) and the region should be us-central1. This is a limitation of the batch prediction service.\n", - "gcs_output_uri_prefix = 'gs:////'" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# AutoML Tables components\n", - "\n", - "from kfp.deprecated.components import load_component_from_url\n", - "\n", - "automl_create_dataset_for_tables_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/create_dataset_for_tables/component.yaml')\n", - "automl_import_data_from_bigquery_source_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/import_data_from_bigquery/component.yaml')\n", - "automl_import_data_from_gcs_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/import_data_from_gcs/component.yaml')\n", - "automl_create_model_for_tables_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/create_model_for_tables/component.yaml')\n", - "automl_prediction_service_batch_predict_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/dc8dc3301c8a590231289cf9537b4dc08089957b/components/gcp/automl/prediction_service_batch_predict/component.yaml')\n", - "automl_split_dataset_table_column_names_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/b3179d86b239a08bf4884b50dbf3a9151da96d66/components/gcp/automl/split_dataset_table_column_names/component.yaml')\n", - "automl_export_model_to_gcs_op = load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/91f2586527c5106c3f72f9d0c3f780fb4fcb7e22/components/gcp/automl/export_model_to_gcs/component.yaml')" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# Define the pipeline\n", - "import kfp.deprecated as kfp\n", - "\n", - "def retail_product_stockout_prediction_pipeline_gcs(\n", - " gcp_project_id: str,\n", - " gcp_region: str,\n", - " gcs_output_uri_prefix: str,\n", - " dataset_gcs_input_uris: list = ['gs://kubeflow-pipelines-regional-us-central1/mirror/cloud-ml-data/automl-tables/notebooks/stockout.csv'],\n", - " dataset_display_name: str = 'stockout_data_gcs',\n", - " target_column_name: str = 'Stockout',\n", - " model_display_name: str = 'stockout_model',\n", - " batch_predict_gcs_input_uris: list = ['gs://kubeflow-pipelines-regional-us-central1/mirror/cloud-ml-data/automl-tables/notebooks/batch_prediction_inputs.csv'],\n", - " train_budget_milli_node_hours: 'Integer' = 1000,\n", - "):\n", - " # Create dataset\n", - " create_dataset_task = automl_create_dataset_for_tables_op(\n", - " gcp_project_id=gcp_project_id,\n", - " gcp_region=gcp_region,\n", - " display_name=dataset_display_name,\n", - " )\n", - "\n", - " # Import data\n", - " import_data_task = automl_import_data_from_gcs_op(\n", - " dataset_path=create_dataset_task.outputs['dataset_path'],\n", - " input_uris=dataset_gcs_input_uris,\n", - " )\n", - " \n", - " # Prepare column schemas\n", - " split_column_specs = automl_split_dataset_table_column_names_op(\n", - " dataset_path=import_data_task.outputs['dataset_path'],\n", - " table_index=0,\n", - " target_column_name=target_column_name,\n", - " )\n", - " \n", - " # Train a model\n", - " create_model_task = automl_create_model_for_tables_op(\n", - " gcp_project_id=gcp_project_id,\n", - " gcp_region=gcp_region,\n", - " display_name=model_display_name,\n", - " dataset_id=create_dataset_task.outputs['dataset_id'],\n", - " target_column_path=split_column_specs.outputs['target_column_path'],\n", - " #input_feature_column_paths=None, # All non-target columns will be used if None is passed\n", - " input_feature_column_paths=split_column_specs.outputs['feature_column_paths'],\n", - " optimization_objective='MAXIMIZE_AU_PRC',\n", - " train_budget_milli_node_hours=train_budget_milli_node_hours,\n", - " ).after(import_data_task)\n", - "\n", - " # Batch prediction\n", - " batch_predict_task = automl_prediction_service_batch_predict_op(\n", - " model_path=create_model_task.outputs['model_path'],\n", - " #bq_input_uri=batch_predict_bq_input_uri,\n", - " gcs_input_uris=batch_predict_gcs_input_uris,\n", - " gcs_output_uri_prefix=gcs_output_uri_prefix,\n", - " )\n", - "\n", - " # Exporting the model\n", - " automl_export_model_to_gcs_op(\n", - " model_path=create_model_task.outputs['model_path'],\n", - " model_format='tf_saved_model',\n", - " gcs_output_uri_prefix=gcs_output_uri_prefix,\n", - " )\n", - "\n", - " # The pipeline should be able to authenticate to GCP.\n", - " # Refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) for details.\n", - " #\n", - " # For example, you may uncomment the following lines to use GSA keys.\n", - " # from kfp.gcp import use_gcp_secret\n", - " # kfp.dsl.get_pipeline_conf().add_op_transformer(use_gcp_secret('user-gcp-sa'))" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "# Run the pipeline\n", - "import json\n", - "\n", - "kfp_endpoint=None\n", - "kfp.Client(host=kfp_endpoint).create_run_from_pipeline_func(\n", - " retail_product_stockout_prediction_pipeline_gcs,\n", - " arguments=dict(\n", - " gcp_project_id=PROJECT_ID,\n", - " gcp_region=COMPUTE_REGION,\n", - " dataset_display_name='stockout_data_gcs2', # Change this every time there is new data\n", - " dataset_gcs_input_uris=json.dumps(['gs://kubeflow-pipelines-regional-us-central1/mirror/cloud-ml-data/automl-tables/notebooks/stockout.csv']),\n", - " batch_predict_gcs_input_uris=json.dumps(['gs://kubeflow-pipelines-regional-us-central1/mirror/cloud-ml-data/automl-tables/notebooks/batch_prediction_inputs.csv']),\n", - " gcs_output_uri_prefix=gcs_output_uri_prefix,\n", - " )\n", - ")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/samples/core/XGBoost/xgboost_sample_test.py b/samples/core/XGBoost/xgboost_sample_test.py index 8f43c17edc5..be3b122b7c3 100644 --- a/samples/core/XGBoost/xgboost_sample_test.py +++ b/samples/core/XGBoost/xgboost_sample_test.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp as kfp +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase + from .xgboost_sample import xgboost_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase run_pipeline_func([ - TestCase( - pipeline_func=xgboost_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - ), + TestCase(pipeline_func=xgboost_pipeline), ]) diff --git a/samples/core/caching/caching_test.py b/samples/core/caching/caching_test.py index a81618e69bf..c8ad398a7b4 100644 --- a/samples/core/caching/caching_test.py +++ b/samples/core/caching/caching_test.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func +from kfp.samples.test.utils import relative_path +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase run_pipeline_func([ TestCase( pipeline_file=relative_path(__file__, 'caching.ipynb'), - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, run_pipeline=False, ), ]) diff --git a/samples/core/condition/condition.py b/samples/core/condition/condition.py deleted file mode 100644 index 0a5f861d470..00000000000 --- a/samples/core/condition/condition.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated import components -from kfp.deprecated import dsl -from kfp.deprecated import compiler - - -def flip_coin(force_flip_result: str = '') -> str: - """Flip a coin and output heads or tails randomly.""" - if force_flip_result: - return force_flip_result - import random - result = 'heads' if random.randint(0, 1) == 0 else 'tails' - return result - - -def print_msg(msg: str): - """Print a message.""" - print(msg) - - -flip_coin_op = components.create_component_from_func(flip_coin) - -print_op = components.create_component_from_func(print_msg) - - -@dsl.pipeline(name='condition') -def condition(text: str = 'condition test', force_flip_result: str = ''): - flip1 = flip_coin_op(force_flip_result) - print_op(flip1.output) - - with dsl.Condition(flip1.output == 'heads'): - flip2 = flip_coin_op() - print_op(flip2.output) - print_op(text) - - -if __name__ == '__main__': - compiler.Compiler().compile(condition, __file__ + '.yaml') diff --git a/samples/core/condition/condition_test.py b/samples/core/condition/condition_test.py index a4551e047a2..f3d6f903fe4 100644 --- a/samples/core/condition/condition_test.py +++ b/samples/core/condition/condition_test.py @@ -15,12 +15,16 @@ from __future__ import annotations import unittest -import kfp.deprecated as kfp + +import kfp +from kfp.samples.test.utils import debug_verify +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase import kfp_server_api from ml_metadata.proto import Execution -from .condition import condition + from .condition_v2 import condition as condition_v2 -from kfp.samples.test.utils import KfpTask, debug_verify, run_pipeline_func, TestCase def verify_heads(t: unittest.TestCase, run: kfp_server_api.ApiRun, @@ -43,24 +47,12 @@ def verify_tails(t: unittest.TestCase, run: kfp_server_api.ApiRun, run_pipeline_func([ TestCase( pipeline_func=condition_v2, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, arguments={"force_flip_result": "heads"}, verify_func=verify_heads, ), TestCase( pipeline_func=condition_v2, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, arguments={"force_flip_result": "tails"}, verify_func=verify_tails, ), - TestCase( - pipeline_func=condition, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - arguments={"force_flip_result": "heads"}, - ), - TestCase( - pipeline_func=condition, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - arguments={"force_flip_result": "tails"}, - ), ]) diff --git a/samples/core/condition/nested_condition_test.py b/samples/core/condition/nested_condition_test.py index d335f350b91..22ab08bca2b 100644 --- a/samples/core/condition/nested_condition_test.py +++ b/samples/core/condition/nested_condition_test.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp as kfp +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase + from .nested_condition import my_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase run_pipeline_func([ - TestCase( - pipeline_func=my_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - ), + TestCase(pipeline_func=my_pipeline), ]) diff --git a/samples/core/continue_training_from_prod/continue_training_from_prod.py b/samples/core/continue_training_from_prod/continue_training_from_prod.py deleted file mode 100644 index 10be8035c08..00000000000 --- a/samples/core/continue_training_from_prod/continue_training_from_prod.py +++ /dev/null @@ -1,167 +0,0 @@ -# This sample demonstrates a common training scenario. -# New models are being trained strarting from the production model (if it exists). -# This sample produces two runs: -# 1. The trainer will train the model from scratch and set as prod after testing it -# 2. Exact same configuration, but the pipeline will discover the existing prod model (published by the 1st run) and warm-start the training from it. - - -# GCS URI of a directory where the models and the model pointers should be be stored. -model_dir_uri='gs:///' -kfp_endpoint=None - - -import kfp.deprecated as kfp -from kfp.deprecated import components - - -chicago_taxi_dataset_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/e3337b8bdcd63636934954e592d4b32c95b49129/components/datasets/Chicago%20Taxi/component.yaml') -xgboost_train_on_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Train/component.yaml') -xgboost_predict_on_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/567c04c51ff00a1ee525b3458425b17adbe3df61/components/XGBoost/Predict/component.yaml') - -pandas_transform_csv_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/6162d55998b176b50267d351241100bb0ee715bc/components/pandas/Transform_DataFrame/in_CSV_format/component.yaml') -drop_header_op = kfp.components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/02c9638287468c849632cf9f7885b51de4c66f86/components/tables/Remove_header/component.yaml') -calculate_regression_metrics_from_csv_op = kfp.components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/616542ac0f789914f4eb53438da713dd3004fba4/components/ml_metrics/Calculate_regression_metrics/from_CSV/component.yaml') - -download_from_gcs_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/5c7593f18f347f1c03f5ae6778a1ff305abc315c/components/google-cloud/storage/download/component.yaml') -upload_to_gcs_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/616542ac0f789914f4eb53438da713dd3004fba4/components/google-cloud/storage/upload_to_explicit_uri/component.yaml') -upload_to_gcs_unique_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/616542ac0f789914f4eb53438da713dd3004fba4/components/google-cloud/storage/upload_to_unique_uri/component.yaml') - - -def continuous_training_pipeline( - model_dir_uri, - training_start_date: str = '2019-02-01', - training_end_date: str = '2019-03-01', - testing_start_date: str = '2019-01-01', - testing_end_date: str = '2019-02-01', -): - # Preparing the training and testing data - training_data = chicago_taxi_dataset_op( - where='trip_start_timestamp >= "{}" AND trip_start_timestamp < "{}"'.format(str(training_start_date), str(training_end_date)), - select='tips,trip_seconds,trip_miles,pickup_community_area,dropoff_community_area,fare,tolls,extras,trip_total', - limit=10000, - ).set_display_name('Training data').output - - testing_data = chicago_taxi_dataset_op( - where='trip_start_timestamp >= "{}" AND trip_start_timestamp < "{}"'.format(str(testing_start_date), str(testing_end_date)), - select='tips,trip_seconds,trip_miles,pickup_community_area,dropoff_community_area,fare,tolls,extras,trip_total', - limit=10000, - ).set_display_name('Testing data').output - - # Preparing the true values for the testing data - true_values_table = pandas_transform_csv_op( - table=testing_data, - transform_code='''df = df[["tips"]]''', - ).set_display_name('True values').output - - true_values = drop_header_op(true_values_table).output - - # Getting the active prod model - prod_model_pointer_uri = str(model_dir_uri) + 'prod' - get_prod_model_uri_task = download_from_gcs_op( - gcs_path=prod_model_pointer_uri, - default_data='', - ).set_display_name('Get prod model') - # Disabling cache reuse to always get new data - get_prod_model_uri_task.execution_options.caching_strategy.max_cache_staleness = 'P0D' - prod_model_uri = get_prod_model_uri_task.output - - # Training new model from scratch - with kfp.dsl.Condition(prod_model_uri == ""): - # Training - model = xgboost_train_on_csv_op( - training_data=training_data, - label_column=0, - objective='reg:squarederror', - num_iterations=400, - ).outputs['model'] - - # Predicting - predictions = xgboost_predict_on_csv_op( - data=testing_data, - model=model, - label_column=0, - ).output - - # Calculating the regression metrics - metrics_task = calculate_regression_metrics_from_csv_op( - true_values=true_values, - predicted_values=predictions, - ) - - # Checking the metrics - with kfp.dsl.Condition(metrics_task.outputs['mean_squared_error'] < 2.0): - # Uploading the model - model_uri = upload_to_gcs_unique_op( - data=model, - gcs_path_prefix=model_dir_uri, - ).set_display_name('Upload model').output - - # Setting the model as prod - upload_to_gcs_op( - data=model_uri, - gcs_path=prod_model_pointer_uri, - ).set_display_name('Set prod model') - - # Training new model starting from the prod model - with kfp.dsl.Condition(prod_model_uri != ""): - # Downloading the model - prod_model = download_from_gcs_op(prod_model_uri).output - - # Training - model = xgboost_train_on_csv_op( - training_data=training_data, - starting_model=prod_model, - label_column=0, - objective='reg:squarederror', - num_iterations=100, - ).outputs['model'] - - # Predicting - predictions = xgboost_predict_on_csv_op( - data=testing_data, - model=model, - label_column=0, - ).output - - # Calculating the regression metrics - metrics_task = calculate_regression_metrics_from_csv_op( - true_values=true_values, - predicted_values=predictions, - ) - - # Checking the metrics - with kfp.dsl.Condition(metrics_task.outputs['mean_squared_error'] < 2.0): - # Uploading the model - model_uri = upload_to_gcs_unique_op( - data=model, - gcs_path_prefix=model_dir_uri, - ).set_display_name('Upload model').output - - # Setting the model as prod - upload_to_gcs_op( - data=model_uri, - gcs_path=prod_model_pointer_uri, - ).set_display_name('Set prod model') - - -if __name__ == '__main__': - # Running the first time. The trainer will train the model from scratch and set as prod after testing it - pipelin_run = kfp.Client(host=kfp_endpoint).create_run_from_pipeline_func( - continuous_training_pipeline, - arguments=dict( - model_dir_uri=model_dir_uri, - training_start_date='2019-02-01', - training_end_date='2019-03-01', - ), - ) - pipelin_run.wait_for_run_completion() - - # Running the second time. The trainer should warm-start the training from the prod model and set the new model as prod after testing it - kfp.Client(host=kfp_endpoint).create_run_from_pipeline_func( - continuous_training_pipeline, - arguments=dict( - model_dir_uri=model_dir_uri, - training_start_date='2019-02-01', - training_end_date='2019-03-01', - ), - ) diff --git a/samples/core/dataflow/dataflow.ipynb b/samples/core/dataflow/dataflow.ipynb deleted file mode 100644 index 5d3fd4e572c..00000000000 --- a/samples/core/dataflow/dataflow.ipynb +++ /dev/null @@ -1,462 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# GCP Dataflow Component Sample\n", - "A Kubeflow Pipeline component that prepares data by submitting an Apache Beam job (authored in Python) to Cloud Dataflow for execution. The Python Beam code is run with Cloud Dataflow Runner.\n", - "\n", - "## Intended use\n", - "\n", - "Use this component to run a Python Beam code to submit a Cloud Dataflow job as a step of a Kubeflow pipeline. \n", - "\n", - "## Runtime arguments\n", - "Name | Description | Optional | Data type| Accepted values | Default |\n", - ":--- | :----------| :----------| :----------| :----------| :---------- |\n", - "python_file_path | The path to the Cloud Storage bucket or local directory containing the Python file to be run. | | GCSPath | | |\n", - "project_id | The ID of the Google Cloud Platform (GCP) project containing the Cloud Dataflow job.| | String | | |\n", - "region | The Google Cloud Platform (GCP) region to run the Cloud Dataflow job.| | String | | |\n", - "staging_dir | The path to the Cloud Storage directory where the staging files are stored. A random subdirectory will be created under the staging directory to keep the job information.This is done so that you can resume the job in case of failure. `staging_dir` is passed as the command line arguments (`staging_location` and `temp_location`) of the Beam code. | Yes | GCSPath | | None |\n", - "requirements_file_path | The path to the Cloud Storage bucket or local directory containing the pip requirements file. | Yes | GCSPath | | None |\n", - "args | The list of arguments to pass to the Python file. | No | List | A list of string arguments | None |\n", - "wait_interval | The number of seconds to wait between calls to get the status of the job. | Yes | Integer | | 30 |\n", - "\n", - "## Input data schema\n", - "\n", - "Before you use the component, the following files must be ready in a Cloud Storage bucket:\n", - "- A Beam Python code file.\n", - "- A `requirements.txt` file which includes a list of dependent packages.\n", - "\n", - "The Beam Python code should follow the [Beam programming guide](https://beam.apache.org/documentation/programming-guide/) as well as the following additional requirements to be compatible with this component:\n", - "- It accepts the command line arguments `--project`, `--region`, `--temp_location`, `--staging_location`, which are [standard Dataflow Runner options](https://cloud.google.com/dataflow/docs/guides/specifying-exec-params#setting-other-cloud-pipeline-options).\n", - "- It enables `info logging` before the start of a Cloud Dataflow job in the Python code. This is important to allow the component to track the status and ID of the job that is created. For example, calling `logging.getLogger().setLevel(logging.INFO)` before any other code.\n", - "\n", - "\n", - "## Output\n", - "Name | Description\n", - ":--- | :----------\n", - "job_id | The id of the Cloud Dataflow job that is created.\n", - "\n", - "## Cautions & requirements\n", - "To use the components, the following requirements must be met:\n", - "- Cloud Dataflow API is enabled.\n", - "- The component is running under a secret Kubeflow user service account in a Kubeflow Pipeline cluster. For example:\n", - "```\n", - "component_op(...)\n", - "```\n", - "The Kubeflow user service account is a member of:\n", - "- `roles/dataflow.developer` role of the project.\n", - "- `roles/storage.objectViewer` role of the Cloud Storage Objects `python_file_path` and `requirements_file_path`.\n", - "- `roles/storage.objectCreator` role of the Cloud Storage Object `staging_dir`. \n", - "\n", - "## Detailed description\n", - "The component does several things during the execution:\n", - "- Downloads `python_file_path` and `requirements_file_path` to local files.\n", - "- Starts a subprocess to launch the Python program.\n", - "- Monitors the logs produced from the subprocess to extract the Cloud Dataflow job information.\n", - "- Stores the Cloud Dataflow job information in `staging_dir` so the job can be resumed in case of failure.\n", - "- Waits for the job to finish.\n", - "\n", - "# Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - }, - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "project = 'Input your PROJECT ID'\n", - "region = 'Input GCP region' # For example, 'us-central1'\n", - "output = 'Input your GCS bucket name' # No ending slash" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "## Install Pipeline SDK" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "tags": [ - "skip-in-test" - ] - }, - "outputs": [], - "source": [ - "!python3 -m pip install 'kfp>=0.1.31' --quiet" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "source": [ - "\n", - "## Load the component using KFP SDK\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "import kfp.deprecated.components as comp\n", - "\n", - "dataflow_python_op = comp.load_component_from_url(\n", - " 'https://raw.githubusercontent.com/kubeflow/pipelines/1.7.0-rc.3/components/gcp/dataflow/launch_python/component.yaml')" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "tags": [ - "skip-in-test" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Help on function Launch Python:\n", - "\n", - "Launch Python(python_file_path: str, project_id: str, region: str, staging_dir: 'GCSPath' = '', requirements_file_path: 'GCSPath' = '', args: list = '[]', wait_interval: int = '30')\n", - " Launch Python\n", - " Launch a self-executing beam python file.\n", - "\n" - ] - } - ], - "source": [ - "help(dataflow_python_op)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Use the wordcount python sample\n", - "In this sample, we run a wordcount sample code in a Kubeflow Pipeline. The output will be stored in a Cloud Storage bucket. Here is the sample code:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "tags": [ - "skip-in-test" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "#\r\n", - "# Licensed to the Apache Software Foundation (ASF) under one or more\r\n", - "# contributor license agreements. See the NOTICE file distributed with\r\n", - "# this work for additional information regarding copyright ownership.\r\n", - "# The ASF licenses this file to You under the Apache License, Version 2.0\r\n", - "# (the \"License\"); you may not use this file except in compliance with\r\n", - "# the License. You may obtain a copy of the License at\r\n", - "#\r\n", - "# http://www.apache.org/licenses/LICENSE-2.0\r\n", - "#\r\n", - "# Unless required by applicable law or agreed to in writing, software\r\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\r\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\r\n", - "# See the License for the specific language governing permissions and\r\n", - "# limitations under the License.\r\n", - "#\r\n", - "\r\n", - "\"\"\"A minimalist word-counting workflow that counts words in Shakespeare.\r\n", - "\r\n", - "This is the first in a series of successively more detailed 'word count'\r\n", - "examples.\r\n", - "\r\n", - "Next, see the wordcount pipeline, then the wordcount_debugging pipeline, for\r\n", - "more detailed examples that introduce additional concepts.\r\n", - "\r\n", - "Concepts:\r\n", - "\r\n", - "1. Reading data from text files\r\n", - "2. Specifying 'inline' transforms\r\n", - "3. Counting a PCollection\r\n", - "4. Writing data to Cloud Storage as text files\r\n", - "\r\n", - "To execute this pipeline locally, first edit the code to specify the output\r\n", - "location. Output location could be a local file path or an output prefix\r\n", - "on GCS. (Only update the output location marked with the first CHANGE comment.)\r\n", - "\r\n", - "To execute this pipeline remotely, first edit the code to set your project ID,\r\n", - "runner type, the staging location, the temp location, and the output location.\r\n", - "The specified GCS bucket(s) must already exist. (Update all the places marked\r\n", - "with a CHANGE comment.)\r\n", - "\r\n", - "Then, run the pipeline as described in the README. It will be deployed and run\r\n", - "using the Google Cloud Dataflow Service. No args are required to run the\r\n", - "pipeline. You can see the results in your output bucket in the GCS browser.\r\n", - "\"\"\"\r\n", - "\r\n", - "from __future__ import absolute_import\r\n", - "\r\n", - "import argparse\r\n", - "import logging\r\n", - "import re\r\n", - "\r\n", - "from past.builtins import unicode\r\n", - "\r\n", - "import apache_beam as beam\r\n", - "from apache_beam.io import ReadFromText\r\n", - "from apache_beam.io import WriteToText\r\n", - "from apache_beam.options.pipeline_options import PipelineOptions\r\n", - "from apache_beam.options.pipeline_options import SetupOptions\r\n", - "\r\n", - "\r\n", - "def run(argv=None):\r\n", - " \"\"\"Main entry point; defines and runs the wordcount pipeline.\"\"\"\r\n", - "\r\n", - " parser = argparse.ArgumentParser()\r\n", - " parser.add_argument('--input',\r\n", - " dest='input',\r\n", - " default='gs://dataflow-samples/shakespeare/kinglear.txt',\r\n", - " help='Input file to process.')\r\n", - " parser.add_argument('--output',\r\n", - " dest='output',\r\n", - " # CHANGE 1/6: The Google Cloud Storage path is required\r\n", - " # for outputting the results.\r\n", - " default='gs://YOUR_OUTPUT_BUCKET/AND_OUTPUT_PREFIX',\r\n", - " help='Output file to write results to.')\r\n", - " known_args, pipeline_args = parser.parse_known_args(argv)\r\n", - " # pipeline_args.extend([\r\n", - " # # CHANGE 2/6: (OPTIONAL) Change this to DataflowRunner to\r\n", - " # # run your pipeline on the Google Cloud Dataflow Service.\r\n", - " # '--runner=DirectRunner',\r\n", - " # # CHANGE 3/6: Your project ID is required in order to run your pipeline on\r\n", - " # # the Google Cloud Dataflow Service.\r\n", - " # '--project=SET_YOUR_PROJECT_ID_HERE',\r\n", - " # # CHANGE 4/6: A GCP region is required in order to run your pipeline on\r\n", - " # # the Google Cloud Dataflow Service.\r\n", - " # '--region=SET_GCP_REGION_HERE',\r\n", - " # # CHANGE 5/6: Your Google Cloud Storage path is required for staging local\r\n", - " # # files.\r\n", - " # '--staging_location=gs://YOUR_BUCKET_NAME/AND_STAGING_DIRECTORY',\r\n", - " # # CHANGE 6/6: Your Google Cloud Storage path is required for temporary\r\n", - " # # files.\r\n", - " # '--temp_location=gs://YOUR_BUCKET_NAME/AND_TEMP_DIRECTORY',\r\n", - " # '--job_name=your-wordcount-job',\r\n", - " # ])\r\n", - "\r\n", - " # We use the save_main_session option because one or more DoFn's in this\r\n", - " # workflow rely on global context (e.g., a module imported at module level).\r\n", - " pipeline_options = PipelineOptions(pipeline_args)\r\n", - " pipeline_options.view_as(SetupOptions).save_main_session = True\r\n", - " with beam.Pipeline(options=pipeline_options) as p:\r\n", - "\r\n", - " # Read the text file[pattern] into a PCollection.\r\n", - " lines = p | ReadFromText(known_args.input)\r\n", - "\r\n", - " # Count the occurrences of each word.\r\n", - " counts = (\r\n", - " lines\r\n", - " | 'Split' >> (beam.FlatMap(lambda x: re.findall(r'[A-Za-z\\']+', x))\r\n", - " .with_output_types(unicode))\r\n", - " | 'PairWithOne' >> beam.Map(lambda x: (x, 1))\r\n", - " | 'GroupAndSum' >> beam.CombinePerKey(sum))\r\n", - "\r\n", - " # Format the counts into a PCollection of strings.\r\n", - " def format_result(word_count):\r\n", - " (word, count) = word_count\r\n", - " return '%s: %s' % (word, count)\r\n", - "\r\n", - " output = counts | 'Format' >> beam.Map(format_result)\r\n", - "\r\n", - " # Write the output using a \"Write\" transform that has side effects.\r\n", - " # pylint: disable=expression-not-assigned\r\n", - " output | WriteToText(known_args.output)\r\n", - "\r\n", - "\r\n", - "if __name__ == '__main__':\r\n", - " logging.getLogger().setLevel(logging.INFO)\r\n", - " run()\r\n" - ] - } - ], - "source": [ - "!gsutil cat gs://ml-pipeline/sample-pipeline/word-count/wc.py" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Example pipeline that uses the component" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "import kfp.deprecated as kfp\n", - "from kfp.deprecated import dsl, Client\n", - "import json\n", - "@dsl.pipeline(\n", - " name='dataflow-launch-python-pipeline',\n", - " description='Dataflow launch python pipeline'\n", - ")\n", - "def pipeline(\n", - " python_file_path = 'gs://ml-pipeline/sample-pipeline/word-count/wc.py',\n", - " project_id = project,\n", - " region = region,\n", - " staging_dir = output,\n", - " requirements_file_path = 'gs://ml-pipeline/sample-pipeline/word-count/requirements.txt',\n", - " wait_interval = 30\n", - "):\n", - " dataflow_python_op(\n", - " python_file_path = python_file_path, \n", - " project_id = project_id, \n", - " region = region, \n", - " staging_dir = staging_dir, \n", - " requirements_file_path = requirements_file_path, \n", - " args = json.dumps(['--output', f'{staging_dir}/wc/wordcount.out']),\n", - " wait_interval = wait_interval)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Submit the pipeline for execution" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "tags": [ - "skip-in-test" - ] - }, - "outputs": [ - { - "data": { - "text/html": [ - "Experiment link here" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Run link here" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "Client().create_run_from_pipeline_func(pipeline, arguments={})" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Inspect the output" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "skip-in-test" - ] - }, - "outputs": [], - "source": [ - "!gsutil cat $output/wc/wordcount.out" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## References\n", - "* [Component python code](https://github.com/kubeflow/pipelines/blob/release-1.7/components/gcp/container/component_sdk/python/kfp_component/google/dataflow/_launch_python.py)\n", - "* [Component docker file](https://github.com/kubeflow/pipelines/blob/release-1.7/components/gcp/container/Dockerfile)\n", - "* [Sample notebook](https://github.com/kubeflow/pipelines/blob/release-1.7/components/gcp/dataflow/launch_python/sample.ipynb)\n", - "* [Dataflow Python Quickstart](https://cloud.google.com/dataflow/docs/quickstarts/quickstart-python)" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "c7a91a0fef823c7f839350126c5e355ea393d05f89cb40a046ebac9c8851a521" - }, - "kernelspec": { - "display_name": "Python 3.7.10 64-bit ('v2': conda)", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/samples/core/dataflow/dataflow_test.py b/samples/core/dataflow/dataflow_test.py deleted file mode 100644 index 3f02123e482..00000000000 --- a/samples/core/dataflow/dataflow_test.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'dataflow.ipynb'), - arguments={ - 'project_id': 'kfp-ci', - 'staging_dir': 'gs://kfp-ci/samples/dataflow', - 'region': 'us-central1', - }, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - timeout_mins=40, - ), -]) diff --git a/samples/core/dns_config/dns_config.py b/samples/core/dns_config/dns_config.py deleted file mode 100644 index 7621faf0298..00000000000 --- a/samples/core/dns_config/dns_config.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import kfp.deprecated as kfp -from kfp.deprecated import dsl, compiler -from kubernetes.client.models import V1PodDNSConfig, V1PodDNSConfigOption - - -def echo_op(): - return dsl.ContainerOp( - name='echo', - image='library/bash:4.4.23', - command=['sh', '-c'], - arguments=['echo "hello world"'] - ) - - -@dsl.pipeline( - name='dns-config-setting', - description='Passes dnsConfig setting to workflow.' -) -def dns_config_pipeline(): - echo_task = echo_op() - - -if __name__ == '__main__': - pipeline_conf = kfp.dsl.PipelineConf() - pipeline_conf.set_dns_config(dns_config=V1PodDNSConfig( - nameservers=["1.2.3.4"], - options=[V1PodDNSConfigOption(name="ndots", value="2")] - )) - - compiler.Compiler().compile( - dns_config_pipeline, - __file__ + '.yaml', - pipeline_conf=pipeline_conf - ) diff --git a/samples/core/dsl_static_type_checking/dsl_static_type_checking.ipynb b/samples/core/dsl_static_type_checking/dsl_static_type_checking.ipynb deleted file mode 100644 index 2aa1f40b02c..00000000000 --- a/samples/core/dsl_static_type_checking/dsl_static_type_checking.ipynb +++ /dev/null @@ -1,814 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# KubeFlow Pipeline DSL Static Type Checking\n", - "\n", - "In this notebook, we will demo: \n", - "\n", - "* Defining a KubeFlow pipeline with Python DSL\n", - "* Compile the pipeline with type checking\n", - "\n", - "Static type checking helps users to identify component I/O inconsistencies without running the pipeline. It also shortens the development cycles by catching the errors early. This feature is especially useful in two cases: 1) when the pipeline is huge and manually checking the types is infeasible; 2) when some components are shared ones and the type information is not immediately avaiable to the pipeline authors.\n", - "\n", - "Since this sample focuses on the DSL type checking, we will use components that are not runnable in the system but with various type checking scenarios. \n", - "\n", - "## Component definition\n", - "Components can be defined in either YAML or functions decorated by dsl.component.\n", - "\n", - "## Type definition\n", - "Types can be defined as string or a dictionary with the openapi_schema_validator property formatted as:\n", - "```yaml\n", - "{\n", - " type_name: {\n", - " openapi_schema_validator: {\n", - " }\n", - " }\n", - "}\n", - "```\n", - "For example, the following yaml declares a GCSPath type with the openapi_schema_validator for output field_m.\n", - "The type could also be a plain string, such as the GcsUri. The type name could be either one of the core types or customized ones.\n", - "```yaml\n", - "name: component a\n", - "description: component a desc\n", - "inputs:\n", - " - {name: field_l, type: Integer}\n", - "outputs:\n", - " - {name: field_m, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: \"^gs://.*$\" } }}}\n", - " - {name: field_n, type: customized_type}\n", - " - {name: field_o, type: GcsUri} \n", - "implementation:\n", - " container:\n", - " image: gcr.io/ml-pipeline/component-a\n", - " command: [python3, /pipelines/component/src/train.py]\n", - " args: [\n", - " --field-l, {inputValue: field_l},\n", - " ]\n", - " fileOutputs: \n", - " field_m: /schema.txt\n", - " field_n: /feature.txt\n", - " field_o: /output.txt\n", - "```\n", - "\n", - "If you define the component using the function decorator, there are a list of [core types](https://github.com/kubeflow/pipelines/blob/master/sdk/python/kfp/dsl/types.py).\n", - "For example, the following component declares a core type Integer for input field_l while\n", - "declares customized_type for its output field_n.\n", - "\n", - "```python\n", - "@component\n", - "def task_factory_a(field_l: Integer()) -> {'field_m': {'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gs://.*$\"}'}}, \n", - " 'field_n': 'customized_type',\n", - " 'field_o': 'Integer'\n", - " }:\n", - " return ContainerOp(\n", - " name = 'operator a',\n", - " image = 'gcr.io/ml-pipeline/component-a',\n", - " arguments = [\n", - " '--field-l', field_l,\n", - " ],\n", - " file_outputs = {\n", - " 'field_m': '/schema.txt',\n", - " 'field_n': '/feature.txt',\n", - " 'field_o': '/output.txt'\n", - " }\n", - " )\n", - "```\n", - "\n", - "## Type check switch\n", - "Type checking is enabled by default. It can be disabled as --disable-type-check argument if dsl-compile is run in the command line, or `dsl.compiler.Compiler().compile(type_check=False)`.\n", - "\n", - "If one wants to ignore the type for one parameter, call ignore_type() function in [PipelineParam](https://github.com/kubeflow/pipelines/blob/master/sdk/python/kfp/dsl/_pipeline_param.py).\n", - "\n", - "## How does type checking work?\n", - "DSL compiler checks the type consistencies among components by checking the type_name as well as the openapi_schema_validator. Some special cases are listed here:\n", - "1. Type checking succeed: If the upstream/downstream components lack the type information.\n", - "2. Type checking succeed: If the type check is disabled.\n", - "3. Type checking succeed: If the parameter type is ignored." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Setup" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "scrolled": false - }, - "source": [ - "## Install Pipeline SDK" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "tags": [ - "skip-in-test" - ] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting https://storage.googleapis.com/ml-pipeline/release/0.1.12/kfp-experiment.tar.gz\n", - " Using cached https://storage.googleapis.com/ml-pipeline/release/0.1.12/kfp-experiment.tar.gz\n", - "Requirement already satisfied, skipping upgrade: urllib3>=1.15 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.22)\n", - "Requirement already satisfied, skipping upgrade: six>=1.10 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.11.0)\n", - "Requirement already satisfied, skipping upgrade: certifi in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (2018.11.29)\n", - "Requirement already satisfied, skipping upgrade: python-dateutil in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (2.7.5)\n", - "Requirement already satisfied, skipping upgrade: PyYAML in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (3.13)\n", - "Requirement already satisfied, skipping upgrade: google-cloud-storage==1.13.0 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.13.0)\n", - "Requirement already satisfied, skipping upgrade: kubernetes==8.0.0 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (8.0.0)\n", - "Requirement already satisfied, skipping upgrade: PyJWT==1.6.4 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.6.4)\n", - "Requirement already satisfied, skipping upgrade: cryptography==2.4.2 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (2.4.2)\n", - "Requirement already satisfied, skipping upgrade: google-auth==1.6.1 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (1.6.1)\n", - "Requirement already satisfied, skipping upgrade: requests_toolbelt==0.8.0 in /opt/conda/lib/python3.6/site-packages (from kfp==0.1) (0.8.0)\n", - "Requirement already satisfied, skipping upgrade: google-resumable-media>=0.3.1 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage==1.13.0->kfp==0.1) (0.3.1)\n", - "Requirement already satisfied, skipping upgrade: google-cloud-core<0.29dev,>=0.28.0 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage==1.13.0->kfp==0.1) (0.28.1)\n", - "Requirement already satisfied, skipping upgrade: google-api-core<2.0.0dev,>=0.1.1 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage==1.13.0->kfp==0.1) (1.6.0)\n", - "Requirement already satisfied, skipping upgrade: adal>=1.0.2 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (1.2.1)\n", - "Requirement already satisfied, skipping upgrade: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (0.54.0)\n", - "Requirement already satisfied, skipping upgrade: requests-oauthlib in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (1.0.0)\n", - "Requirement already satisfied, skipping upgrade: setuptools>=21.0.0 in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (38.4.0)\n", - "Requirement already satisfied, skipping upgrade: requests in /opt/conda/lib/python3.6/site-packages (from kubernetes==8.0.0->kfp==0.1) (2.18.4)\n", - "Requirement already satisfied, skipping upgrade: asn1crypto>=0.21.0 in /opt/conda/lib/python3.6/site-packages (from cryptography==2.4.2->kfp==0.1) (0.24.0)\n", - "Requirement already satisfied, skipping upgrade: idna>=2.1 in /opt/conda/lib/python3.6/site-packages (from cryptography==2.4.2->kfp==0.1) (2.6)\n", - "Requirement already satisfied, skipping upgrade: cffi!=1.11.3,>=1.7 in /opt/conda/lib/python3.6/site-packages (from cryptography==2.4.2->kfp==0.1) (1.11.4)\n", - "Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.6/site-packages (from google-auth==1.6.1->kfp==0.1) (0.2.2)\n", - "Requirement already satisfied, skipping upgrade: rsa>=3.1.4 in /opt/conda/lib/python3.6/site-packages (from google-auth==1.6.1->kfp==0.1) (4.0)\n", - "Requirement already satisfied, skipping upgrade: cachetools>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from google-auth==1.6.1->kfp==0.1) (3.0.0)\n", - "Requirement already satisfied, skipping upgrade: pytz in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=0.1.1->google-cloud-storage==1.13.0->kfp==0.1) (2018.7)\n", - "Requirement already satisfied, skipping upgrade: googleapis-common-protos!=1.5.4,<2.0dev,>=1.5.3 in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=0.1.1->google-cloud-storage==1.13.0->kfp==0.1) (1.5.5)\n", - "Requirement already satisfied, skipping upgrade: protobuf>=3.4.0 in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=0.1.1->google-cloud-storage==1.13.0->kfp==0.1) (3.6.1)\n", - "Requirement already satisfied, skipping upgrade: oauthlib>=0.6.2 in /opt/conda/lib/python3.6/site-packages (from requests-oauthlib->kubernetes==8.0.0->kfp==0.1) (2.1.0)\n", - "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->kubernetes==8.0.0->kfp==0.1) (3.0.4)\n", - "Requirement already satisfied, skipping upgrade: pycparser in /opt/conda/lib/python3.6/site-packages (from cffi!=1.11.3,>=1.7->cryptography==2.4.2->kfp==0.1) (2.18)\n", - "Requirement already satisfied, skipping upgrade: pyasn1<0.5.0,>=0.4.1 in /opt/conda/lib/python3.6/site-packages (from pyasn1-modules>=0.2.1->google-auth==1.6.1->kfp==0.1) (0.4.4)\n", - "Building wheels for collected packages: kfp\n", - " Running setup.py bdist_wheel for kfp ... \u001b[?25ldone\n", - "\u001b[?25h Stored in directory: /home/jovyan/.cache/pip/wheels/06/14/fc/dd58bcc821d8067efa74a9e217db214d8a075c6b5d31ff24cf\n", - "Successfully built kfp\n", - "Installing collected packages: kfp\n", - " Found existing installation: kfp 0.1\n", - " Uninstalling kfp-0.1:\n", - " Successfully uninstalled kfp-0.1\n", - "Successfully installed kfp-0.1\n", - "\u001b[33mYou are using pip version 18.1, however version 19.0.3 is available.\n", - "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n" - ] - } - ], - "source": [ - "!python3 -m pip install 'kfp>=0.1.31' --quiet\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Type Check with YAML components: successful scenario" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Author components in YAML" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# In yaml, one can optionally add the type information to both inputs and outputs.\n", - "# There are two ways to define the types: string or a dictionary with the openapi_schema_validator property.\n", - "# The openapi_schema_validator is a json schema object that describes schema of the parameter value.\n", - "component_a = '''\\\n", - "name: component a\n", - "description: component a desc\n", - "inputs:\n", - " - {name: field_l, type: Integer}\n", - "outputs:\n", - " - {name: field_m, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: \"^gs://.*$\" } }}}\n", - " - {name: field_n, type: customized_type}\n", - " - {name: field_o, type: GcsUri} \n", - "implementation:\n", - " container:\n", - " image: gcr.io/ml-pipeline/component-a\n", - " command: [python3, /pipelines/component/src/train.py]\n", - " args: [\n", - " --field-l, {inputValue: field_l},\n", - " ]\n", - " fileOutputs: \n", - " field_m: /schema.txt\n", - " field_n: /feature.txt\n", - " field_o: /output.txt\n", - "'''\n", - "component_b = '''\\\n", - "name: component b\n", - "description: component b desc\n", - "inputs:\n", - " - {name: field_x, type: customized_type}\n", - " - {name: field_y, type: GcsUri}\n", - " - {name: field_z, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: \"^gs://.*$\" } }}}\n", - "outputs:\n", - " - {name: output_model_uri, type: GcsUri}\n", - "implementation:\n", - " container:\n", - " image: gcr.io/ml-pipeline/component-a\n", - " command: [python3]\n", - " args: [\n", - " --field-x, {inputValue: field_x},\n", - " --field-y, {inputValue: field_y},\n", - " --field-z, {inputValue: field_z},\n", - " ]\n", - " fileOutputs: \n", - " output_model_uri: /schema.txt\n", - "'''" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Author a pipeline with the above components" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from kfp.deprecated import components as comp\n", - "from kfp.deprecated import dsl\n", - "from kfp.deprecated import compiler\n", - "\n", - "# The components are loaded as task factories that generate container_ops.\n", - "task_factory_a = comp.load_component_from_text(text=component_a)\n", - "task_factory_b = comp.load_component_from_text(text=component_b)\n", - "\n", - "#Use the component as part of the pipeline\n", - "@dsl.pipeline(name='type-check-a',\n", - " description='')\n", - "def pipeline_a():\n", - " a = task_factory_a(field_l=12)\n", - " b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m'])\n", - "\n", - "compiler.Compiler().compile(pipeline_a, 'pipeline_a.zip', type_check=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Type Check with YAML components: failed scenario" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Author components in YAML" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "# In this case, the component_a contains an output field_o as GcrUri \n", - "# but the component_b requires an input field_y as GcsUri\n", - "component_a = '''\\\n", - "name: component a\n", - "description: component a desc\n", - "inputs:\n", - " - {name: field_l, type: Integer}\n", - "outputs:\n", - " - {name: field_m, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: \"^gs://.*$\" } }}}\n", - " - {name: field_n, type: customized_type}\n", - " - {name: field_o, type: GcrUri} \n", - "implementation:\n", - " container:\n", - " image: gcr.io/ml-pipeline/component-a\n", - " command: [python3, /pipelines/component/src/train.py]\n", - " args: [\n", - " --field-l, {inputValue: field_l},\n", - " ]\n", - " fileOutputs: \n", - " field_m: /schema.txt\n", - " field_n: /feature.txt\n", - " field_o: /output.txt\n", - "'''\n", - "component_b = '''\\\n", - "name: component b\n", - "description: component b desc\n", - "inputs:\n", - " - {name: field_x, type: customized_type}\n", - " - {name: field_y, type: GcsUri}\n", - " - {name: field_z, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: \"^gs://.*$\" } }}}\n", - "outputs:\n", - " - {name: output_model_uri, type: GcsUri}\n", - "implementation:\n", - " container:\n", - " image: gcr.io/ml-pipeline/component-a\n", - " command: [python3]\n", - " args: [\n", - " --field-x, {inputValue: field_x},\n", - " --field-y, {inputValue: field_y},\n", - " --field-z, {inputValue: field_z},\n", - " ]\n", - " fileOutputs: \n", - " output_model_uri: /schema.txt\n", - "'''" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Author a pipeline with the above components" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "type name GcrUri is different from expected: GcsUri\n", - "Component \"component b\" is expecting field_y to be type(GcsUri), but the passed argument is type(GcrUri)\n" - ] - } - ], - "source": [ - "from kfp.deprecated import components\n", - "from kfp.deprecated import dsl\n", - "from kfp.deprecated import compiler\n", - "from kfp.deprecated.dsl.types import InconsistentTypeException\n", - "\n", - "task_factory_a = components.load_component_from_text(text=component_a)\n", - "task_factory_b = components.load_component_from_text(text=component_b)\n", - "\n", - "#Use the component as part of the pipeline\n", - "@dsl.pipeline(name='type-check-b',\n", - " description='')\n", - "def pipeline_b():\n", - " a = task_factory_a(field_l=12)\n", - " b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m'])\n", - "\n", - "try:\n", - " compiler.Compiler().compile(pipeline_b, 'pipeline_b.zip', type_check=True)\n", - "except InconsistentTypeException as e:\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Author a pipeline with the above components but type checking disabled." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "# Disable the type_check\n", - "compiler.Compiler().compile(pipeline_b, 'pipeline_b.zip', type_check=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Type Check with decorated components: successful scenario" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Author components with decorator" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "from kfp.deprecated.dsl import component\n", - "from kfp.deprecated.dsl.types import Integer, GCSPath\n", - "from kfp.deprecated.dsl import ContainerOp\n", - "# when components are defined based on the component decorator,\n", - "# the type information is annotated to the input or function returns.\n", - "# There are two ways to define the type: string or a dictionary with the openapi_schema_validator property\n", - "@component\n", - "def task_factory_a(field_l: Integer()) -> {'field_m': {'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gs://.*$\"}'}}, \n", - " 'field_n': 'customized_type',\n", - " 'field_o': 'Integer'\n", - " }:\n", - " return ContainerOp(\n", - " name = 'operator a',\n", - " image = 'gcr.io/ml-pipeline/component-a',\n", - " arguments = [\n", - " '--field-l', field_l,\n", - " ],\n", - " file_outputs = {\n", - " 'field_m': '/schema.txt',\n", - " 'field_n': '/feature.txt',\n", - " 'field_o': '/output.txt'\n", - " }\n", - " )\n", - "\n", - "# Users can also use the core types that are pre-defined in the SDK.\n", - "# For a full list of core types, check out: https://github.com/kubeflow/pipelines/blob/master/sdk/python/kfp/dsl/types.py\n", - "@component\n", - "def task_factory_b(field_x: 'customized_type',\n", - " field_y: Integer(),\n", - " field_z: {'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gs://.*$\"}'}}) -> {'output_model_uri': 'GcsUri'}:\n", - " return ContainerOp(\n", - " name = 'operator b',\n", - " image = 'gcr.io/ml-pipeline/component-a',\n", - " command = [\n", - " 'python3',\n", - " field_x,\n", - " ],\n", - " arguments = [\n", - " '--field-y', field_y,\n", - " '--field-z', field_z,\n", - " ],\n", - " file_outputs = {\n", - " 'output_model_uri': '/schema.txt',\n", - " }\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Author a pipeline with the above components" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "#Use the component as part of the pipeline\n", - "@dsl.pipeline(name='type-check-c',\n", - " description='')\n", - "def pipeline_c():\n", - " a = task_factory_a(field_l=12)\n", - " b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m'])\n", - "\n", - "compiler.Compiler().compile(pipeline_c, 'pipeline_c.zip', type_check=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Type Check with decorated components: failure scenario" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Author components with decorator" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "from kfp.deprecated.dsl import component\n", - "from kfp.deprecated.dsl.types import Integer, GCSPath\n", - "from kfp.deprecated.dsl import ContainerOp\n", - "# task_factory_a outputs an input field_m with the openapi_schema_validator different\n", - "# from the task_factory_b's input field_z.\n", - "# One is gs:// and the other is gcs://\n", - "@component\n", - "def task_factory_a(field_l: Integer()) -> {'field_m': {'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gs://.*$\"}'}}, \n", - " 'field_n': 'customized_type',\n", - " 'field_o': 'Integer'\n", - " }:\n", - " return ContainerOp(\n", - " name = 'operator a',\n", - " image = 'gcr.io/ml-pipeline/component-a',\n", - " arguments = [\n", - " '--field-l', field_l,\n", - " ],\n", - " file_outputs = {\n", - " 'field_m': '/schema.txt',\n", - " 'field_n': '/feature.txt',\n", - " 'field_o': '/output.txt'\n", - " }\n", - " )\n", - "\n", - "@component\n", - "def task_factory_b(field_x: 'customized_type',\n", - " field_y: Integer(),\n", - " field_z: {'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gcs://.*$\"}'}}) -> {'output_model_uri': 'GcsUri'}:\n", - " return ContainerOp(\n", - " name = 'operator b',\n", - " image = 'gcr.io/ml-pipeline/component-a',\n", - " command = [\n", - " 'python3',\n", - " field_x,\n", - " ],\n", - " arguments = [\n", - " '--field-y', field_y,\n", - " '--field-z', field_z,\n", - " ],\n", - " file_outputs = {\n", - " 'output_model_uri': '/schema.txt',\n", - " }\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Author a pipeline with the above components" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "GCSPath has a property openapi_schema_validator with value: {\"type\": \"string\", \"pattern\": \"^gs://.*$\"} and {\"type\": \"string\", \"pattern\": \"^gcs://.*$\"}\n", - "Component \"task_factory_b\" is expecting field_z to be type({'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gcs://.*$\"}'}}), but the passed argument is type({'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gs://.*$\"}'}})\n" - ] - } - ], - "source": [ - "#Use the component as part of the pipeline\n", - "@dsl.pipeline(name='type-check-d',\n", - " description='')\n", - "def pipeline_d():\n", - " a = task_factory_a(field_l=12)\n", - " b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m'])\n", - "\n", - "try:\n", - " compiler.Compiler().compile(pipeline_d, 'pipeline_d.zip', type_check=True)\n", - "except InconsistentTypeException as e:\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Author a pipeline with the above components but ignoring types." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "#Use the component as part of the pipeline\n", - "@dsl.pipeline(name='type-check-d',\n", - " description='')\n", - "def pipeline_d():\n", - " a = task_factory_a(field_l=12)\n", - " # For each of the arguments, authors can also ignore the types by calling ignore_type function.\n", - " b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m'].ignore_type())\n", - "compiler.Compiler().compile(pipeline_d, 'pipeline_d.zip', type_check=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Type Check with missing type information" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Author components(with missing types)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [], - "source": [ - "from kfp.deprecated.dsl import component\n", - "from kfp.deprecated.dsl.types import Integer, GCSPath\n", - "from kfp.deprecated.dsl import ContainerOp\n", - "# task_factory_a lacks the type information for output filed_n\n", - "# task_factory_b lacks the type information for input field_y\n", - "# When no type information is provided, it matches all types.\n", - "@component\n", - "def task_factory_a(field_l: Integer()) -> {'field_m': {'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gs://.*$\"}'}}, \n", - " 'field_o': 'Integer'\n", - " }:\n", - " return ContainerOp(\n", - " name = 'operator a',\n", - " image = 'gcr.io/ml-pipeline/component-a',\n", - " arguments = [\n", - " '--field-l', field_l,\n", - " ],\n", - " file_outputs = {\n", - " 'field_m': '/schema.txt',\n", - " 'field_n': '/feature.txt',\n", - " 'field_o': '/output.txt'\n", - " }\n", - " )\n", - "\n", - "@component\n", - "def task_factory_b(field_x: 'customized_type',\n", - " field_y,\n", - " field_z: {'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gs://.*$\"}'}}) -> {'output_model_uri': 'GcsUri'}:\n", - " return ContainerOp(\n", - " name = 'operator b',\n", - " image = 'gcr.io/ml-pipeline/component-a',\n", - " command = [\n", - " 'python3',\n", - " field_x,\n", - " ],\n", - " arguments = [\n", - " '--field-y', field_y,\n", - " '--field-z', field_z,\n", - " ],\n", - " file_outputs = {\n", - " 'output_model_uri': '/schema.txt',\n", - " }\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Author a pipeline with the above components" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [], - "source": [ - "#Use the component as part of the pipeline\n", - "@dsl.pipeline(name='type-check-e',\n", - " description='')\n", - "def pipeline_e():\n", - " a = task_factory_a(field_l=12)\n", - " b = task_factory_b(field_x=a.outputs['field_n'], field_y=a.outputs['field_o'], field_z=a.outputs['field_m'])\n", - "\n", - "compiler.Compiler().compile(pipeline_e, 'pipeline_e.zip', type_check=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Type Check with both named arguments and positional arguments" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [], - "source": [ - "#Use the component as part of the pipeline\n", - "@dsl.pipeline(name='type-check-f',\n", - " description='')\n", - "def pipeline_f():\n", - " a = task_factory_a(field_l=12)\n", - " b = task_factory_b(a.outputs['field_n'], a.outputs['field_o'], field_z=a.outputs['field_m'])\n", - "\n", - "compiler.Compiler().compile(pipeline_f, 'pipeline_f.zip', type_check=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Type Check between pipeline parameters and component parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Integer has a property openapi_schema_validator that the latter does not.\n", - "Component \"task_factory_a\" is expecting field_o to be type(Integer), but the passed argument is type({'Integer': {'openapi_schema_validator': {'type': 'integer'}}})\n" - ] - } - ], - "source": [ - "@component\n", - "def task_factory_a(field_m: {'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gs://.*$\"}'}}, field_o: 'Integer'):\n", - " return ContainerOp(\n", - " name = 'operator a',\n", - " image = 'gcr.io/ml-pipeline/component-b',\n", - " arguments = [\n", - " '--field-l', field_m,\n", - " '--field-o', field_o,\n", - " ],\n", - " )\n", - "\n", - "# Pipeline input types are also checked against the component I/O types.\n", - "@dsl.pipeline(name='type-check-g',\n", - " description='')\n", - "def pipeline_g(a: {'GCSPath': {'openapi_schema_validator': '{\"type\": \"string\", \"pattern\": \"^gs://.*$\"}'}}='gs://kfp-path', b: Integer()=12):\n", - " task_factory_a(field_m=a, field_o=b)\n", - "\n", - "try:\n", - " compiler.Compiler().compile(pipeline_g, 'pipeline_g.zip', type_check=True)\n", - "except InconsistentTypeException as e:\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Clean up" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "for p in Path(\".\").glob(\"pipeline_[a-g].zip\"):\n", - " p.unlink()" - ] - } - ], - "metadata": { - "celltoolbar": "Tags", - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.5" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/samples/core/dsl_static_type_checking/dsl_static_type_checking_test.py b/samples/core/dsl_static_type_checking/dsl_static_type_checking_test.py deleted file mode 100644 index c5e02bcb7b7..00000000000 --- a/samples/core/dsl_static_type_checking/dsl_static_type_checking_test.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'dsl_static_type_checking.ipynb'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - run_pipeline=False, - ), -]) diff --git a/samples/core/execution_order/execution_order_test.py b/samples/core/execution_order/execution_order_test.py index cc05c533e09..e80f73f9304 100644 --- a/samples/core/execution_order/execution_order_test.py +++ b/samples/core/execution_order/execution_order_test.py @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func +from kfp.samples.test.utils import relative_path +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'execution_order.py'), - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - ), + TestCase(pipeline_file=relative_path(__file__, 'execution_order.py'),), ]) diff --git a/samples/core/exit_handler/exit_handler_test.py b/samples/core/exit_handler/exit_handler_test.py index 5e801d4dc6f..854b12d6629 100644 --- a/samples/core/exit_handler/exit_handler_test.py +++ b/samples/core/exit_handler/exit_handler_test.py @@ -14,14 +14,15 @@ # %% -import unittest from pprint import pprint +import unittest -import kfp as kfp +from kfp.samples.test.utils import KfpMlmdClient +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase import kfp_server_api from .exit_handler import pipeline_exit_handler as pipeline_exit_handler -from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient def verify(mlmd_connection_config, run: kfp_server_api.ApiRun, **kwargs): @@ -50,8 +51,5 @@ def verify(mlmd_connection_config, run: kfp_server_api.ApiRun, **kwargs): if __name__ == '__main__': run_pipeline_func([ - TestCase( - pipeline_func=pipeline_exit_handler, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - ), + TestCase(pipeline_func=pipeline_exit_handler,), ]) diff --git a/samples/core/helloworld/hello_world.py b/samples/core/helloworld/hello_world.py deleted file mode 100644 index 9ef87faf3fa..00000000000 --- a/samples/core/helloworld/hello_world.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated import dsl, compiler -import kfp.deprecated.components as comp - - -@comp.create_component_from_func -def echo_op(): - print("Hello world") - -@dsl.pipeline( - name='my-first-pipeline', - description='A hello world pipeline.' -) -def hello_world_pipeline(): - echo_task = echo_op() - -if __name__ == '__main__': - compiler.Compiler().compile(hello_world_pipeline, __file__ + '.yaml') \ No newline at end of file diff --git a/samples/core/imagepullsecrets/imagepullsecrets.py b/samples/core/imagepullsecrets/imagepullsecrets.py deleted file mode 100644 index 9eb89bf0c88..00000000000 --- a/samples/core/imagepullsecrets/imagepullsecrets.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Toy example demonstrating how to specify imagepullsecrets to access protected -container registry. -""" - -import kfp.deprecated.dsl as dsl -from kfp.deprecated import compiler -from kubernetes import client as k8s_client - - -class GetFrequentWordOp(dsl.ContainerOp): - """A get frequent word class representing a component in ML Pipelines. - - The class provides a nice interface to users by hiding details such as container, - command, arguments. - """ - def __init__(self, name, message): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - message: a dsl.PipelineParam object representing an input message. - """ - super(GetFrequentWordOp, self).__init__( - name=name, - image='python:3.5-jessie', - command=['sh', '-c'], - arguments=['python -c "from collections import Counter; ' - 'words = Counter(\'%s\'.split()); print(max(words, key=words.get))" ' - '| tee /tmp/message.txt' % message], - file_outputs={'word': '/tmp/message.txt'}) - -@dsl.pipeline( - name='save-most-frequent', - description='Get Most Frequent Word and Save to GCS' -) -def save_most_frequent_word(message: str): - """A pipeline function describing the orchestration of the workflow.""" - - counter = GetFrequentWordOp( - name='get-Frequent', - message=message) - # Call set_image_pull_secrets after get_pipeline_conf(). - dsl.get_pipeline_conf()\ - .set_image_pull_secrets([k8s_client.V1ObjectReference(name="secretA")]) - -if __name__ == '__main__': - compiler.Compiler().compile(save_most_frequent_word, __file__ + '.yaml') diff --git a/samples/core/imagepullsecrets/imagepullsecrets_test.py b/samples/core/imagepullsecrets/imagepullsecrets_test.py deleted file mode 100644 index 7bf33720027..00000000000 --- a/samples/core/imagepullsecrets/imagepullsecrets_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'imagepullsecrets.py'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), -]) diff --git a/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb b/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb deleted file mode 100644 index 75906cb0695..00000000000 --- a/samples/core/kubeflow_tf_serving/kubeflow_tf_serving.ipynb +++ /dev/null @@ -1,350 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# Copyright 2019 The Kubeflow Authors. All Rights Reserved.\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# http://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already up-to-date: kfp in /opt/conda/lib/python3.6/site-packages (0.1.30)\n", - "Requirement already satisfied, skipping upgrade: kfp-server-api<=0.1.25,>=0.1.18 in /opt/conda/lib/python3.6/site-packages (from kfp) (0.1.18.3)\n", - "Requirement already satisfied, skipping upgrade: tabulate==0.8.3 in /opt/conda/lib/python3.6/site-packages (from kfp) (0.8.3)\n", - "Requirement already satisfied, skipping upgrade: cloudpickle in /opt/conda/lib/python3.6/site-packages (from kfp) (0.8.1)\n", - "Requirement already satisfied, skipping upgrade: six>=1.10 in /opt/conda/lib/python3.6/site-packages (from kfp) (1.12.0)\n", - "Requirement already satisfied, skipping upgrade: cryptography>=2.4.2 in /opt/conda/lib/python3.6/site-packages (from kfp) (2.6.1)\n", - "Requirement already satisfied, skipping upgrade: PyYAML in /opt/conda/lib/python3.6/site-packages (from kfp) (3.13)\n", - "Requirement already satisfied, skipping upgrade: python-dateutil in /opt/conda/lib/python3.6/site-packages (from kfp) (2.8.0)\n", - "Requirement already satisfied, skipping upgrade: google-auth>=1.6.1 in /opt/conda/lib/python3.6/site-packages (from kfp) (1.6.3)\n", - "Requirement already satisfied, skipping upgrade: PyJWT>=1.6.4 in /opt/conda/lib/python3.6/site-packages (from kfp) (1.7.1)\n", - "Requirement already satisfied, skipping upgrade: requests-toolbelt>=0.8.0 in /opt/conda/lib/python3.6/site-packages (from kfp) (0.9.1)\n", - "Requirement already satisfied, skipping upgrade: argo-models==2.2.1a in /opt/conda/lib/python3.6/site-packages (from kfp) (2.2.1a0)\n", - "Requirement already satisfied, skipping upgrade: Deprecated in /opt/conda/lib/python3.6/site-packages (from kfp) (1.2.6)\n", - "Requirement already satisfied, skipping upgrade: kubernetes<=9.0.0,>=8.0.0 in /opt/conda/lib/python3.6/site-packages (from kfp) (9.0.0)\n", - "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.15 in /opt/conda/lib/python3.6/site-packages (from kfp) (1.24.1)\n", - "Requirement already satisfied, skipping upgrade: jsonschema>=3.0.1 in /opt/conda/lib/python3.6/site-packages (from kfp) (3.0.1)\n", - "Requirement already satisfied, skipping upgrade: click==7.0 in /opt/conda/lib/python3.6/site-packages (from kfp) (7.0)\n", - "Requirement already satisfied, skipping upgrade: certifi in /opt/conda/lib/python3.6/site-packages (from kfp) (2019.3.9)\n", - "Requirement already satisfied, skipping upgrade: google-cloud-storage>=1.13.0 in /opt/conda/lib/python3.6/site-packages (from kfp) (1.18.0)\n", - "Requirement already satisfied, skipping upgrade: asn1crypto>=0.21.0 in /opt/conda/lib/python3.6/site-packages (from cryptography>=2.4.2->kfp) (0.24.0)\n", - "Requirement already satisfied, skipping upgrade: cffi!=1.11.3,>=1.8 in /opt/conda/lib/python3.6/site-packages (from cryptography>=2.4.2->kfp) (1.12.2)\n", - "Requirement already satisfied, skipping upgrade: cachetools>=2.0.0 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.6.1->kfp) (3.1.0)\n", - "Requirement already satisfied, skipping upgrade: rsa>=3.1.4 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.6.1->kfp) (4.0)\n", - "Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.6/site-packages (from google-auth>=1.6.1->kfp) (0.2.4)\n", - "Requirement already satisfied, skipping upgrade: requests<3.0.0,>=2.0.1 in /opt/conda/lib/python3.6/site-packages (from requests-toolbelt>=0.8.0->kfp) (2.21.0)\n", - "Requirement already satisfied, skipping upgrade: wrapt<2,>=1.10 in /opt/conda/lib/python3.6/site-packages (from Deprecated->kfp) (1.11.2)\n", - "Requirement already satisfied, skipping upgrade: setuptools>=21.0.0 in /opt/conda/lib/python3.6/site-packages (from kubernetes<=9.0.0,>=8.0.0->kfp) (40.9.0)\n", - "Requirement already satisfied, skipping upgrade: requests-oauthlib in /opt/conda/lib/python3.6/site-packages (from kubernetes<=9.0.0,>=8.0.0->kfp) (1.2.0)\n", - "Requirement already satisfied, skipping upgrade: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /opt/conda/lib/python3.6/site-packages (from kubernetes<=9.0.0,>=8.0.0->kfp) (0.56.0)\n", - "Requirement already satisfied, skipping upgrade: attrs>=17.4.0 in /opt/conda/lib/python3.6/site-packages (from jsonschema>=3.0.1->kfp) (19.1.0)\n", - "Requirement already satisfied, skipping upgrade: pyrsistent>=0.14.0 in /opt/conda/lib/python3.6/site-packages (from jsonschema>=3.0.1->kfp) (0.14.11)\n", - "Requirement already satisfied, skipping upgrade: google-resumable-media>=0.3.1 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage>=1.13.0->kfp) (0.3.2)\n", - "Requirement already satisfied, skipping upgrade: google-cloud-core<2.0dev,>=1.0.0 in /opt/conda/lib/python3.6/site-packages (from google-cloud-storage>=1.13.0->kfp) (1.0.3)\n", - "Requirement already satisfied, skipping upgrade: pycparser in /opt/conda/lib/python3.6/site-packages (from cffi!=1.11.3,>=1.8->cryptography>=2.4.2->kfp) (2.19)\n", - "Requirement already satisfied, skipping upgrade: pyasn1>=0.1.3 in /opt/conda/lib/python3.6/site-packages (from rsa>=3.1.4->google-auth>=1.6.1->kfp) (0.4.5)\n", - "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests<3.0.0,>=2.0.1->requests-toolbelt>=0.8.0->kfp) (3.0.4)\n", - "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests<3.0.0,>=2.0.1->requests-toolbelt>=0.8.0->kfp) (2.8)\n", - "Requirement already satisfied, skipping upgrade: oauthlib>=3.0.0 in /opt/conda/lib/python3.6/site-packages (from requests-oauthlib->kubernetes<=9.0.0,>=8.0.0->kfp) (3.0.1)\n", - "Requirement already satisfied, skipping upgrade: google-api-core<2.0.0dev,>=1.14.0 in /opt/conda/lib/python3.6/site-packages (from google-cloud-core<2.0dev,>=1.0.0->google-cloud-storage>=1.13.0->kfp) (1.14.2)\n", - "Requirement already satisfied, skipping upgrade: pytz in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.14.0->google-cloud-core<2.0dev,>=1.0.0->google-cloud-storage>=1.13.0->kfp) (2018.9)\n", - "Requirement already satisfied, skipping upgrade: protobuf>=3.4.0 in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.14.0->google-cloud-core<2.0dev,>=1.0.0->google-cloud-storage>=1.13.0->kfp) (3.7.1)\n", - "Requirement already satisfied, skipping upgrade: googleapis-common-protos<2.0dev,>=1.6.0 in /opt/conda/lib/python3.6/site-packages (from google-api-core<2.0.0dev,>=1.14.0->google-cloud-core<2.0dev,>=1.0.0->google-cloud-storage>=1.13.0->kfp) (1.6.0)\n", - "\u001b[33mYou are using pip version 19.0.1, however version 19.2.3 is available.\n", - "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n", - "Collecting tensorflow==1.14\n", - "\u001b[?25l Downloading https://files.pythonhosted.org/packages/de/f0/96fb2e0412ae9692dbf400e5b04432885f677ad6241c088ccc5fe7724d69/tensorflow-1.14.0-cp36-cp36m-manylinux1_x86_64.whl (109.2MB)\n", - "\u001b[K 100% |████████████████████████████████| 109.2MB 279kB/s eta 0:00:01\n", - "\u001b[?25hCollecting tensorboard<1.15.0,>=1.14.0 (from tensorflow==1.14)\n", - "\u001b[?25l Downloading https://files.pythonhosted.org/packages/91/2d/2ed263449a078cd9c8a9ba50ebd50123adf1f8cfbea1492f9084169b89d9/tensorboard-1.14.0-py3-none-any.whl (3.1MB)\n", - "\u001b[K 100% |████████████████████████████████| 3.2MB 7.2MB/s eta 0:00:01\n", - "\u001b[?25hRequirement already satisfied, skipping upgrade: keras-applications>=1.0.6 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (1.0.7)\n", - "Requirement already satisfied, skipping upgrade: numpy<2.0,>=1.14.5 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (1.16.2)\n", - "Requirement already satisfied, skipping upgrade: absl-py>=0.7.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (0.7.1)\n", - "Requirement already satisfied, skipping upgrade: grpcio>=1.8.6 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (1.19.0)\n", - "Collecting google-pasta>=0.1.6 (from tensorflow==1.14)\n", - "\u001b[?25l Downloading https://files.pythonhosted.org/packages/d0/33/376510eb8d6246f3c30545f416b2263eee461e40940c2a4413c711bdf62d/google_pasta-0.1.7-py3-none-any.whl (52kB)\n", - "\u001b[K 100% |████████████████████████████████| 61kB 25.5MB/s ta 0:00:01\n", - "\u001b[?25hCollecting tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 (from tensorflow==1.14)\n", - "\u001b[?25l Downloading https://files.pythonhosted.org/packages/3c/d5/21860a5b11caf0678fbc8319341b0ae21a07156911132e0e71bffed0510d/tensorflow_estimator-1.14.0-py2.py3-none-any.whl (488kB)\n", - "\u001b[K 100% |████████████████████████████████| 491kB 25.5MB/s ta 0:00:01\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[?25hRequirement already satisfied, skipping upgrade: keras-preprocessing>=1.0.5 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (1.0.9)\n", - "Requirement already satisfied, skipping upgrade: six>=1.10.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (1.12.0)\n", - "Requirement already satisfied, skipping upgrade: gast>=0.2.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (0.2.2)\n", - "Requirement already satisfied, skipping upgrade: protobuf>=3.6.1 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (3.7.1)\n", - "Requirement already satisfied, skipping upgrade: wrapt>=1.11.1 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (1.11.2)\n", - "Requirement already satisfied, skipping upgrade: termcolor>=1.1.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (1.1.0)\n", - "Requirement already satisfied, skipping upgrade: wheel>=0.26 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (0.33.1)\n", - "Requirement already satisfied, skipping upgrade: astor>=0.6.0 in /opt/conda/lib/python3.6/site-packages (from tensorflow==1.14) (0.7.1)\n", - "Collecting setuptools>=41.0.0 (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14)\n", - "\u001b[?25l Downloading https://files.pythonhosted.org/packages/b2/86/095d2f7829badc207c893dd4ac767e871f6cd547145df797ea26baea4e2e/setuptools-41.2.0-py2.py3-none-any.whl (576kB)\n", - "\u001b[K 100% |████████████████████████████████| 583kB 23.7MB/s ta 0:00:01\n", - "\u001b[?25hRequirement already satisfied, skipping upgrade: markdown>=2.6.8 in /opt/conda/lib/python3.6/site-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (3.1)\n", - "Requirement already satisfied, skipping upgrade: werkzeug>=0.11.15 in /opt/conda/lib/python3.6/site-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (0.15.2)\n", - "Requirement already satisfied, skipping upgrade: h5py in /opt/conda/lib/python3.6/site-packages (from keras-applications>=1.0.6->tensorflow==1.14) (2.9.0)\n", - "\u001b[31mfairing 0.5 has requirement oauth2client>=4.0.0, but you'll have oauth2client 3.0.0 which is incompatible.\u001b[0m\n", - "Installing collected packages: setuptools, tensorboard, google-pasta, tensorflow-estimator, tensorflow\n", - " Found existing installation: setuptools 40.9.0\n", - " Uninstalling setuptools-40.9.0:\n", - " Successfully uninstalled setuptools-40.9.0\n", - " Found existing installation: tensorboard 1.13.1\n", - " Uninstalling tensorboard-1.13.1:\n", - " Successfully uninstalled tensorboard-1.13.1\n", - " Found existing installation: tensorflow-estimator 1.13.0\n", - " Uninstalling tensorflow-estimator-1.13.0:\n", - " Successfully uninstalled tensorflow-estimator-1.13.0\n", - " Found existing installation: tensorflow 1.13.1\n", - " Uninstalling tensorflow-1.13.1:\n", - " Successfully uninstalled tensorflow-1.13.1\n", - "Successfully installed google-pasta-0.1.7 setuptools-41.2.0 tensorboard-1.14.0 tensorflow-1.14.0 tensorflow-estimator-1.14.0\n", - "\u001b[33mYou are using pip version 19.0.1, however version 19.2.3 is available.\n", - "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n" - ] - } - ], - "source": [ - "# Install Pipeline SDK - This only needs to be ran once in the enviroment. \n", - "!python3 -m pip install 'kfp>=0.1.31' --quiet\n", - "!pip3 install tensorflow==1.14 --upgrade" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## KubeFlow Pipelines Serving Component\n", - "In this notebook, we will demo:\n", - "\n", - "* Saving a Keras model in a format compatible with TF Serving\n", - "* Creating a pipeline to serve a trained model within a KubeFlow cluster\n", - "\n", - "Reference documentation:\n", - "* https://www.tensorflow.org/tfx/serving/architecture\n", - "* https://www.tensorflow.org/beta/guide/keras/saving_and_serializing\n", - "* https://www.kubeflow.org/docs/components/serving/tfserving_new/" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Setup\n" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": { - "tags": [ - "parameters" - ] - }, - "outputs": [], - "source": [ - "# Set your output and project. !!!Must Do before you can proceed!!!\n", - "project = 'Your-Gcp-Project-ID' #'Your-GCP-Project-ID'\n", - "model_name = 'model-name' # Model name matching TF_serve naming requirements \n", - "import time\n", - "ts = int(time.time())\n", - "model_version = str(ts) # Here we use timestamp as version to avoid conflict \n", - "output = 'Your-Gcs-Path' # A GCS bucket for asset outputs\n", - "KUBEFLOW_DEPLOYER_IMAGE = 'gcr.io/ml-pipeline/ml-pipeline-kubeflow-deployer:1.8.0-alpha.0'" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [], - "source": [ - "model_path = '%s/%s' % (output,model_name) \n", - "model_version_path = '%s/%s/%s' % (output,model_name,model_version)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Load a Keras Model \n", - "Loading a pretrained Keras model to use as an example. " - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [], - "source": [ - "import tensorflow as tf" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Exception ignored in: >\n", - "Traceback (most recent call last):\n", - " File \"/opt/conda/lib/python3.6/site-packages/tensorflow/python/training/tracking/util.py\", line 244, in __del__\n", - " .format(pretty_printer.node_names[node_id]))\n", - " File \"/opt/conda/lib/python3.6/site-packages/tensorflow/python/training/tracking/util.py\", line 93, in node_names\n", - " path_to_root[node_id] + (child.local_name,))\n", - " File \"/opt/conda/lib/python3.6/site-packages/tensorflow/python/training/tracking/object_identity.py\", line 76, in __getitem__\n", - " return self._storage[self._wrap_key(key)]\n", - "KeyError: (,)\n" - ] - } - ], - "source": [ - "model = tf.keras.applications.NASNetMobile(input_shape=None,\n", - " include_top=True,\n", - " weights='imagenet',\n", - " input_tensor=None,\n", - " pooling=None,\n", - " classes=1000)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Saved the Model for TF-Serve\n", - "Save the model using keras export_saved_model function. Note that specifically for TF-Serve the output directory should be structure as model_name/model_version/saved_model." - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [], - "source": [ - "tf.keras.experimental.export_saved_model(model, model_version_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create a pipeline using KFP TF-Serve component\n" - ] - }, - { - "cell_type": "code", - "execution_count": 36, - "metadata": {}, - "outputs": [], - "source": [ - "def kubeflow_deploy_op():\n", - " return dsl.ContainerOp(\n", - " name = 'deploy',\n", - " image = KUBEFLOW_DEPLOYER_IMAGE,\n", - " arguments = [\n", - " '--model-export-path', model_path,\n", - " '--server-name', model_name,\n", - " ]\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "metadata": {}, - "outputs": [], - "source": [ - "from kfp.deprecated import dsl, Client\n", - "\n", - "# The pipeline definition\n", - "@dsl.pipeline(\n", - " name='sample-model-deployer',\n", - " description='Sample for deploying models using KFP model serving component'\n", - ")\n", - "def model_server():\n", - " deploy = kubeflow_deploy_op()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Submit pipeline for execution on Kubeflow Pipelines cluster" - ] - }, - { - "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "Run link here" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "Client().create_run_from_pipeline_func(model_server, arguments={})\n", - "\n", - "#vvvvvvvvv This link leads to the run information page. (Note: There is a bug in JupyterLab that modifies the URL and makes the link stop working)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.7" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/samples/core/lightweight_component/lightweight_component.ipynb b/samples/core/lightweight_component/lightweight_component.ipynb deleted file mode 100644 index ff50886b11f..00000000000 --- a/samples/core/lightweight_component/lightweight_component.ipynb +++ /dev/null @@ -1,273 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Lightweight python components\n", - "\n", - "Lightweight python components do not require you to build a new container image for every code change.\n", - "They're intended to use for fast iteration in notebook environment.\n", - "\n", - "#### Building a lightweight python component\n", - "To build a component just define a stand-alone python function and then call kfp.components.func_to_container_op(func) to convert it to a component that can be used in a pipeline.\n", - "\n", - "There are several requirements for the function:\n", - "* The function should be stand-alone. It should not use any code declared outside of the function definition. Any imports should be added inside the main function. Any helper functions should also be defined inside the main function.\n", - "* The function can only import packages that are available in the base image. If you need to import a package that's not available you can try to find a container image that already includes the required packages. (As a workaround you can use the module subprocess to run pip install for the required package. There is an example below in my_divmod function.)\n", - "* If the function operates on numbers, the parameters need to have type hints. Supported types are ```[int, float, bool]```. Everything else is passed as string.\n", - "* To build a component with multiple output values, use the typing.NamedTuple type hint syntax: ```NamedTuple('MyFunctionOutputs', [('output_name_1', type), ('output_name_2', float)])```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "skip-in-test" - ] - }, - "outputs": [], - "source": [ - "# Install the SDK\n", - "!pip3 install 'kfp>=0.1.31.2' --quiet" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "import kfp.deprecated as kfp\n", - "import kfp.deprecated.components as components" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Simple function that just add two numbers:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "#Define a Python function\n", - "def add(a: float, b: float) -> float:\n", - " '''Calculates sum of two arguments'''\n", - " return a + b" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Convert the function to a pipeline operation" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "add_op = components.create_component_from_func(add)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "A bit more advanced function which demonstrates how to use imports, helper functions and produce multiple outputs." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "#Advanced function\n", - "#Demonstrates imports, helper functions and multiple outputs\n", - "from typing import NamedTuple\n", - "def my_divmod(dividend: float, divisor:float) -> NamedTuple('MyDivmodOutput', [('quotient', float), ('remainder', float), ('mlpipeline_ui_metadata', 'UI_metadata'), ('mlpipeline_metrics', 'Metrics')]):\n", - " '''Divides two numbers and calculate the quotient and remainder'''\n", - " \n", - " #Imports inside a component function:\n", - " import numpy as np\n", - "\n", - " #This function demonstrates how to use nested functions inside a component function:\n", - " def divmod_helper(dividend, divisor):\n", - " return np.divmod(dividend, divisor)\n", - "\n", - " (quotient, remainder) = divmod_helper(dividend, divisor)\n", - "\n", - " from tensorflow.python.lib.io import file_io\n", - " import json\n", - " \n", - " # Exports a sample tensorboard:\n", - " metadata = {\n", - " 'outputs' : [{\n", - " 'type': 'tensorboard',\n", - " 'source': 'gs://ml-pipeline-dataset/tensorboard-train',\n", - " }]\n", - " }\n", - "\n", - " # Exports two sample metrics:\n", - " metrics = {\n", - " 'metrics': [{\n", - " 'name': 'quotient',\n", - " 'numberValue': float(quotient),\n", - " },{\n", - " 'name': 'remainder',\n", - " 'numberValue': float(remainder),\n", - " }]}\n", - "\n", - " from collections import namedtuple\n", - " divmod_output = namedtuple('MyDivmodOutput', ['quotient', 'remainder', 'mlpipeline_ui_metadata', 'mlpipeline_metrics'])\n", - " return divmod_output(quotient, remainder, json.dumps(metadata), json.dumps(metrics))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Test running the python function directly" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "tags": [ - "skip-in-test" - ] - }, - "outputs": [ - { - "data": { - "text/plain": [ - "MyDivmodOutput(quotient=14, remainder=2)" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "my_divmod(100, 7)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Convert the function to a pipeline operation\n", - "\n", - "You can specify an alternative base container image (the image needs to have Python 3.5+ installed)." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "divmod_op = components.create_component_from_func(my_divmod, base_image='tensorflow/tensorflow:1.11.0-py3')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Define the pipeline\n", - "Pipeline function has to be decorated with the `@dsl.pipeline` decorator" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "import kfp.deprecated.dsl as dsl\n", - "@dsl.pipeline(\n", - " name='calculation-pipeline',\n", - " description='A toy pipeline that performs arithmetic calculations.'\n", - ")\n", - "def calc_pipeline(\n", - " a=7,\n", - " b=8,\n", - " c=17,\n", - "):\n", - " #Passing pipeline parameter and a constant value as operation arguments\n", - " add_task = add_op(a, 4) #Returns a dsl.ContainerOp class instance. \n", - " \n", - " #Passing a task output reference as operation arguments\n", - " #For an operation with a single return value, the output reference can be accessed using `task.output` or `task.outputs['output_name']` syntax\n", - " divmod_task = divmod_op(add_task.output, b)\n", - "\n", - " #For an operation with a multiple return values, the output references can be accessed using `task.outputs['output_name']` syntax\n", - " result_task = add_op(divmod_task.outputs['quotient'], c)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Submit the pipeline for execution" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "tags": [ - "skip-in-test" - ] - }, - "outputs": [], - "source": [ - "#Specify pipeline argument values\n", - "arguments = {'a': 7, 'b': 8}\n", - "\n", - "#Submit a pipeline run\n", - "kfp.Client().create_run_from_pipeline_func(calc_pipeline, arguments=arguments)\n", - "\n", - "# Run the pipeline on a separate Kubeflow Cluster instead\n", - "# (use if your notebook is not running in Kubeflow - e.x. if using AI Platform Notebooks)\n", - "# kfp.Client(host='').create_run_from_pipeline_func(calc_pipeline, arguments=arguments)\n", - "\n", - "#vvvvvvvvv This link leads to the run information page. (Note: There is a bug in JupyterLab that modifies the URL and makes the link stop working)" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "c7a91a0fef823c7f839350126c5e355ea393d05f89cb40a046ebac9c8851a521" - }, - "kernelspec": { - "display_name": "Python 3.7.10 64-bit ('v2': conda)", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/samples/core/lightweight_component/lightweight_component_test.py b/samples/core/lightweight_component/lightweight_component_test.py deleted file mode 100644 index c4bb16ae097..00000000000 --- a/samples/core/lightweight_component/lightweight_component_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'lightweight_component.ipynb'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), -]) diff --git a/samples/core/loop_output/loop_output_test.py b/samples/core/loop_output/loop_output_test.py index d2648cdc9f4..5ab25bcf230 100644 --- a/samples/core/loop_output/loop_output_test.py +++ b/samples/core/loop_output/loop_output_test.py @@ -15,11 +15,14 @@ from __future__ import annotations import unittest + import kfp +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase import kfp_server_api -from ml_metadata.proto.metadata_store_pb2 import Execution from loop_output import my_pipeline -from kfp.samples.test.utils import KfpTask, run_pipeline_func, TestCase +from ml_metadata.proto.metadata_store_pb2 import Execution def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, @@ -76,7 +79,6 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, run_pipeline_func([ TestCase( pipeline_func=my_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, verify_func=verify, ), ]) diff --git a/samples/core/loop_parallelism/loop_parallelism_test.py b/samples/core/loop_parallelism/loop_parallelism_test.py index 1ca2c0975fb..638928a6006 100644 --- a/samples/core/loop_parallelism/loop_parallelism_test.py +++ b/samples/core/loop_parallelism/loop_parallelism_test.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase + from .loop_parallelism import pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase run_pipeline_func([ - TestCase( - pipeline_func=pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - ), + TestCase(pipeline_func=pipeline), ]) diff --git a/samples/core/loop_parameter/loop_parameter_test.py b/samples/core/loop_parameter/loop_parameter_test.py index 648d96cb623..fb4510a260d 100644 --- a/samples/core/loop_parameter/loop_parameter_test.py +++ b/samples/core/loop_parameter/loop_parameter_test.py @@ -13,11 +13,16 @@ # limitations under the License. from __future__ import annotations + import unittest + import kfp +from kfp.samples.test.utils import debug_verify +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase import kfp_server_api from loop_parameter import my_pipeline -from kfp.samples.test.utils import KfpTask, debug_verify, run_pipeline_func, TestCase def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, @@ -33,8 +38,5 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, run_pipeline_func([ - TestCase( - pipeline_func=my_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - verify_func=verify), + TestCase(pipeline_func=my_pipeline, verify_func=verify), ]) diff --git a/samples/core/loop_static/loop_static_test.py b/samples/core/loop_static/loop_static_test.py index 276e5259977..bfc377281c2 100644 --- a/samples/core/loop_static/loop_static_test.py +++ b/samples/core/loop_static/loop_static_test.py @@ -13,11 +13,15 @@ # limitations under the License. from __future__ import annotations + import unittest + import kfp +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase import kfp_server_api from loop_static import my_pipeline -from kfp.samples.test.utils import KfpTask, run_pipeline_func, TestCase def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, @@ -35,8 +39,7 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, 'b': '20' }], [ - x.inputs - .parameters['pipelinechannel--loop-item-param-1'] + x.inputs.parameters['pipelinechannel--loop-item-param-1'] for x in tasks['for-loop-2'].children.values() ], ) @@ -50,7 +53,6 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, run_pipeline_func([ TestCase( pipeline_func=my_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, verify_func=verify, ), ]) diff --git a/samples/core/multiple_outputs/multiple_outputs_test.py b/samples/core/multiple_outputs/multiple_outputs_test.py index d702a8fa596..9c3eb26a173 100644 --- a/samples/core/multiple_outputs/multiple_outputs_test.py +++ b/samples/core/multiple_outputs/multiple_outputs_test.py @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func +from kfp.samples.test.utils import relative_path +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'multiple_outputs.ipynb'), - mode=kfp.dsl.PipelineExecutionMode.V2_LEGACY, - ), + TestCase(pipeline_file=relative_path(__file__, 'multiple_outputs.ipynb')), ]) diff --git a/samples/core/output_a_directory/output_a_directory_test.py b/samples/core/output_a_directory/output_a_directory_test.py index ae39d0a05fa..ca520c5fa5b 100644 --- a/samples/core/output_a_directory/output_a_directory_test.py +++ b/samples/core/output_a_directory/output_a_directory_test.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp as kfp +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase + from .output_a_directory import dir_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase run_pipeline_func([ - TestCase( - pipeline_func=dir_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - ), + TestCase(pipeline_func=dir_pipeline), ]) diff --git a/samples/core/parallelism_sub_dag/parallelism_sub_dag.py b/samples/core/parallelism_sub_dag/parallelism_sub_dag.py deleted file mode 100644 index 968e556f3dc..00000000000 --- a/samples/core/parallelism_sub_dag/parallelism_sub_dag.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -import kfp.deprecated as kfp - -@kfp.components.create_component_from_func -def print_op(s: str): - import time - time.sleep(3) - print(s) - -@dsl.pipeline(name='my-pipeline') -def pipeline(): - loop_args = [{'A_a': 1, 'B_b': 2}, {'A_a': 10, 'B_b': 20}] - with dsl.SubGraph(parallelism=2): - with dsl.ParallelFor(loop_args) as item: - print_op(item) - print_op(item.A_a) - print_op(item.B_b) - - -if __name__ == '__main__': - kfp.compiler.Compiler().compile(pipeline, __file__ + '.yaml') diff --git a/samples/core/parallelism_sub_dag/parallelism_sub_dag_test.py b/samples/core/parallelism_sub_dag/parallelism_sub_dag_test.py deleted file mode 100644 index c6d92e2ed5b..00000000000 --- a/samples/core/parallelism_sub_dag/parallelism_sub_dag_test.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'parallelism_sub_dag.py'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), - TestCase( - pipeline_file=relative_path(__file__, - 'parallelism_sub_dag_with_op_output.py'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), -]) diff --git a/samples/core/parallelism_sub_dag/parallelism_sub_dag_with_op_output.py b/samples/core/parallelism_sub_dag/parallelism_sub_dag_with_op_output.py deleted file mode 100644 index 158f80b62b7..00000000000 --- a/samples/core/parallelism_sub_dag/parallelism_sub_dag_with_op_output.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -import kfp.deprecated as kfp - -@kfp.components.create_component_from_func -def print_op(s: str): - import time - time.sleep(3) - print(s) - -@kfp.components.create_component_from_func -def dump_loop_args() -> list: - return [{'A_a': 1, 'B_b': 2}, {'A_a': 10, 'B_b': 20}] - -@dsl.pipeline(name='my-pipeline') -def pipeline(): - dump_loop_args_op = dump_loop_args() - with dsl.SubGraph(parallelism=2): - with dsl.ParallelFor(dump_loop_args_op.output) as item: - print_op(item) - print_op(item.A_a) - print_op(item.B_b) - - -if __name__ == '__main__': - kfp.compiler.Compiler().compile(pipeline, __file__ + '.yaml') diff --git a/samples/core/parameterized_tfx_oss/README.md b/samples/core/parameterized_tfx_oss/README.md deleted file mode 100644 index c0bed325e28..00000000000 --- a/samples/core/parameterized_tfx_oss/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Overview -[Tensorflow Extended (TFX)](https://github.com/tensorflow/tfx) is a Google-production-scale machine -learning platform based on TensorFlow. It provides a configuration framework to express ML pipelines -consisting of TFX components. Kubeflow Pipelines can be used as the orchestrator supporting the -execution of a TFX pipeline. - -This directory contains two samples that demonstrate how to author a ML pipeline in TFX and run it -on a KFP deployment. -* `parameterized_tfx_oss.py` is a Python script that outputs a compiled KFP workflow, which you can - submit to a KFP deployment to run; -* `parameterized_tfx_oss.ipynb` is a notebook version of `parameterized_tfx_oss.py`, and it also - includes the guidance to setup its dependencies. - -Please refer to inline comments for the purpose of each step in both samples. - -# Compilation -* `parameterized_tfx_oss.py`: -In order to successfully compile the Python sample, you'll need to have a TFX installation at -version 1.0.0 by running `python3 -m pip install tfx==1.0.0`. After that, under the sample dir run -`python3 parameterized_tfx_oss.py` to compile the TFX pipeline into KFP pipeline package. -The compilation is done by invoking `kfp_runner.run(pipeline)` in the script. - -* `parameterized_tfx_oss.ipynb`: -The notebook sample includes the installation of various dependencies as its first step. - -# Permission - -> :warning: If you are using **full-scope** or **workload identity enabled** cluster in hosted pipeline beta version, **DO NOT** follow this section. However you'll still need to enable corresponding GCP API. - -This pipeline requires Google Cloud Storage permission to run. -If KFP was deployed through K8S marketplace, please follow instructions in -[the guideline](https://github.com/kubeflow/pipelines/blob/master/manifests/gcp_marketplace/guide.md#gcp-service-account-credentials) -to make sure the service account has `storage.admin` role. -If KFP was deployed through -[standalone deployment](https://github.com/kubeflow/pipelines/tree/master/manifests/kustomize) -please refer to [Authenticating Pipelines to GCP](https://www.kubeflow.org/docs/gke/authentication-pipelines/) -to provide `storage.admin` permission. - -# Execution -* `parameterized_tfx_oss.py`: -You can submit the compiled package to a KFP deployment and run it from the UI. - -* `parameterized_tfx_oss.ipynb`: -The last step of the notebook the execution of the pipeline is invoked via KFP SDK client. Also you -have the option to submit and run from UI manually. - -### Known issues -* This approach only works for string-typed quantities. For example, you cannot parameterize -`num_steps` of `Trainer` in this way. -* Name of parameters should be unique. -* By default pipeline root is always parameterized with the name `pipeline-root`. -* If the parameter is referenced at multiple places, the user should -make sure that it is correctly converted to the string-formatted placeholder by -calling `str(your_param)`. -* The best practice is to specify TFX pipeline root to an empty dir. In this sample Argo -automatically do that by plugging in the -workflow unique ID (represented `kfp.dsl.RUN_ID_PLACEHOLDER`) to the pipeline root path. diff --git a/samples/core/parameterized_tfx_oss/check_permission.png b/samples/core/parameterized_tfx_oss/check_permission.png deleted file mode 100644 index 7c2aa5fd4a5bd2c55aab5c5450c1e8175b842c56..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 130803 zcmdqI^;g_W_63T&ySoPqL4v!xLxA8Cq;Yq*-~Vc==E7ub#~b~d+&=nlmFSafH_eN`CvsI9x;|5$su@cjJyxvcu+8EDd%F~Tf%7VBV?|E>Hr zEK@H+77V@_oJt5P_rL!ExxjJx&BWJE0C=9qKX^YL+KP`uuXz<7-2bnS0SNi`key>r z0JZ<&Ya-x+Q{1iG|1|ceWf!tg;ge2xZ`b=leD!*eXaf85=l}Hh=c8VPTJlgM=Pplg zSO2Sh-T%+laslmocX42@z&&N3g^m@(Io>lUanEh@)7eyZZyEFHOi~cR>M7=`%ic&I z{C_+6JJ@KIH5KUfQj3FaBP6vxa71_|v^?Jr?PfJtJZQf>C=T(MwhrqU{2rwqw2UqX zi1_C~KP#+#`}C^A2rxwlu<=Af4&eOq4b9I!DSNf&gH|zgUcTYE?~0% z7hSB7-t566@aoew<2Xm`4}tEp1oG%67RSBPY*1NGQ?F=?c)OkP5GBwFTj%@TA<^7~u(cYRfyh}mSS}1p1XRBv-9G!ryi&l0cEy4_ zPP)m(ALqooc)xh|d*y+wL;X>dL+@QQ5ey*j2@C<5{|u4>_iY-e>!kaBI|J<9eUV2J zCE0xYMW74yFTjUZ1yI4NP_Mk+z|9o-(@^XHnAK~iRVGO0Z!@^el$X%Q-#S2S|982B zj?2AZ?)AFp$I0W$$$HmGU>(Vg^80f6a__-lC`0>D$tC4x`tRX2-B^|Rk zeDElD^}#0j;9_orL_>}$bw!FmP{Vn*@ST-ph}3m50@JOw=W|*g)azM=t9yv7VT(pn zN~aS3H`Ez)Bh1@?DV&0v!AfDR{vDeSPh3K)mE_Trfh7m)`z6lr;j+zUW*+S~nZe74 znaIH5f6wgN9o}0YsO%WfxH3ES6BXhK_0n9N_3SxC?+J;f@>7`hSw47=eD_-{i4%Qo zhof*Qh8~%`=dm5mJh_-uXHgTumP^T_2ijVV{v!`pv(cT^_>v3Q z29V>Y#Y3+lYaAeh36O+C!pl}!Cz@>X<Gl29v&1s#@nyzhkr zs(1*{RW9gT3|HYFuZ@`{OYRlkV0YM+JjR2`Qub(c>2)eaol092D9~SHNkHuR;p7Ya z3ouL)c&0Qx*$#Q78Gb6oPh2FAiOr4*4cM%GaBxZh)zE>=;om5&AMu2iFUJM{cfV55 z*Au%c*E|h7KOl+S_F>17cizp{4P!s8VRJ4!&V+Ta{1vpxDrE2@LUO#}yklT!2>yQSlIk>A~&Gj}*xu zK1<%%mWRV@M3HfM){fCQv-0nuOilX%bFH9hNF0Zur64cBbuCY0^jqGEy&`{+TawcSy1s zMHNPu1gl@j>X&C2-C5{;6h()+6S5eVC=zzwi7+lNb2#Cw05km2&j%v7b>`?dxj@YYV(pL!pocLQAlgkeMkSH8g0JK zx%&?NE2mWscJYif5fJ&6yMAq%E`B9ip_^qis_X=1RZ zU~zX=rZd>jFw*=U1>RvO(nlg~BxaW43I*ZI2L0vm+a*phc))hB@H0HvH zI{%8V1iDre;2L^#wJMq_?TOc(tQ2qH!tk;?0L+$d%I}#0Lp(g|R%l^7;nfrc3n^(~ zG&7paq~F>HXy)!GjRy9buUdxPjC`gak<|3)7{+U|4k+Z(`ztI;ts`4Ai^%xHSb760 z(b$-$#!(U{V{ytZgN|NLQ{-YjOb-PJQV-@P3-|pw+@Pq3st_%G^`@0sK?axAi~O6A zhy!-tAC6ho$KTUwe;@Qb9dJeTt3dohUl|}jfz?9FLR<By=aZr-mkU|3H)I!{Rs%vk3I&lWTkzaF~`1~rM*M*TsAD~2-7GOCzYF{z;5$EJ~ ztm1c}dqPqS&9ZULaEHwj3|@6MXW4kX(-qxlv5b6tX=U!uHmqF1gG7fiVWyj> zKUATuumte6Cw*EUmKkDZ(Pr_=oqjH~LNhGJtalT!r+(732tv7~YdG4yonLd!eTgE} z(@?=aUi2+hjMckIv9(3j6$JYy)>-^J3w1$3xMl+Y8&+lu1p*Iy?w3!9pqYt7jZ(z0 z_M8AqVrdGR@NgDmoOgk-qv|ulxKe_V#9Z$HNT)|R7mGQP;)F>wvvR0!;7v3^otRFN8rRMKWkTAp=IAyCA7KK`5RZIR zKaZo@blDm8$C&jI%uDXRcS)!5S(oHLwrS(Q;l!=wabQIXUH(}ixL;3lkL_?FxJ~r*@I5jGAui;C6^?nE?-eh*>S~F~5Q&M8wwm?e(JhKVzIW!gZ@X z@~r7IrLql!O!eq_DXo+ns$RvrG6BK~^!Dh#B4ugD4!+-r?@R+7naK#www+vu%nGa% zvOUm+%S#0{hA9UnI7_ibT6I56;V10*1_&SNb z-P5eMlHJN0)K_!M;0mU7SB+?i1u2;+<_;dg2pFl{@)n}8gMLpyJC5#_-b22rMM-73 zjaU+_pW`OgY?n#zXT|to`>s#9bAHUY#scWM;Z>7guIzw${w_0m6KL!8U6rL5hy6ML z71jN>s?L6Mkw04~hmKfezDFULV!uh;W_%OjGR^=}gejrBS=9w@Zj{ihgV@8EfmUy1 z8q*v06(Dzd@w~LkN+HuTiArY}gcjm*C28ar#cdWIrcAG?qAjJ;(*|p{%TJR!pA>Y} zjwWUHTT#p2Sp|hiWkd33N>Bnz!#&x-mlBG>?<9a2ENP>9iYi?13L|ud4Dt2^wsAqX z9aK4>E`t7R9ar@9Zlt>UWZu^*Ds*4mQ);Y^oGg$5U+8KTAs?kGym)lb`{8y|+K{B$ z-w)c<#-&$2{*=|L9R#QWnjSalR5RoT>dcs~cKMCv>pv4H>Cl=m>k$tNjp9WGu{YA^ zzK*j}oh`{5Jj&6Mctb0CW$Q8iX>DiBZLE+jVlT6PmaJswlppHluf|~G^ePH(p+C|n z1}Y=F$%~?eX((&q1^GPMY<;t}Rqllcl#AgJYs|nliD!$)vEfT^ONKB2)S)``Egu52 zHtycK20C#8u0VYjF{doo9Us!Hw)KLRIx$8~C?%bG)$x4}qXW*kJi6ikl}%IdhIYk+ zk*Nc4Tq)uuwZCjQrAca38GR9=vA(X+q!t1bwNMUi3r5QEADCG8Fce zJ~4st(zyNV(Vw!wYM^~0>+seC?!Y8CgNdquIFJ*==&`| zATJ0&w1n|14~95~Mr~wcb1VS&=2Q#9Nqa--j>=alw{LESRl|Y>IepI9vm>78ARhh- ze`U@2Td@AC;Mx-W--@q+Z-77Bw!FS=f}O}di5U*^#~Kx)d4SpkW4Clu+|pS=YDb^^ zrqg!k$|K~YQCHu_G3O0+%jw3cE_o4|s<>W774#lS`4}S8Po&iTq!z9N9ch7VB38yz z?w6Vs>wWo1ToZH#v7PAw-fIzLDO)rPiXi(@6Zn*zBDYs>^^o$uCfXw{#L(0Xb8~rm zOS2Zb#>2GZqfKQu#DTG(UQ5j+4{&V@N!i8fCZ;>Yz&GgUAm65;&St>?E}0puWL@|u z^WH{EgNt){Ti@B=WPh@;%(UK>ArAA~Uj>hb7z0eYUP2X~V-;#W_P{s8j$@{H#;ho#9?8zSwextS!3vHEj{H0K8IB~RjR}E4KP!>t5jxIiM zf7!tqV5cFck3yo2Scx|kxFig-HoyGFt`Ma1z#~g9KJg3xw zV5Ak_D|Skl!Bv^0fCw;_R?W5jE7hnpF){`sr@tJP`E!zK{Kz9vT-l|$kHh!4wR3tm z&ZzNLac|(1EvqP=^J8e5acZR47mRk&=6acZm7lI`h*sL2?cwa$NzmhJn5f)=MIIwJ zgD>O9!><5io>u*eTdeD|ir7mp$4;MUK+EMMl3M#h-<5S~Y=lpoHqlvmS)|4kEO^D=&1Ck@ z`#0gxoA}pluir$e-36X^lgPch`Me_{Ke!0+e@fgC@w$8MDm;{I6OPp440F$dJNMf4 z6?+HcW|@_$C|k;z!MloRf4>PQUTY4jCN6C5FuS;1G7VWVs9Y#{WPiJeFw4e=>UYHy z$+w(p#xgCh-{sjdtNu#0zGpcjc(hZ2o0nY8(y)(1&X2eeiKzaL6D9c0$)+u@g|8Q5 zAQI!7c8{b@s(DeS>8AjhUP+58Dsrhe2hF5Y_>SsXhVp=!Ru^rAwQMA=CQ#!6oJYok5yOFtp{6gNP!q5wSL1gek zOCG%}LI3SAenSoR{E|K=pqUp$j{yGt_ud|6^=G%?W$Z~}PrQE5c- zL&|Q!hGP_jQ4Ry?$h^9Voy8#suHE2obnhkt6s`!FFGE3vx zQc-837_YbC#17;IA&vd~%%tp$NMau8aqKVBmOUqIp-1ht@umygRhcnOCo2&?t!hsr zw!AU2J$=R}_o_wS75&>{z*4718x6VIlH?o8#V<&e1>1o7Z*RPX^Ha6pBCyB0y zb;N@xha5>tcZnRfW&;~32pPS|7)Ilm#2}P`ed)ryqfnho#Jem6>!j!G2v1z-c!(yI4C%fw1 zG))qHgI6vfq;!{?lDbHsNlXnerWd2Zk&>*zRHwEaMjqj^Q;T*m$gy{@b3hfGah|>p$$O&P@i|+{tUuIDOz#PRyxUIOW%7A)F->@cUG*PC zN%%$Bwpux-`Xsv<+qD0X>etd{&d?0w)ZpWtAi&D2Mo9im;Uf3|H`sqAh-&jH4J4|E zwo4;?e|h|aHvXmX0QsAeeQWT-ier1(gDn35 z;l=b<1>wznv-y&oy6hx=?FivvF|9is^WQtA|Bj#r%g6f zv4l}@e_cFm5eu;a9u8kfRgbGtN-$LwDaJ981(-JX@q0u+JVR*7Wsb+1}pakxPg4?W6J*na!1LRFo=?G z3H}en-#Y%a+x&w--v0;W^@uNXWh4RB{u9rC2cxo9|FZW*4}Y(({6mBK*B8A+FLMQe zja4`QU%l~Pr%&hrQ!n(oeka<0 z@Ow)Fw*cc&fSezQOtTF4a2y>X?hsn%FJ{(>;j`|J^`#nu`)97qT*u@8yC&btt-VQt z3w9BA0U`wKEWlw0klpI{Ad7fGBpoaq!*8A>C)S{$->c3G-k2sxU!s4F zFxSBq+)+#iAlXqP9S1UB;MTDf|MK=978LNkIl<@pkJfcjPmywNx zyX6Q`GYxUnih7+?cF6G97v(Ef7xqEq81!y+2j%+o;Rou2lr%|J^3-(806vg-$j|64 zI{?V&X*kv4hiOL*D|Sh9dTQBw%0Q#sdteF+@qBgK(}jJz%dAed+;oEQ{#>`$yBklV zz}ryKrRQH;c8v#ru?bYR_8HFHU?2w009xHBxj#YzA<6IV8f|0Oh^`$ftVni!O6nQs z*><}BWesbwwCnQ2i3|3QD6dpXa6?OQP84{utbdrD`xPL65S&`YOYp}FP+)gW1Bk|Q z;*u8KkP;GcMK{JcaN0R~I1y-t0LJg`@^5and&_%J0@l)c!wZmWHIq3)a%*8$fAg$4 z6d*r5$%o!Nka9on{=iys!$19Q3g3DCsrMou72-8snR$x(XU|?snLR@@HXZlrL!%ex zX}-3WC2}`M%(qtKF&HLzB^U1Vi%bzbuMp#g{gi`#urPBXis7fTmDn%e)j@TFUo?+l zEP1a_tIDIZ3^OW3u^L_pBTd>@XSHiL<0V?Scp89Sg)l9zhefFcEa7>``N@LT!6uukYllplvqyDM{kF0$g^~4J)Ns|nwTJ<)!DU4g!iQfmURRYS zijG31yS#|uYc!JH6uVBLoqN;RXev|DC5p`T6dFTfN|xKTOC>1qePb+v47$-&JOl%> zo21E6NV2JPB>o zyAiPtB(a$x?1)#I_oF5;3R)Y%pvjeirLi%2LCoB zELGdxdHB@&@POeI%1f&jQcCJ#?A;Bx6O#R$I28q`+I@wpb|_Vi?-LpImV4+z*uNF2 zV%o&n0tv%2>g}tU*;E}TP&ebxUOCr}SVp@CK7umoIdYUyWozjkfu4gO_$#X5MyhG6 z*Qt$tG#|M63L_$AIzdE9sw({7r)*sPyZLx)r#w$iyB?x8545(qbW_TkLxxjEkk!J<4Rt_7l7~zPA4EF8kg)HL?Rt7`k2=D}O4IgW4t!pd{+h zVAvaI!bsa|%HZxR`f#jXq+dyXl){b@*&wE^ucsXE#yJ@H7)Ua({Y0_Xf&rT+?lkFP z7*m+MmK+_ZA-;4*YqtcopO)-5Qd!_=GxWjt-3K`(BBL&!j@RKy*GhYW_do{?GL z6wksTB*qn~;SOO=u!J!RD1;HXz_05{(jso+sZF6camv>4H8M*QGw$YDF>ZUwx*#$( z(?nD&ZZs_#%xzEiYb`HMS+DaYnd*OdAY~AUs7|v~_?XN;PKW+YgZA2eRBSm7;X?#S zYrQT8oIgDK3nf5W#K@KRdM=J*cKTo|kx2S0u&O7v`9TsFZh8lj@(a@auulhqsWtKgpIWhc5SMVg z*UOt-nI!de?Q&qbl-AfEkHsERT(JMX0WuSMhho5E=VbcuNJDbGLtQDd0=h)0%^rwh z^@P+r-(sQs;ZoEk4sc%4`6`^}B&!`}q`ISSV0Huc9D*P=)OaOeDnB*lV5`BC;YG2+ImQhX4b--fiUg>~Aui7HwAD2ah<_JUI<6;Rh%4VRXJ zgGW**k@X%T^aEeL?wPb#HzO_fR0Xh6`X zEd74R0Cz0%k=EVl?$Za1cp?LP>%$%b8JCLZGO^$u4Fwq-PnXQWny3Je%-#v%Zf7Nr zMpRW&?ryPRe{`<2y}X8>ZvKv#T=fjD9dDl)$Isk)YW&hNzuC2!s6ad~0q+(S=v_NP zJ$uf(ai;E}{w@LWG~hv*mF8=r!7GdY&Zka?n2h^7?^~=sjLf5(W6V7+Lz|O(_S4J) zB(dLZ-%UUldTZ1Hlr*cb5Btm^XiLHXc?mc%CQQNcIwiu?=0!e{f;rTZT zHIEx@W70Z|#ad~Zvp&CUKqp5=_G&+qz{$7E+LcmY?sQ#@9Mj+r5s$im zWq#SW8?d_}94_^<|Cg};F^ciZb!RR50`8CY}OU zwn1$!%3_=^eml2$%v4K>6-FZr+w7UNLJp>6F607wV<8D}Ib0>2M^ z<5Jo@qf0tlEU8|2kCk_f{%Yr)YscafnzDl|BvHyYuqV&U-_`#Tpq0S^BsZu7KwcV4 zI1MwHvYLDNN(|%`-!YQztzRd#cI{Zj4pG1CJ>ef&P9i>$GED5b-|CIGUIVURKb)CT z4y_$8|8bd&dwgJ7 zbuR$!$0zY98c$aC@6o9y!LGRnuV=HX7p6-%v}F`g$2c^G2)yP|xH!*IE^e|ggAE~x zv?`T{^oEeE)e*SGhJ+$Cvv;UM1Cb{=8Z$*ImZhH%1nO|Oj$2O_uYaqHQ>g7wqXT;p z02HC|_&~d4pE=|o&7`17w^v`ot=dd}lNcwqEi(W{{6{5WqTvvU5e8kyk9GmTpKi*W zegjoo86V?OAMmDyP;EBR4%}v0CCYozhu{9X+6taDYdaFYOyyzH+JD7hjx-i;o=(NX zCnD%y?*i{^mgc`-+J4s>T?+U6rICn$7YE7<&zNVd9BVBMWJ^5jdb932;ie4elJ0D|dnQMSJlO~L;7K}wf8%7D@@aEZ z_l#Qyx2=8KdJaB2(1v_~3v&=QaE+bH>X%XL?TIo3Z7RIQ5^UDSIBr&{-q7@8gywr%>*QII&s(=P1V0Jak#4NpPyRB-4S+ZVU`{80_G6HnHnz-GCB5{P))n7q1{M zvJ%%C85SC&jOJaMz#KpO4Ku)9J_(zKXphxN(A+u(ozWy>Wn1>vIHmW`PW;8yWM=nz5&|!>z^s!EO8veQ9H@$1 zX=`)QFYdAg3tk2T8L3K4s8?K@_5)7qQpp%TgZ2bv+pOP3);BC`f?|!aF7J;H%1FdT z&QiEiJb4ym`d?DNP6dT|*{~78wZ>u-CphH;M(v#*+I? zZ&6uLYN~3<8+Pxo9$3zpTqqEX=NhWHR>0PfS4!u73Q<)A4GX?oC}@fSzov4-18T#S z^r_)w<#GU(y7zJ zb!5v8@1<`}`2w!d7hdXj(*fJN>RX7V<>c1$E*KDg(*?};PacQ1 z%Gte(5k@!sMCHp4%VKQk81o&GSutlYf~aa#A;1C;$h38PE<0RmZL1Qab0JssMwVQ}&sCD8X~F`&r7dB&OC(YG zQc1=bhIAq^g(UR@Bx72=E=fGsxJ-!qYGL+^C}}?X;`RF(pH()Beizp>xWRIA z%V(q@lfHsi4mr%|WiHYB^%HvpNW4jUEkEApW=+#$3N8olgLb`%i&!BXT_lr_25#^# z6bv4y8|uYs#Pfw6XxVDzm#vT57BwBO6V2&uH*eUw`5iE3nXz;>WS1p$*B(viH$5t4 zsMWfO{FW|%+1Qvl8A3}v(?QrkeVRt}siy5uW0G`82LiQ{RG6L^Oljb!+_*im-L zh$Io+o6+Q451PSq9_QH4*De{>n6>4-mH()CU`b2#Vp;A;v5sa_MO4nA|7Ia#^GmEq zyh(9sh{MoDNrSf;pqiraF+Z|*U!UKPXd5H9mR3D{$ZSlV4OBFidu};ziJsd6{N~ zA~|@!z>2=sAI zTF5_Qv6JwOi1_f^B>N`9_n zoDKBx(m(lh;So!ZAhtAy7(`hIR|}2#9v+vq#1_CRnJbuPJLl+L zB)P{bk=xG`kaEa-&Y_@6CsmgYLW&*Jgxpmyz`x%enB^t0iI#BGP;t_@*@tO2?B#Tb z@B`8T)GfKMv-z3iaMb`OY@J%6`uuZIBX%{MCS=6-1CaVm?$z6NJdlK|zsZQ}q0NEi z_u)Cv3KZb(jyJyi4%*}_0;{z7NjiZDTJIcCemxh?kDnd(XjobE*z70u>fW~P#5-;D z-Eu0(T6i~-D;mBVmNB{52kf~OFj{uo@g6In1w-VV}Lu z6yP}U#OMq=gD4(nXJw)>%BUvywoM{aHlv(94jGbG2~Jpg;v6JZG5v!W4j-Zdb8I-C z@CPimXh@7Ftsiewn0{veH~GhUWmLCnn!-&(ya>ch=t@TbQ!kOcNUTw{1mzslYF1`x z%^QT;B&k@9OalB)HHh3aPb1aTk^^pTy8_DmHkOt7azY`)_Y`Yw(0k~k$YH^hykJrvNKC^O|Z4TZn8L)Mt!Xk)IUD$YNOA0}(oe__9$Ai9v7X+FsF zv@T@TWXM*`o6wXZgbQoobP-16X){M|viGvpsCKRA5f`;C=$EQ3=x!H}7SN^TGnbVS z*1l_I&8Ni^e9R75uu>)}{t{s{kAmCJol3YaUo01?`}+bxi#Z+)+g1{j1<@7IY(!bQ zV#l4gV?Cs6{N{;*iSCpvg-YpR9{&S?yN|?HC;3``bY$pAtJUhvvMTSGR!#nd(h&zJ z)V4GE@Xj`4(SsmrX$fP#WOQmmYVbB6&}L%&LOy4FKra27WZU1Hb-`n!LZiZVT(jsKjw*y(OhJp;TEgz4B#P z?s*&4+wLq>W4OOwJoPaw?{YSRx@O+Ebl+n;#qxfS^cT~itd+Mo4RE6RHgEEY?)$5C zy{^stAW6zjyNn$o;I5`EYGbOBF?+OtF%BN!8V_eNU1c9}3aR_#AjAj&w{lUg4kdm* z+K?K2pA?>|`nW3&qI|wRS#_%gCqP@Ha6hp`2v=eO`IGeozo7XQKfah}90r~6r384g zWWz}as&hh$-zHL4CE2^tiWx#C*LyY1}p+5K)pW3AGd;{S^TpiQPL<7BQrH7YN%=RKMB6uGbhiTW*xrM0b04OGB&8zj*-4o zlB>$Cc(m~lL_*aj|J5!ulNuWH^=Od>&`|%u+@`RQ#-=Zm%q`Mdxb`7J%op&I)6QA9Sg^=G0^+HApVEeaB7+TH|8hoL-);Gn4 zeUfhrvsw{>A$SA3pqGp2h4DjLedYi{3!ZzO=&H{RlQw>(y1#7?*NWF9ToIu^ z&K^37+tL%z+Eq+6<6|+_Dy%HAhx~Nw$b!Vx8nr^CK*Lo=oJz8x)AHICvqZs7e` zB>PGB^bteLpv0KiR7p?UAwEJYZ77Gmp?^R770Re=XAz11Ywc44@=s~Bxr{eC-g05Z zmC2Dt=Vap?ZCq&%Zq!B1S|qWdRdktFLN5jVMpXaUXWOwhvc`S+O`;VVp3yR6d7E-> zE&Z7Dp+N)Nz@N85OfH5mvRxPZo1w@5h-pZ0Jr6-^Q1@V1U~Pciqu`gqJv;=nt+yIY)dCjU09a+n%ijuR z2*0s5{n!;c!^ zXrMT9`#uGMH9BPqPeoOf1j(F|NMD1UbqE+YD^wku`ty^?J0jqDobGx@kEp8eO}^acjQXOW$rh(9nq-LH!EfVy^m z3D7>QgNwROnI6{V3;MYpJGkNzbPf_b3S1FS9vXyQGMi0ZmFY^*FbrZ5>b}a*QJP1( zv%S3KSWTTFL@h3rcqORaSfL3S&Vw63T@y|p`Y@5f{#)L{i)iPAhHxudEKfx=?Y<#g zfi?$FrdX=bNP!VKrJ_mHMmdoj-Y_rLam-z_@*w6N$OPLNP6e3 zFZrz5CyQiqp~<2Drfy^<+E6RBqDQ&8`o@$Qwu`5(cXJLm;U}!vKJkW|L~~D&gj3R8 zs#L&#N<+u>q+)BtDPRDU4_b{mzTf%%q$dEYW=q4+VgMFg+mtUWVIloQRQTmtrjbo7 zt5+W@^yM`ir*xKsODRPQX;_f>SJ8@}mPNE!U)X~d;(xb33b%+?mwBjlnf*qn%I+`_ zvK|>1$m=#xvJI@6biejje>4uIRXW_%K~mH*^M$QI8@Ljiq(1TK^6wmfyyia9v0+VH zzaNHFH7H}(?zEYZ(ofo7`t@dq)uG>tey!ts-1K?AP=9~rx)xp_V!z{X0{i}6Yt6X_ z=E3xt>-4K|$U>0l=~GVt-t+gm0(D5L!ebpJ>gr|ctJ_6s)8TPb7vZNK=H;So`-rGm9~VuX*BF)Zr0u73crPL>ALZ>kOmW?e zeTbeK8ZaI=HZ_n9LqJN&vt>uL^z9UkWO|bIk(J4h{sel5W4OSaEj`{AgB`D|eaLW= z5K#VY2G6P5GyC92Fg>1PEPs)35-6eYj&_qxj`Q0aGM9dYvsq-5csdlEZALOpfNR_E z;qM`igo920k6R>x5x*9Z*}_3O4qos%MPoP!uj8sD1xAbI>Zqe!5;nM_`y=-~91OLW zeZ&=#wS6TN39T7R2BW7(nB%~#O>SG?(~=WRObCHs?!uPzeyt=P-6SVIy7erDQ_oJ6 z^XvkF?qpPBB)Avrbct59=%!tb*|%XJ(va_v(`um|LsnEfINB@Iy!t6m$e%f2Vg|i# z8V6~+!42WU=7t5ZN9@FOkHUeNsR_Z-NdsxD`x99y&C@#IF#6S9yl`E!TvH@@_^K@l zb?)Vpt{p|JntdUxadZWm0qkP73;=i1Q18x|e!+I%VP#<*S@e0HMB6|rw!t7+MwGRR z1%KL_Ylq*@#0B$juma-8k(za5C7WE7&ZqgD@~)QN)N<|jY0#qC8>AqcyxSgvdd2UF ztrM_qr3Iq++%}w*`X9r+e|D2-M`R)Y_7Ov5r&bdZwEI+k`lhFqQ+o0`@Yei7uk*|k zxGs9~ECMmTMR*z#Z%2N9K+cZ_J9mTuAGbGd(|UZ+)WupHhXOYooClIRuYVpn_uIZ_ z53PKdu$QWg?_vf1?O|J_{>v|==+uAccA1E5IS1T&0Wht1Tt|i$wU{!Ib3pH9YmDKG zZASqhq+*IiN1Bjg=NZK+oQN0bS`9-GsEQM8p0t@QxtJXrAy_62O+{{hya4f%n%+jX z{(CdO$@Usy&ORTI4vj$FnFed(*7}u>4tZA48ua^P!Z%K`x+O@wTr6&oKo&2+Z9Nm` z&zs0~fZIglSR@>n7Xzy%{faw^0M|IgeaNYN?0Ut04@@_n zdVDE#j(5cQtzWLI>i~$`xT?h!*9odfo7rICZ*)LE`1u#5Z5yMhJMK&R{T@LV zkoMwEJ5cz`fB?78+R!i3AQ8=`m^{z%mjaDVHBW1Z1C++4wL5CNUn^cD&4EMx^cojQ z6u)*y!pS1yLjoX~<`vRIjF?4|lG$Y{(!iArM%3+FHby(?aJj1#pM9n0#5t-ek|1ju zqT@`;xWAnSEWt9lv@l5jy=1zi;wO~JHTD+k6*mhFsB z>_)-0!Sox|;6;X=k3lzjYca9;B-_Bfd0r9=s>jPPVAee>;k7`pxG9y#McSbFaG8=< ztj*YrX%MI3w0<}X4+5dpALmX zbRE_1Js%!t;?2I_%T;FFUlows1zWcj_<3G_1^6-9Y^XsAO5_V+%bW^-H?^nBpM2f8 zpeezyNH^g!e&A!4G5bcckKrnJoRvQDBkI72CS2@l-R?mgjW77V!9hJIw?u3|`2fq0 zgtKcPXHfo26v_wl)X1M)=4#}4fw;7Xj)Q7RqjlzyY;m7_TfitmWxX=tjeE2om_$}_ z19vt&;IKtr>{g~9cLEp2Lo>y(Z!gs0s2!7P1HT}2KI!&8VkbbaE33E71Y@u}d+8lTmv9s<)62ou!hdF%PiT14oTpvM;4)AuPzsO_@v zH3+=yKcVl)&00kwx+WZjT0gdb(Yvu8q`GPKr#1}e+|qX;z&k(?Z=GQ zpSqtt)Dq~CPf|nT3XYb2 zXz@j;X3cNF-#`v5e06yPgUBv}8=H?I$v%+8e)+2w$}eS!fU@`hu=UjeQ8w?|f+!%} zDM%wJAl+R8(y@Savvik$w4`)*H^LIq3rKhF(kZZXm*4uvIo~ceh5{OkK?*j%sumHkX$&~zM3;HY1w`t>}jo& zxOqt@Ml$b-ud|1XRYnTOg`7G+NMp10hae2Vp}M&P`A36YDyp4usNA5w=fAs(}y17kP-_v@kel!Fx;Z%zev+4k) z?-{BTEqxs-Ulm6msN{_I?cevuDK|mcytVB7>8+qtOsrySrNsOAs?e^lT3}ee7co4w zFkw@$q6&$)U@Um{qUyD5TJ~_?X^jq0B{Ks%3Fl2M&$W{iUKo$!Lt)@f-w8%;{A!f^5b;?`OUNnVofvh4b~?_cnQCY z@;AZBo`e<50Q7wIF}jOE#E(GqiTQcxfCty|(0DVd=u@mi@9H$LU-bTyXov?s)@E77 z0OY2XXeH+NdZ(vI$3aGRAQto*-~V(~B=BmwAi@8DjLvtMd5prtXOg_d?{EHLvk(%S z4pez#B!lIH*|F6hnBJb*iJ6L~qKu>q#MtCcHF`HC%e%}Vbv%Xy{>*fy3P6R51g*w3 zI{48wU#d&bEfKdJD4INW{42PVvb-s);)-2F@x1NTN2 zRH$Aup|?n2U!Is)evsgZX3)oBG^-k&MG4fRR`4mcqVho)p_f?^O84vUNg`k#ej+PI z>!Tl549dPfTQC(+1*-UZlI2L>lgZW5G8qzLnJ{;=A@nUgR=^VaVF`YS09<7iP~q0{ zz@r13pS)A&>2MJ8BXVk6Py1*sSw)!PVyZK+7oCe#vEe=~RM|?1Ma&U{R$qeA8u9y- zyVDoe-;#&K?9YWdrAtYa-9|DLcqf-($rgviXii}7Kz;W4{m+m!9>@A-1K$8j1(aU% z8v+%=h3ge$-+dRj?#nmO7T%~16FAxV2Sv`HEQ@d#r+v@h>f=cXW6MS70ZZGI-_*Rvn^s-I^q>D}$o^Nym-N@Ve) ze^ww2q7n&irc+e^tbc0EQz!Se$b!RrUYc^+^QTE8scF-RYTFy&L1T{vu0I5YPC;3d z1m!#8ueBcOAdNF)ZX6~q+_|pv2*@ON-2|fmS7;7TT(N+g#K$3e30(E_q#s_&u@TKL z5a^29q9EqtzC2EflxGJ6g#{NWqW#zOKxykh^cUs<^r3#peFlvv4h{X!oQGA;C$kcz zT#)go?88wGJP=r&Ilvs=OJaIurXJ2CuzVE3yiPI`$o+rw9meVbXLUM*cwl>iHo5&W zp0(y^ggtzTTnCEZ@bjVfaB;U98eqwqCcwpwSYTMk$R4$9J`4-BlxXgZB||X14aL~d zvLmd{2-}3;{lZx&9h^;QwwgEXSC}m#a(}MGK;@TG*+I73f2394?hx5R$Nb)ljc;!Q zSH>zA-8Qu&=<8D&&-s_OGmBwZezSq$blB6uW&hQ&qRO-5N9Hwv=9(_(dB9Cd7Xzip zN(sc_MaM2#pvtxODPAY%+Ig%!1JI!Zwbsy&dBtuuRP^R&!Oa-R_XGGU+g%Q{g7lkG z1WWWW)W>mqXtYQAc87WXgewNg<{Q?(+~r*=$L@`*=A_t!^Nadd=Hky@X|Gxsj8p|x zljSlNxRs7lOI7hFV+mNa5YShd7KM#v#(j}8N43gA;&RTz$4O8|5O`rICSTcv2O3}j zHJGV0rifzl>LI+1sR?EAh+D?S+4RZR{eE7P;{i8E)MX z(08aBv?P33zlu{OLo1`dvmK6N4#V!?5@r%!bZ&d{le}b3SuPrX0|YffftkqGi>$hy zM(B>x!lSQQK^Z%tEcFn$%SjJ${6J3{MH z%t3qT3-0AOo}Yw~1B`>gg%7mxZADEyzvg85|BNAgg0!SxEJ`UUToOB%6n=<)wT)?s1=oLqu! zbSm3#ACO>X;-m(4TwP!3c^6^es`pVX7;m$`ei8e!go0x=>kdC$tauiWF=r-jANLZ6 zX%VL`r6=4kITau*wq_HTYyxBki(@X5nJR=eiLLBAj+jr@IVoFmfbeOUuS*jgsdn5w zhO68qMocqqsh`r3Xrrkj6&asc@@9ghlPd?v`rpU?Po%!l;5+<6t7H?e&5Pfb$9}n) z>-ZG;XCErEI@ya@ij_1uZEbpUjrsnRo8?S@q>B{I>^~a!VNXSi;dpKNu@BM|-DsZ-Q8V0* z`exDsd)hUW*GRf;Zh2woPo}IPkI(Ij0`-1QF5n$WFiuR)=WF34X!i4s|D7<}RYY*w z^%|Rh7XQ>^5J6VihNF7ym{E`UMgo-PvvWK)Hllsr~j!P@U3tSfP^B9^%-NwZ+LVWRoZ5 zCa*5mU-wdABUDjO!G0MpreN_wDu@Z-Tu0^}FRj|Hs5uOH$}VI0ZmvCM$pX?GEYDIW z@ivuz-y1q@Ej_-()DKPf6G{q@eJ|N)DJ&8olHnji=2&z`W@bu7{noJpZ1&p!+@jJ$ zUZMcY5>Ju221yn-dEl!kj1a%D*Y=A)Mx}Ig zh3q-{=>DjYMblP(pF9r8k~fMUie6(MMq|0q;r^7+ER&g<3jRngzS775A^4+m4QOgFEKPqYI>b~2A>h|mzI-XMMXjQtU5EVvoAnwylw zYJs8Q7;(6|-GQ%YX@NGE*bK8P`hoAo7_epaE?O{im8tv_r5<#|_jT!-$a2lFfkrdh z{O?zDBl%U?Y=jnSV`z$NkHRNy29|8SpZM23hpmG$(68QW5`lt|u{N%L3ypkuy6Aj@ zz56{M>+W(Am9Jzw{!kie`LNlpetVF?ey~34oc6bBqXc3irZhLthKFqwLmJwI){`z0 z&{V`!aOHJQ?p{HVLkfs5?UOIiKK(!gAa3JeQsEZLG=I@?_GoYFEnWWCRWXMcv?f(}-MSY(wqg3IwmxWS*Ja`bWJEcC651g@E-Ntw z<60*QdUDTiyZA~`e)3w|VYQQoqC)<6dq zZL-(A7>=8+w0pNuq}>E9zge0MFkRmo#Cy1GU49sDGxvFwMJL?IJuwd?9~NEqpyQ^Y zRXWq%_%x8J{+9_sH$uF3K4Jfjz)L|V$GM{WgU>_}ee|2(WGd;nx$JjEA(&t*-GAy*0p5X^I; z%&M&Zl#ZazE;#xeKESmI69I@+{=h1C9?f*c1`Ry91qk6Ep`f#$ltL zD0^RrryIp~`NAjo)5C>0fsAw)+swL3T7t#I48lkBQ(lv}+(f&CiXbojraW<%q&es} z9Nc+a4^M|7uoqZ|2j8?F7@u4wd19fXKwe}b(;Vf)~JoSE`E&ivXOjq<=%g= zZbxhG_ny|_bY-Q#zvz07Uq8`6+loMGYI9`6;}~m7L$@f&u;b-)h(KhM=fQQ^j!%Z_ zF6^xJ{L6!nmk zVD2dClYT?kHJdU1Yy5i1T;77U*}#3qzkl!}(lS}U2NeUpjTn~2$rrTyKCLZneXZxL zSv`)Pj;=%H23H`LqX8nw{WL)a+A_zuN`RoYlaJB;?vHLO!ZA|+yg*HV|7j{72g;#` zmV%Fh@g0wc$dzRLX`^U1ca~m78V0yG4``l#g5n)Iz-Y@D}^#p(WN%a>xvnyK}9bv-PFO8fdt5b*B)CEB8A zoM)dsrVqW`y#ln{Z_o6UmV`CTxNhC`{>1CG9~jgIGh7mm8V;Y0X;=vxU*;)6E%2#0 zO>B}AmkX`yTiEqP?SaKW4&p`$L->e3d%0fWDh_RHFnVawcLTD>xrK-TO znfhOHDcWfI-6?8N_Y3HAvFx>5(3w@yynNb&{E+L0TLYa`*~9`J8?PiA+j%=8x^& zDc+|5x)-E8b|}Bc`@NsQx&xxWk5E<`DP3-K=I@scg{}E&s~)-VH*gv;^upsx*r$3uK+93BZP~Qs3dYQdIKoOSzT}Aq;l!b_? zlNvTKLrw{)_7$L^&La6CgHd@ylo9b2vDmH_D9D_i`Tm&(82sDk&913pOG@t@2UD%@ z+{KdHt_1=`2tYt#WyXZQ*RSF!OVFp{pS{)ol6zb9@wp`SC|RF_5-z%Btspw5HHIDA z4+PPpp5clnf{6ZUtI%0}H%`zORSD|MZj0%`9et!;Wwk)!(CL`$(fnN&K0kzq5aIVs z%QR#d*h&XmwI(jNDE9`rP3z(PgL_79bDP&bwHcH=+$S>f<}OAqACJ-2e%UORGdtO? zFk4jfRQR;`A-Jdv|Hx25Uj~W~OZW*2_SKy5hG9rfpIx4NJK4zN>Sn>usZ_mOlSsR8 zO|%jUw$Q1~xOB{5_OrqMx=UV)tm9YC`t^#{!EIRgVh{jlYr(wCI&l?OH_{d-CD$#@m--+c;gt<;lGM;|DYsb)T zr|c{_%m1QE{Fw$J<*mtR4i^QrNU+@G=jxJtsSOfWd#Tm+hl{p6ZWhVGQ?$e>m zo_oAn+^V710-d69j5VE*$d+iOkZaj0f7#h9p8a;`=F{7D*CPu}Vz*fa^k}p93e?|v zO+41Wn@t!P2OVuh$UZIeXVm3QUo^fQNH4iQKUglwb*J@U&mPGqk=(-uRed>nM8ALJ zIw>Q9Er=&q@r|(BmT&}Y$-Dn(3e&8iF#UA!=5P$PQdfwuYF$zy?HjrIyT%6kv-2XY z1+UK^CGjBXMpGlhD9hY|2eR2F2kOxe-@RxLMP!zPDi+^?e%{U6oGA2CWRHAzHd}ZH zMWa8T&_|Osg!E}QVOY4ka2+e>NbOg1SV69EmYl!z&vomITH;nXPzFc(Lo*f!dPN0C z+QM?(`}rhR>o|U0Q#$P?TcoEOyTXX)DzitzO?VIIHtm*wgfXnzbs-SCICPp4 zROb9W9(D!-VeEeT?M8(4`|-f{>9$TesaAo}XDn+h**;8{a5*?zP_?c;**JI+ zKQ-vOc%{jsoP#4&7@y6Taz?@{UFX$)J6q+}z&9NxTYvV!Z@A46mhvfae)gv%EZkWEI)s(-@U(l;SxJb{yQI^xNcG&gn5g9VhQu z-@f!uhwNz(&pM5}Y-T6oxf7X()Ul5{;{q$;SQ`i?b8n2fZ>OfEXgH>9TGyRTM)&y8 zR^x3pxg%9+C8!HBHp_$2`Bfm$s;V@|zLDg3sYB`O6#m`%RLCq{mVL5Rum+!4zrlV_ z`P*KJG40mhE!k8>5J$oDZ=E(@XCLaEA?2}t9v!Ir-T&{`XeMgRL_;OKrUnSbPB=bk!OdB}a zaz-Pgl8I2S=tnejb2?F<;_G2nUU~Z>U^UfyL8rWfmFism4K3X^crmnRm-5ej=SZ)= zbwy~hcBCGMrg-~=nyGQ+*w$R*oiJ83!@bHb(jA<@3N_1@uk66~w*>GC2RCEvXrIi$ z!!(*NQ_2!lSUi^!vab8V2>Np@rymnD*%OGs-S{y^*g3!0%MA-r(i!AFwv#UscZUbE z?+>WLC}JL>R)8(s^-{qcCoe{wE%UwRDa>a_Wgfwa8L{($@iDlO092_Tvj$Zak&p+G zB6%PG#x%Q3N2%V|m>W%m`)f>a1f+hxD%Ilm}ET)ylT8$~nVdxWH3Zvr;HJ^#@&M=u1deFvwDgz3(Hs|1i zo(i*!en$10#KpXn$4^Z=VumF{Hc+LB(7!ghW30RAoNW(#f22-wr9CDrz6;5wCRf;C z$zH%ew{dBSDDCE5y`HK@3EY?+(ocx{m0zi~!=14Hj^zBgA8rG17ZSf!v=+2Z~9cG-C`-58!K^<%?0%MxRf+F9XTqnfCez@m@Y_sHx($HdzxNG zW0*3WZ@{>VoBh7uI`iBOxQzZ(tN0Z|B(g>E$jQUjo+*E~`(V^7Zo%Fb2_f}3O{$XA zY4PPex0mEwX^xdv0o7XF;uFT$Ta23e@VBlp&3zCZsh@!VJSTMAJ&1_c3S6u|$PAhn zOWST*P^+@PjiZY;^)y*TIvfr2bZsAtzpK!>rVHj~7FloSS6^&4!tW)!KD#V|ml9xR zz{IRi-cNl;eawKBM78{Xxd19X8M+HD*u~Ld8RRD?JDa`_i*M&vtc`z`ZQgV|y0}{- zGmE%QtbB{I$(ZE`#tdE+?XrsX#gm);PGSYfhrjJG}Qrj^|uYHWY~p!W|gsj zK>$nu=-LJnFQ-@tf$&?EzXE$@ka;dKT8U z88v&CHr*Pi6MT7bp(XZp^`1cWOWiqKrzFKt)UxB~aq%VtXaWe>c6LrpgGW>O6?6R^ zh3(~S3~qK-zox@?qf|yhka2x=ELa67jZ1_G$N!Az4iqPXHEs+}}s^GE6 zqoM-`)WN8WMjZM=``y>hAkCEMkUmleSUX^7Q+ z+?7Ht;*+0kG@rMV;tw6KyCg^_wKMyO9{H9K1;$kT?Er> z4#WMfMF+&wdO%4G{sN!%l+c^8+z^4XUXH1EE&IuQ5H|)qgWb^7p%6J9r>|0%b8QKSg8+s zr0H1k{E{$+dbjwH_OQGrcSf2-g6qaC7Mse^PuiN$ z?4t|lbBCji0}|#MDA!jEW|1P`ttLU`Ldr^5Q|H{x%v-q*G@8(!(T26O-|!3J0z|&z z1fxG)`P7^o`P`920qfAWeF6l-q*us1cO`>64X-((ePS}w5|e7%&wu~;n&a!i*&BM* zRBO+iF3v;15U-JK;GUW7?Mo5HA38AH|8jbmag!dm_M5=B?YT@NZ6c5!sIzXigL3<* z+g^f6q03NUB`v>r=o*@?ffgd-0xan8SiF1CSzrS$D|DGf>8nI#S7>G&!vl+~Bo>hL zCX=+ABcT_ewJIVi@_JC8;HtA1LXwj#M>}Blnm&=!B~gc5gqjM=d)9mY+pm0kyyaW^ zt@80pfggToDfV>uyEyGyxcRJE=V~j7SOXq%& z_h8r1zVTQ?$wj$Bdpp=c$8 zj_MPslX<$9+aX%b1y_uPbB`F+gyja$F6x>`A<94(*%gtN@t6#BN3g{^zRHPm^24dr z!ZXXyR_!Z|%|)BNU+T5ATEkX#se!X-ufxq1r7Dcw@YIfILzM+^7}CZP8U#)~{e)c< z**^7geFMITNC(o(T=&_lKAst_481B_kG>mBLVr`N0=yq8quXIv6!B#0#;3P`)!JkH zdHK^F4)g0ch2vd~qmI6fIoC3HxE?tRu4=TaWw%%x(;@)zpnuoM^n^$iuK?r@Ki(#3 zd`rvKQdvl%PISdpV}sWZ@SPWFPo*NizI68x7ICNp)Tc9M7B#aHroj*=mGj9mAz=IM z>s>prJ=RJN8@P(F?CNPj5p17w9Ggv*Ef=Q+1Y+_Uerf?Y6ZuVMmrpSq(mv;~>&66u zfe{d33+-WbD`UPxEJW<9@9%^Z~baJ1g^qO`gE&{ z)*E#W`${$(I?IEpK>(6Psk-@^Hf&j2YA4~i)@UFNe?bEv$7(CEUmRan7&C`EqDoP= zpuOyTD`V7Zbg1YKNr{W$G=r_rzT#H@{w#>BX`|kc_lT8d;bBeAgl>appq?PC+5GbT z4s#Z(Y$sLi?{{X`J^IMx)*_=n*Gp~()jPM-79bXgWk=*RvY1?bFoV@w+~|9~mg8T3 zKc0?dZ$*peYqtZfKY%VRGVL#M%RGCeSY-MM-yEvj{&2DzdTbgcYklj<$rMe`>co_HvXv!(b`MF5AZ{+#rOBH!6^;L1HyJc?;X=NOjV%L~V>aj*S z?A|QLoTj*Io2l?f^%S!V^Z|cqvVevJvVZRW`U$W6tMKSl*~4gCPnAmN+fol2J?f4C zz}d&3#k;G8xvAPVn+>g`M!`&_X_lvE0`vDD19gT>-)d|j3`Ed1 zY@u`K%uHq>m=srN>K-K~q(USpOK-B>s&m32M_)^w$69dr$ebYX!v7mGc%d>hnCVH{ zD=E2Z34Z{pRWBqRv|!@DarFxIr2yUWTS_ZIxYjKRzY3Ph0MzHP1Q#MV+=atrQ{u}@EV8^Cp2pia*Zn(;Nw#ur7J6L>=DMza) zX|w6xtqCfF3O)-yAz=k25xbhVjQ-kUY4T0@s-JGf7}K{ZT`bWjP`K`DHYgy0G{YoT z*UWuZr_9r+XI;9Gg8kMqyjHfwxv|geGPGHfdEIiyE)r6H9IMw7ZOUFRk_?7sAKXtT zZEN*eYkMPWLVYKPY%i||-fi02HT`hYkE%$e^YA&;dzw{c?$RXIWCdWcf~1d?wM`}O znU@EheQ@n9(F#iDPE_jvXOWK!%(~0~4EIdpe(pxeH65;u0?PA^3uZ4Taiy|tcZlrB zDo?U@@+S)m`XjzG4$v$s$mdlu;e||yf=^U9gN@1SED&T%( z{v?{Y(?RgfF_sL9oD9GL?cUWL4Ij9S8nM!L`X5xO=$tJhF{rq!ED)v~9H7;3Z92vh zulYRS`PYAsd8>VqN7))#qZB0TrQS#YuWBFA8vvJW$9q3ov{dm_XBTZZo$)rcZSVvb zYUy&e8MoN`B==r$Mkcr{{z@&S>&yMA0#~8J(s8VPB1ulf3;*|WrnmB~pOcoH#o+=sAK@QqpnMJ** zYu3clV)rB-eNDYrXN-x@v@(6%W73~qT>Ih`YSF8@$PS4S`)1aixd$B*PtAu?%_=s! zpE?JpI>VLxSy%eZ&Tps6p}k*!Y1GlJToHekdlxvH6Dz2Jiin$zR|7F=ohZzrLSzyT zCS)L=J1t$%aIH^&A$u#rp=k+E2GH@ZEt=ImI}NUC>v6igT=%YlfvaB! zSluqGhTs^1)^~qzjWv4X_H3iBkkOyGXwrE<=^9WO~f`t ze^cmLXC(yP`V=zvzf5|pLJPD?e*BOLH^aC0-(s_dN<9Ycr4?n~edCcqLoFJ+eEgMv z*kp5@OVcT^Iws?VqTwB&PZ}?V3wUEiFQksjuaOCIR%bRGc8JP>%@D!iRU$v?LKEO~ zV5Lnr4S8QFu?yhA(9OYaZ%^em6~YT_PN4LXaindCH5U!m%o~*E0Ac%V(KL;w^~dgD zO8eMj4I75s&na&v8~EG|S-?{k+@m@M&u!~y9he4Jvikrb0lQ$fr^aq#YhE7-i)fSNMX43xmRU+wdWagDaO#_}zbN`c~BGKzrh0K04oT z@mB#nX1==y#^pxmB08to^-nGA9o+ESt9451DT$!YXlM65P?8~Z?i|CW`LYC{XNI1ho z&@7y$m8O-PD*cb9d%vE>ns`1GGuO{f41%epTrPJI8;4x5$C^=I3d2^T#yCK?Uxx_> zG^1X{D!9^~R=|(N_dD2WZ}0>i4Tjm2I+)9C6Iw^Luu>ei-aMrb2MxZ-YYyaESTA0} zak1r)cIi(Gh#4A%kvS- zHh7=cm278Gl{9TfnW{#7pP3Udhoy{Pe)OU+4K2~M11PSPdlfqbwRbT4XMu78W(?w!US>xk17gF#iJKafCTe63BolvV*MI8xKk4E$|9q0e%%+^&PwLC zXgd|(SbO1bL9VV~t~i!3iC!7w+qC5i#7cU`BBADi`*qGk6yQuiRZvfy5e$^!9bU@j z2h_zt1VDlHc+S4v39%_z-*VKx zj!pvWB7A4u--!28t%&fX9%K^gahBXBr?K85IXEo#rt@apmbzh2TNw72x6!}zp{>1t zrpQCSW&n}$uY2H>d4I%kaVXxzon`R>#(30G5fh@0>}K5ZLNCLs&X`?v^t-d~mLn8x zUx%L|hqZ-fM@v)2+eG!Ge@QIkV>-lcmrR z(BLITkZQqLELpuSJ;4!n>^B|{^pB>VB6vO`vthWns^^g8@#MqaPvHZsD?NQbwMevC zUmQ2!-&SdtTPPmwlR=W2qw^~42Gi_Ri`ONZws2K4H!8b}#)r7V@zA%=2D#8MW~qR~ z4Rmd_)}%2(bd5`%%WnbobduH6hz|bU%Z|E}YJH0dkYpCk5u4A6G|=qhLUmC!I7KDM zRg24mww7P$_ST>~5n#Wf+HQaDl0NleJgEb9=s9~%cjvTzO=aBcw5Rj52F*G99Dvtp z(imPM2d&=EWINWeO`+)-a2BT zmfE-E^Q-uGZuT_kRXf(wM0~(HW==u#eVz|$QHVBiR|RV;g|rw4g-Nkk&~3h_)d^01npvSbF1kOsvGFS@UFJ6 zs^!`m5gHliS(~I z{)@r?H7XlF9NT7Zp-*}BFE8USeEzStmlT03M4&b|Lg|b;6`|1>6#&1$_t3WE^q!uw z;6S_|?dDjvD|Qx?!D+Xac)BhFeEXj_?{f4`@t?)*!r8-fYGd?{yw{rN0Ib`LI;}>Ir#-Hr z6V86mZN%&oQPy=*slSX6JxMsX;TaoR_t`@0X!IsM{}6SkI#+p<$T3{qQxY)pP(`!& zu;w}4qqu3fBPy6mDQvfyjxbNIM)hBU{BN=Cp$2vkfutqe(ngIA(C)pe>q3h0CY(zN z?qLDo{$I#C&FlCTbA(7vliX-z9%o58jqiM|R4@CZ%_TEDkL0ViN&iEq#2=CF451W) zKkEsQokWF#7ad?`@|)&%dfp^%y~=f$5&hJ|FpiGKyP>gUh>D2GQ&ZiK`UlQ_bjAxc zlK;@(9xI|BHT*`is}~I^%I33`R6p)}WLa#L3b`TL`vpKE{w>eq9w*27^`hw_@j#~q051z zQvkj#k-zB>f9dnTTIk5}CwXh@Ai2MQmz5S?HPYzLC9~d6P|Zn+ZqFK&5OevlfM$56 z`HEtqvq@u^>+Wa#)pB|Y$z(W->p~WX(H0ZPuJOFD-ayioxr*KKJytabddhO*(L*NT zb=s_aAbbB9gE=AM{~r2+Fi->sM2A}Btu{3$S*bS(c4I=-v{wPXng9QG>%W5RDw4{D zXPe&&Nk!Iqc|UB3>>81NYty}|A%rI=v?THDMJtSM1pjxCI5ANikfYfy0>$zJf0>Fh zhs6-%k)cppi27`@xbuG)=ri_Sy4Z0PfQs$^>~i>utqI{_JNaY>4sZSk1<$=k4R40| z6+6rR2NCm)!%alC<=PqFe`wtEgWCw7oHD5Df7q{MTyT@POVTW){2v;UNsF2e}Yc zz;ACIog;Renxzw8_wT{*|B#BX{+%rPd&J^q4AWRTRDtdZE^x1SetMXvK#))FFHj*? z#j1i@)Y`r>U}s|nLc=$0ErIDK9^)z$xPG z3YB0pj1}H`lZ0VvW5_DrMM<4K*R=!2L&3Xjv*J1=O(PBzOO=vA&OmcuQ~=v*6X7}u z%6m2xKDs?fB0?N70FwM=i)KL8b9|_ z7uM{DlTXD97UFw&yHs6^JZ0AV5}_?j|9#1m9fdd37g1uGq)}rx&y$dXFHoJ|WGl#OMBdqr`f(vm)C&;QjOTQ-!Qi3;=aFe+5 z;imQ_3+;DCg?Wk4+-b?7*IWk{vzs>ulGtPECd9&_132ADL)e^}{;e8N3yEeZD)}U_ zF^GVbq=)F|25U11vlGPJi>;`G{_ib&K?B!_aZ}U7)y|yf$e0sQsGUe-adBt_9)lVH z;6tHE6^r|qdIut?Y9ktIPU52oN^>sc!M_St&Ud` z61rFT0yX$CnF|1H$DW6Pi0$_7c3D9NNtRt6@5=rn%jE{(!X-$tw7oU>?Qo&4z+fg>R4oE( zTip^$B^jC-HuaO{U+;*w7rH%1@qKepda&_!or%#tNL?|tXTBgzNqc8ZmX`%Yx1rT? zzZB6rRd`O4IbzwOPlA|UPhEkeBBBPvoV-;>&y?bP3+32OCYMt2M6;wlU3}2NC7q1i zApB~VY4i?$tDUsA$j?xCRYO9spjk87nn=D=X6ilKE|&U@tgzV>0{XO4Yt$L~4JJLo zU*_aO<&Rs!U=^RBq|H`yV#9=XysN84CbMyyLyjOnmH40tsgSyvzO$OzX&YJ*`fdv_cPG^j9a62Kw|@-(-Z)aZnIdM3J}vatZqNcA zEEa%a%?*J13SoLJP3*LB zkTj)z&rvhSdYoe5 zU#9xcl-PjLg;;=gWhpu@E5%&mOB{{|Frq+5&^AwU#Q9xd)@P+FM-7sw_o>p%22zO{%z)-gZ{t(X%4K zdhWX{^B9fSvDOVsTKkd>mg5%3EA596(D*}xjsx;Z2K_&rQ58h{duabRi=M*FWO|BM zIFrJ(OII!2XQ)N>*^HC9WKo1PYZ#_T)3lGTQHxwZWrnXJq@nl+AX@6Tz$66Ly1u#N zMZI()w$2ZHHd&b=1!&r8zy-xcH#aH$ z0WR}EdiTK5V@T%BDfK3mc2_Z9R&%Hf$*_WwCX;r<&4fMt;0i9$A2kZ$1T}o z5@mN9OIvR@u6*4sypsQTsoZ-cZv$v~Y-n0;zHo$;ld%9ap&|m3C>%bDHTePEgI>;0 zj3F*^bCeJOM2mQP$bm~m)RIc1G9u5pO>1^taKE{u^-?*kO460|W$0asB95ca;7o~} z>>*efGbs7ppp|fFs|r|#g=~QI`?|cBd@&6lm%n~C`qKwem8lTsBYGSxqxqg0pQ)RA zo4yv#3(%^6+XiJK0jRnflg=NYr?)h&8l;W+?|#~q>cR*4tL^yn4mHjPR;;KnZuROO zK5Bd?h%<(B&0?`po0q=5BS~X?68mB%Lucun0YC^54oN!DkBl2vd0#{rX5cBTm7p4a z@ewK!;|f-JlTK}WB(z0`AD!z?A<&RIf?HL?kjPo8?bHmMz(e@a?dcE5yL;vjNU$1n z@IS61E|k2yAwht}|H}oSwr24+AFQq=_*h>{5LeuU=uFgvI5ht_vdv#I_buul%5vDB zRkl&(jd8&SS*sv|m_Llc`T(C|=MVa;^&=yf6IgBf+z`gOf^b42DF=FsSLg9MslL)A zebPMg-J|`%=9`|Q!4$4eS6lnCmj2Zu$*e-nZ9@#Zac_a_$QFSX-3$yTvkX5bH>=2@ zE4&HWv^!&OfvDyoEeTvT6jP}m7iEwDsm61Hzhs2v@9gy7h~$s&mB-t2bF!svU)Xn% z1`fXbRhm<}&am_vWM+;yYA#QYOKYOK0S|xZBN^1qS{fN~_JdP#E2I@ChA?#L3QMVh zm!87P0Wlv1Le6UQ4MwuFnlTpfEXb8T&h#{$6|?zWiqnkB;icLtBI4#<^iP${<@=ik6Hp z0Q`+g-zMXk$14YVrFTSKN6MQjbSTZ}CKL<^%PAgMV=}0822v9IzO!I^o=6A*7*`ob zTSS=O;InLJLq3Q1H5 z+)MXEk^g&}{ntd{vhn#7vSjPV)_lBQ^uPO60;ATBSb_&ELn}?f^h`vr5dQb-oWbLN z1(koZt~Y<2%-cfJqibKE}QHgDh&v5{!=cafy)mj2CZ)(VY39Ln`= zKXq199LSngihX}%soA@0XB)J%w61@>EnnHzvQv5FB(5TI!LLZcHw2>S#J#RX)5G<; z@uIKwI5s3DC0*zY5RDamy7BUF{gXijuF%5^SD=WY$G_KS?FT$S10Y;q36DU8QYovl z!I%5vRWCDmeZf(UZeYS#;}1!M`)SR>V><)ySDKAURXjrr>=*kIict%X%C8c$-C|#} z`*yvq0X2OR`eDJ>u=`;x4>ItsHeC)?S>$yP zAlt>XkljSxx6mtpbZdtAnbO^--UHgw6hQYD+D_S2itorJ_h-64{gYj4& zk!Up`h5v*=g!U2ftQ%}LfB1i~(3iTG^X3X%5}T{Gfch+U2mAY_?uM}xOhoXynPWG~ zMMQHJx9!}Eia4Qv4&Sh`B0@k8DUPXU^bR1XvYDrv=^zizo#M|v;4TJ{S}~Ya>ql+HwT|VuA4?{hU^L&iQKGAYTl~jpwU~z;>Y9b;*7JRrGum$Mjze9B z-v!&>q3YQToK4irLv@k{P^J~06nYKeBx6T7$#@59LF4MOqPFOt7!G-Gh;>-))#kz4 zkPUI?uXhp63k|lj6&Ot|Gu7$v#67nw$7Jy0Z3&y0Gd;9@Jul`;L+Iw0l$-Sd64N`M@rOF{v9vHP5qf5*&F{D=s})1vW;Fj6B|Kh9lD zw0!69a6!=f=^69_+ksk(OEa1G@9TkwNp^-tfi`|21(mva)9-Ehlms!p_Xel68@)Q% z(QDH$TYpe%3vYoDmdVw0n1k9m}>s-!fO{lycK|8 zEDo)%6zy)!$?5_5GNCCM&tN#+?;9NfL_$KEv<$uWP5t!*({ffhBK-!lSZ*J!{J+}? zab;lEn)z{|NlvGVR<~|zLC&ZYNe52A^{5LOSf_j5Su~g0bw`R-cAi#PtWh;P9VDZ6 zo_jRey+g%e_WBI~%k}A6na2Rt*`YqzX|ZIbj|nB^XjXT>sG#1T+MYZ5^_ewG1_uhP zG4{lsO!}}J^@9y`Aio5l^Z)32>!`N2ZhiQiDlMfrlwt*nTZ=m_MG6!v#jUsp4-SRm zF2#zK;%D2ye0(l-`gL{BwRvE10>RM*Nw=^gct<*a2 z{DoQs95T~i-p*fR4V;6zSSkoB8UYS%aHeNXe>W0G?KE8bJi~1|$Qwh|+j8R}wIP$OYKS_WVB$clJQ33O5SIT%<_?++IB-ym)$RY4eg)+QUpR6Fao z>mdTxE8@`oUV9!fm~J;&aoBda<5mm#{_B-k0H{RVAjhzKnn~`V$A}XZ?WeW8< zD2{U3ykw$>D)-5>)(=tsHF(Omm8Z6b#U0zOwG-8^&GG681Hv0nZflfLh_S}yUo zo5yuK+1;7`OMzD*_Jb|!kAN&lSr7{eC>cY?K8szmvD1?ofVEfg!lb&#OwMT zqjY0Xu)|fRUfVm@i0rcIBP(r?lEcE5kXf-X(cGbO`Ail+po8Z?WyowVX=bew5op{laKrXE;PE9EI`Ei0v{V85CQNsw2v>j+Q zS1Vy(g%+fvRDrILW8)i-pq|IfDw6O{uE&2*$<(%qws7v%OiCiA_c9gOrPofsn_YyZ|m>;fWPUSrrc*U zdis_5d&`c@HugG6gjn9;eBglD2}#|C6C^W+`ivr-0RvL%v;bAs9Y8Clgvg-5hRf-GFNT5h=I|p=e!_>xdkZMeGgJ^Y=MKpTRwNtWmXKj z)2EV*zcRX&1r~QB6173-SNk(Gd26VNhp5$3Pwu`KCYb`F^-X(=w38G!(~5+yC@ z)ppT=!0+)7V2f5#shj9ge;ongDh$l2&&$l^=)x-D>Ge`EZdJlUHi9y!(j=c86j%NHouij&O~Qe2%KwfNkhn z?RV&0UDElD{dbJ|hjvkrJ+&stQv(rpwj20X6GU_G0{@m4nbP;PJ!LVYURNR|6FF84 zhNd6BV{CEvel(NIt!X+pupM3BL%+A$Uu%l;YKlc7n_%(_u5t2gYm?O4 zgBUG-pA^oPOEsU<-)IL5@%2faF*YBh;Cho6D>qeY4|loU&-uKugBtO*bNBs&VtT&! zoTgnxm&dj_b?DjhLyTQBOdhwc;%DVyKO|23NNCu5Od8c~u->oJ(ktAe?GGeoa8ZJ~ z-)Mvj>&B?Mav7JVx`z2`e4wrQX04&+zJ4m>KqkA_c-WxnQx%_&xGIo&uky$1My9lq z+cmazMUn$M0`?GfXIIx8@+a4reR+vOM+sA%zbO}*GU>OY?XRjf5gMt+y>}nhfzEmM zYSR~|aHx+h;uA6<^xr&OUft?^0&58yFHU4&R`~~gl)M0Qrs#7Jn!tFuQfD^pPzw^o z;)er^Xizd-Z{=v}*T2f1JR{KYUS*{W_|W87laGbeXw8HCB0$5s>-hL#cj}CDcyu*B z1e+S;KO>)CWi^R6@4ZBq$Q#9H46+6`SH^~E=>50AsFi=s9>~OERJlL{Dn71gLH&*1 zNUC<&T$_%jr1=G=W~;AX9MUUL$+hWK;vcQLm1k(m4A}Mv22hrJw~N|EF@@CyViJ|w z^_U-pj$GX?#k==Hvg`O?@H;0 z^(@RN3fTf??`=RIo#jIuZMUdUpgx0G?=vMW6Oj(#Xdz%LjM}vCsn7zyH{?bBwx&(3 zmk0=D2+}ni--MDkYUF<{nN`sFd1{ZvaX*F#)WlrFh7Nd#z^qQpqk z*WQh`1#10-PN35+q$k^f%T!8IAL@` zqmDbk3(thZxp6cTY99-Y@%{mAE*SlM=s~v9iAZXn7v)6W#%; zKVScdK3cOwY93!R%(lx7JSSy?u!2Ff`-9d$%z4-p&ki#=+TGSEzrC$J>*4*T18QIe zsj>l6s14^;Sn#J08Lor$5slA+rHr?6Z}9idg|C;_bW%>c&@;yaBGtMx)YpuAudW&G z%2hzU{VAsqlItbk@-|c1YOl$=A24gy?hum|Qxd%z5=?l@>VBHw{MnN08LFTEU>S7^ zt@TOm6>kpR?L4fp^GYqIY6yiC!w_cbgJxM^+;Y)lg_0TM`5t9|Z|PZ#0RU01h9B+5 zN&v{r2qzZPYp*$%snhVP_cRxO-)eP!;OAid@z=25EnbG}yr_1-UpQY<=L!E>KF<@y zFLuFlgQ=mP{*|P4ovtR5IogII^>P~IF1fJ^#PK)*uI{Rd+`uW+ks|7s!1r-S?+jyn4V2f1%*2Z}WOJnssUV|SK$DHrN4_z&*$Lk93RP*|zD z|55sUZp8;|4cw~o?eV{{puwF z-FU#m^_waC$a;IFmHrlkp&uLnsPdm1^aubA%(ngeCxs`%|6CIgzgIO=^75OIc)=R; zZAae1br-z1U0d&-WB<<;dxl%MQS$xue^d#Jzh3~A`)y%fn0pH&er^S7BK$d3orU2) zP+tkwEuLp%vl_-)_IV)rwYeIe;J>8)ZN&u~SFtj^QDofaLn3hbwf|{>#OE*gOh8_j zzfQ~hR{+wIu{eF?_iz6^%YvIent#ehMz{WWOWEOic$3*ekaoj}LoG(pFc)=EejBO% z+cUWeKO_-UHw5A$6F3gSCvaRbH%sIA**`sbLH#$qg@TIb)Vpe@Cq_Xd1 zU;lx4=V%Q9$aHy zXEZN~j(^zvv}DkmKJSi9cOCMw7JT>bK*qHKxh<4$7_N@;Kb#};2W2QPFVG4O!dBx< zK{(_Lz*%VIZ}!;#9ke%CC#-kW*dbF*{qb*De{j(y!IrO=aDC6-wG;&sN#7P}ABwL$ zsht3AuBl0wWml@z0f`&9DK$XV$O1YYJwsid6z+X_QDM{>hVm$I)n)%ws@DL1Gp@k$ zXY4lQ?+|zHKK+_caWi!D*ONubI_T90TNb9^f1qgHS9jzP83GP>I$k}iJNJEcNUD@# zYI!lsrHY865}id*$-s8+TmSS!FAj7gpe{TuP$2BYZJbTliv(}8g=H)^Pq++-HLwnE z?KF9!e~*l`USDC{FAuwix3vGUJ$q(?0w$2S$WXk-O$Ki(>HY0nm$@iUlK@BvrbUuX4|9m``de0Y}&p7+awBRDNX`Bl!hIu zLXl2+_{W;{7LBSD(=#9X$5qC0ScOYTJ8kJ*SBwmGG&F(<+LjX$xC`9OalimxMFBM} zrbLErQZCo#E+!&CO`@eA9{ekEYQ0bMBk1#Kd2Vj59zy?mnGg^qzPCOZ3nmHIsSe4X zK5SRq6ZEQogDFj@r%24TEN{2 z=isuvvDS%|BBA`E`mMAv&U^!Ed6_wvKGd;D=E925skaX&uG99U;}z;b8TfC{6;i1C z)I+zUe(C~|f??5X5d$BL(KQ&eUw0mRrR#Z)5kYT>IBi=cUjzb|KVcXLq?OC3bUs>Sd-EZ4Tp2pBozD2#M!No)jHejd|K4vii<9H7vx-@OZk=nXIq z>G*&?5SGTbt0Sbs;A>ia+%JKAQUrMS)LB5?B9M=WzFxXV@9)I5_R_z&{hOgnp7y(x zngQh3woi54m1`4Xu!!<){?dY|477E!v&X!{)QvX94L9pKuJA9sZCa57>v})!RMKzv zTKDqz4mq?;YRpQPTLqn|IO=cyQ!N?sq2$Xr#f_T zQnmLnjnBylw#d0%8SU9g(Dihl2dLA%Wxdi>lZk{>_@_^BTVA)Jjvk=K|Vuf@BUS52JU zu46I|#k#-WiJXrdzr4B57MVWVb<4^*=Wv}rWtf_C;^lAQVq>a_rAA;Ldp@L6_V zUN3OkEjDmpplTonKzbvw_eY)KEPKOZ*Vf=6gT8o3T1x;_w8wbC%g_9}L&kkI9w~u( zT+~wDSGRmLc)2T@dfxdMwwvy$Z;Dw+@5XJZ&o%g9{gTZcKCHi9t(4hqjJDYfmfW8m zG;e7MYH2KQtLYzzv82+XvF8UN+_r|;XSe}7in*s~cn7eW_O4jH_8M!S z6ohyW@7}+(hvNJ$K%!ET&UUy-k5(zKL{{ho6A2;xO};N}d!6WSKWG7REDBc#%=rzf zoOK;+Pvx;+w$gq)bs*k+V#opK(u%Ub4+&Fvo*_q-Ixch5;xE6(ymGJJFDP#Mn;&@G z!lN+Q5d7YXL3T2WcU#f?%iKqqHV`bQ7OP13sm#*or2J(rlrq>XYK_FGypw<4?duZg zGp+MpoIHo&@p{qY2oc2>z_=^D*cq!VthDNV=#ENRSQL9LUXaKp;bE}IT0ZGMJ=*)h-RyQ>SRBj&m@R=YSnri664e=gF*Urub{!1ncI}XHJBV-h zuFHJ5?-NShRP84`R}2LXj4ubyUXZ+<6inqq;Cc-+*Gvds*RR_{sRa*2(G0@P*AKSs zpb_QXTVAyTXt#?>jkv(GgtuQZ@XRvm0wO!)y~)2HvagO;DljF zng`!^1${LTxqfVLj(j_`n*o6yliO7R*?p&HwEM{$=2u5G_>OXwGFy~ettN@I6&pFw z*P8;orkQ^RK~v#QF)hgVY~jX!M(oDAGE?05$Ko7%aqI>j=yi;p#O|b!^v3$PijiY^ zliMEIi@3zV8h^ri%Qpd0GiBwAJ4?)>Rj+;&YZu@MfE{!QENp4%Xt|pX){1;-{B#htrcd2!uL1TqNX2_LYkgcTS_QL*c!w(*lMTSyb5`K8su*U zKF}k6O>R{EHa|U9v*WCdPKFMAW7hxIBJ>%35ymUWcIwpx@`LU6s21LkkO!3}bGchJI2Hyv(M z7w-Dd%JsvtH|d(J3z)uCk+W&?mz>T0%{A6gN4ONi-TJFIe!C|Kpv4m|&nD0_UK0$p zP1VsjwB~MA+DF6S>SMe4b%S}W{?km|;XOU;89^-3`G#!>bip^=e0|=0Z2kDW-BzdW zFlagoUIxvdMK7im$fYSUJ$o}%+*pSmc^t^Dd|_dmRFX%yF1BTV zulQx0B7~XtUexGuuMoF)u4>SyWouim{8o+VVCbN|Hc8E^PCZ4Pvc$Q*h{j0?L4ru} zv1u}p4&%g~I~uv8g9i$f2BCmnP8)%uz1u1G=Ct=+`qExRJ4$;d3s3r5VhuJoE9Ck*Q`k={4J?HCWycz6ts=yH`_pbYu<#U_Djxe# z=3nWAGiG$pxe5C5A-s@0NZJO@+ctf9X|Ii^spcT;yWu@qO3A7Dn#E=q{+IX_wQmKj za$x??H3s!j8rLJa2(@kkT4r2!Hh$XeNz2Wr&2~RO(F%4jU)&L_As*0vGmk&Ncu#(F zMA3NQwZk{UWQKI;4;_t>xxW@RJQ*4a!!7T>l#0%?TDs3s1g(v5_>lU7y^#5N6!qtt z01D<%N%-gn4I9taqsA@!By`D>MWW3lWIFnwZA+b6;=xA{f}NK8NF$9qP=?tSvMsb^mvH+x%b&S6^}oaFMEjje9D zn*?0SY*DN98B!1`IN@wbziePQfYn6ynFQYhr@r#5Atq;zp=dJaJ+Z%+wt#DrvK{^= z2Nd6f#PG?0ATx;78rX7MW*{XP;SOqe6lJJ(9G4fi54>$w@ne(kVJE4^ zGU^{q((*bj zYB*I7XvU0vxOGeZmR-C@pvmiinb>dcg17jR?#m}t8|sPQnEp1#T(Wv<%3^U7x!FcG z#;=e(--R*Y{%S^ACBE;Y(3TD*eE4k8e`I`P{4Q5{6r3b0y}coMr55fsol_VdhrRTX z@x!j%r>D^amWE*Am(9}bF3P8O^07ukUujc*e;YB<6`F0Bz@arlJ3bqrEolPTkS6)A z!`LGh%)p>0$`iy)NHO=K8YUR|+96d_A~taQx+v^CQmW6{{JC4DcwNs8l|m-lM!(iBP|K&BK`Ao@f)kfUHlJPDPt$g}2nWc% zZL(eV*wfUk5-|OMP_z_x(n%Jj`6T1ym_z~zNAIxD>~}JYbAY5K)*Qsz!M0qG)_nz1 zNVgRvx3jb-p8YT^v}nX=$v6y3Ow%+QCIX?a<0lQBe z;OM`iY{ovi6%=7vqf{Db>YtBUknjjS;^44_$QLuI#K&Y&?eS0I9^(88fXaO*y24*7 zRTY7JTP0(PgdTAKS+tnECb9<}Td^N#KWKWfQc$jKX@&f`AIrtUj~-8+ zgl2wzC*EJAwMLn*Kd?HAK{7M6^vQW$oNBFgqA!Y4ZAy#*BUpn0v61f|tyV zs-s**)XRREXA1p54C$*QrvDo7P^E2C`)iY?>Q18c^hOMhged9YG@3}pL*k{QlGNWX zxH|?u?sx=@qLlrgLFS4b`3Kt>uqB&I!ep%UmVS}GUww8n?!%etwEYn+1+q>T85#9Xu(&S>k+Fslhr%Z% z%ZjtJWrC3iT9}&-Be#S^gMxRcRj*oGHS9$g+c%JZsg^Tktdq`iAxkK8txQq;IYa$j zeX=ZEM^JH^t=g*1_OsC?LC8;mZ;1`Y#BrXLq02m>TuHMgL%T5Zk)Oy!tKvN6=wf%R zL2UMJlJgb$@o&#dLmCm?sX+8jk+p(u1{US$+{XC-8U?a@>UxN$_xDV782Hlfs?&l3 zyGMzsrYoCFJtTwtrxf24B?aZ9L{kU_Gr}k0tbg&4ioYT)npgMFq6iXtr)H7=dPZ@f z=cjh1q09TQ=jgd-DM)c$=h!XaPGuOKi4R+z$Ncyc6{VXGlA^C)Mym~WE^jzGLTF>=8 zai$1Y0d5gJMpi|wQ?yar@#ct&XLKjIK9yjC?Hf!wfH2&+@%==JPc(Stp!ADmjfs;o zdz}MpA}$&MVg<4d6eJ?bKc=~)T6n~ZSS~B6%a{Gt2~AX0OW&JJ{dKB^K_p7nYR%Lb zu+Pv{z}3#&IbB8-qJPO=jyOd`R^xKnjy_fd!yzo_A8d8c8H8!2OZGsOKEWS{wG8!?Bb| zvU-;B0&`$AEcD^;uypXF*idZ=-JvwMB`&e)#NIkq|CBfu+zRAPpd-5uE25|F#aXM1 zYpuG+MKPDZd%j_m%Ji$d~a#;@g7TN@O5~Zme;GX+KFFWLGx%39$O# z+o+QLr@TK=d+sA3YWI|jRtGB*Ji}w6CGW?P4SycCzW7j`_znlI?4t8+MK+C?XO#Zy zDV19!LM6vk_#bpi2AB}2WRj26|NQ+Y29U;!Sw=;?%BBba>@&^w_5nB>M>p$#W{mfD z`tgCKO;F;<`%l4&Z*O57f9Y7gxc_Z&yxf~Gczl=vAIG0)X%u(@wE^ud{!fkfQkJ9xyDn4e>~FLj*AKd&@!1o1^t(Q3ITA&&fVTMPMxIxeUEDN zEufa6U=sbOWX(k2Z$E7uBhZvq<~NIF4SU)fotKv)z~a2(_mE29z6V3Y?c8Th>!)gf zSf|wX2+j2TH7tppwrni7!vLmZaQdd&n4fQfj+(k;zsp14wH36{ZCdb1eJqPCzhwHC z%v>;4irr^{*LtEqpZr?k@~^&P0Ghwc{n@4Krb|hG^1Z*ZpK4T8ajVyJeihY#KBR6rH5_+gtYKt?9#XY6aaLmzt*7g32a9R}b^}D^ zeWK!nfiE1D+Id^6!Q=RWhWk|~wQjP5!k=*#8h^LlY7dGpkB-;#43&aUy_aaRV%+~# zxs6{rA)v^fRu9qQbzlwnDBRU2ank1ooIq-W+Kqegx88)bwR*hi{cvvj=De}fM7-}Z z>}=)^dQuL;=M<0M&YvhRCo5uzxKV}NyHQX^z(QVERd4$nG0&K(4bzc`#aM-Ag1TwDt3 zx}OH_Ku-zA2a8#`x*J?CVY>yv;NRTBM75B457rE%?C`D}GDJk_)@YJ%`-d<-XpC0D zHX!Qr(JB5)GWcyY%!+V>*-!m0S6Xd2fmE`2bFJwFj9stlSI&(31vX*+fI|t2{a~Cg zMfIZ~lLjpa8+YQKcxB2awRk#gchyPL?bk$_;!<`77g+2{m=^f5>^jsa+mfN8l1EMc zEpC{vUgq5oEjgA_M1&kLo>Yw(mr0$eij~KeLzj2fi>bAV9Xv7cBsPSr-OO5>j8Dfl zynI-yXY0Z+6mF4qZ*$&MrVxv_^QaR#N0=v(7|}Z6Tq;`K>4Acj>CgzmO+0p zi$5?%jiuaX((;s1lC#Su_?hLKTwB#t^?U__&73@cxZL|)hEL0Mu$~pU6OxRu@B6Du z^Q6$5^&s`zc@(1{u> z*RQVvKzf~@JnxC7a*Y>PvIo(VO4f!F=6pNl3nE?IURJVC$}u)_e`H#&*nfg0lFraB zY@u3&)5nv)!8;;E2?uI2kKGAegdstg~G2c&TRbF@Ve?A>#Y9Db;uvsk3q}`v~i?{<~})Gy`u()wX2) z5bdSX%SgnWP zc{&C6$EX1x^U-c75yU*UGTN*Fdl^MA*?k`(uDa4z4q6jhn($h30bRB%bYFE%Hf!*E z-ajqG5djZ(mfZA)Uv6?+pAXbuU7mHKryEP7ta&e{9AZ{6<%f&IxUYcx_57DJraPCb zo+3x5N#||F`)yskHkk1>D7kMn69|cWaAI73xSL670-Dn7eCU z-IBRZzkiIdTc3e|=?tK|j-W;fH_3%;w{;f;BdW^U2PYGH{jr%wq$PV9mEMWB7JYN_ zzl<5cry;y((0kBt;winx@7Cyjk4ERbgoS(NUACduK@axwhp9>D)I2@U2E6Hv(-yvZ zqA3O`HH`HSsfMpkc=tTF&_w$sSu?ZkO%R_L>#4Ujf4$CD@^nqu@p8s(nLZXvErZ6C z&(#vEy-G8E$N<~T@D@3;=#2(b8g=}^36pT3?Bb8|LnmboS@(d+;fQUG>=sWuxu=Nh z06$W>Om4M_nPu@v+!Cy%z0M@qh?#yaz4m>7at_fCug-ZwkhE2*-c?xT%sdSw9n9fu<2| zg&V!cA`yjz(qZRfz2Xi3&HO%sD0@vAO(4=Fy+>x@CzmGptA2C<{cT#Z(!)9Kuk4H^ zG9kse@-BWBP~e-7bm;<)%3)8xCmNaIOPl=^aPfz|+m(7|8~)t@&w^Qf?1w!8t-RmB z+X=1`wz7Lu5>E>cc0%#ic-l+Moq5t>eIebtEBBjc8OCX8^QHKIjUa^Hu8;QF{DFv) z2mM?BKtwGHIa3Z>G?p*!fkH;PWNX^C+hwf7uhFD0FJ{4`%|>k!cK3g+JdR>5=9RxT*?G(;1B>p0;5~om|2vqBYRhIYER~*zvv&c}Jd0YFNyt$4|#& zHNkD|B0S+)TCbTVz=ff_l3Chx%>~?w%t1wq=!eTF>kAsH%Tz!Z4D}}`|7t(A2HRD& zh8{=t*HykAj`uy~TA1@96K`De(vSYXDNlVKxRZ{^|4D70^}18lp?ROqd*Hb0H)YD~ zd52OfBg^-#?RQz}`F(KrNN=e9_?SVs1G`IY!3M}&CA~q{Ym0?sXwzZp{*QSFdf_s$ z;AKWmLop|xHT)*$scpn%(^mvrm+O+x-px7)<-JFT@SVu$+-8LBW{P=ppnXmqYwg5f z(NrFOj5mX7_cG^|-3#&$m2Xbz#W3-aUdUq7u{6)GQ*(#-1FX^ys<(M-#p{ED zTO2m0p9Py!OxL6i`|jfzKvx{a8nmXD=$k!9)qUKy@9z*}yV5pF`Spv)QGQ_d~lh8D1MwKK8w)B3}LO6qlQc?+q1sDWe59!AuE@Tn5~cHudtvNyJsDDe zqV~jAe%MLjBv*n(lPK|;SvA(i<*JFF*OJ1~d zyF6}hCa2P=EC?h^$oBc&+{6)hFwM7}Fx7aVrX8xlPekkR9SIHsHnjOl8B{;D&42rH zy#~wOj_l-;!cJMVG{pNTFOZKGv^`mJ_<*SzRvMpGD}6rLSC@Wy^(1xBgibx2;rIP6 zz`eN)v}*1ok;;jGmOZ?)p{wn8r3hnOXC4m%zJi$7ik)Qu^R0mqX!WGfb)_=;i^00(IwAGG4C)mZ;LvzIfl)OoTWcn>%~-Eom(i zM%+XrlBJc-*y4rXKy}5t(A3!LDmK{(90~>Bs63r4t}~npzdUE9UhdW5ct+>xHnb}w zb|^!=Jg>uHv|WI#IYKmdr!P$Rxhy|D*4t~`Rw!%8@2yd~dhWVnD&jtNKGDQ_c!i>0 z$yfkf!uNiq=F~U5!bnbFA(}ksc|yfeel(Lmv}O&_Ye%t8IxeQaqtJyUDr7G$w(DHO zg#0njRV89D89NxxMt^E=kn*?fgt$~{k0ee}douXF4*H8<+ZX-kv-!j3touh@mx4Be z`>h1v>7MAI`?-8Aq3-+pb%%be=OqUphW8Eg7uWSWBUuYL`3C7&@9lL(=BpT1svc@3 zdIN!yA04f|M+kh>s&|6aJn*GUlqMq4?~GuM)OHoW3{Lam@s=&&l+$P`;j%hWyNYUB z31^9CrK~W|`@6Pyv}}zaTCLx?uil+#GL9IPD7U?)!MoFVG(Z{c6a?N4j^ta(>Mo`3 zV}{v&c`K~QYLOZ4{gGFankBL!cclOJ zz*pf8mhe=eU|;Q^m2gy$Oe+I0S~x_WL_oktj^@E6JJWc~d~6t8>sZ4=CqJ+lCYi7; z&}Jj_D~zTBrA^GX@wL?}I<}b)EOz@w6NgzLIE6awJHH>)(C_c?mQjEJ z$4+m^{Q6uCoao|T4%);nq{D}-+0~tAx$k)`N@QG?mmjU1J9T+@&nZkD^yEJsz=JH# zp9+6>)os$uUQUu2KYjOu|7OSZ`UFXOlx}Oc1gIpYQ2Xmy*qkfk&1VGS*V)$47G8Z# zO901|Fi#|a?5tcj_;ln^D%6TrhBvsTcU;7(l-x|`ig+J4c)4q`yFYSZ1x{R)Lo^2q z$3GOo#4jA|X6~)jU2Q8v4BFG<4SVH1KnrV|!M>fUnU_q_y1t;6{QbtxsD}Ac%p6%v&nTWI;|rFN^)pGAp{D1OA*o203!s985k z11OS?wdgf%N2JEFUm`jYFu?=@fnr$c^pN!a1yP1X2HK+b`}jtj&~t68C^r$zS>GF( z{<*i;w$alzgwH$LKVCUWhE$B%j_z*0%#toCei_MNIY>heU5J_zydp_DKuoefZ|38A zx|G6G7$^uT8}_p6%+TrlLCWSq931~4AjK+0iXfTip?lewD5TwKmNw-C0d}eaeO0f1 zs)tWfieF%a$?{`V=ixa(2uV#3d*{i=l8kG!AN4s2SLvzD7e0Pg7pOF)SqM!xGN=?V zL>&-NE3B$;x`Ar^Ts^pei4Y0SgDO<&w0BS&ep0J>ibjBseSE692)gFb2t{(cF9cZo zRqyzB>>$-KPM+f#a5BEsUNb=l;j|Xo0k5+(OQzU5qv28_h&Kp;Nl%6mS|0NjZwt>C zj(esI)>0;e&$owdO*&Z1owCJ+kzj-q<~cx?f!6B0w{B$|;jflu^Qzn9l8z8QUlcs& zO}g)5rhmC-$lvVH?IioAr)GL&+BzNUq9^>&_Bn;wB#?wxUf8G23`t^#tj*r;@!h|0 zR=#SoAi*&CdZw3CpJ-ag+;-C8!-&~P9!~$eYdSk#8?H9>yU!7$rQp1E#9pV1uZ)vS zypjmJ2!wiFCX<}4K3pouZb%70o=-OOT-1Jz=4zCf-wk#{9VX2j8l`tPvouy$i);{P zz{~f$y%xi=XP;UMIy4|K=9}>>!R!Dq(w)+3$pK>jt{(Z~R5?)Ky|e+^2~?UPL!r%N z8d5lWr8@!+D05>nl$0rr_w-1R|IR>i%EnM!SReI+=Yw|OAYNXNYBtba<(cA zFT*eyCcO!GQ7o5W$KF_3(`(gB7yED!mbVL$ZTC4WlADiOLp(^Xmth^Se-2>GRATzX zu?KUyJ-$(uE?*@TdPJAz>bW^h6(e#)?MvE&Hy_1D6osh z=W?U}vQsym`YE=YXiX63E;NFiJlJi^lM~;jIgQb7rl&d@kRIum9#_-^T2E0fZZgjM zEAdY^yQieVNjbL+xLcAi)U$&K99krp-TuqsHs{gq73mQrzJy3iG-gi2-D-uz$8DH| zp_(UJ5;1=^xZpHLTK*Z?J+wB{ZD7>>lC+V3qndnT=V4j+^5+bk{VZ0icE zu%G|#O@r?@!CZ%R2I|s*g?G>iIp<_^wVZ&cvsDnEIzy(q?G)<>FtwNG&l>U|!OgS3 zH@o2+$k^fZ;fuPa%NJq+9{ms1KQ*}v9@YbPzW$X-h90nU#_zW{f*tHrhFu~=}E zlNzOAn?;oK*3b8!Ac}C^5&)$3CyyafwzGu#e3|f{{%8*%*{o8$^=Gt~bz#oD%Btbx zFFf@Kx}SZL2W3O*C3(>YR=rJWBenLD6IS1Uu}A;N9%VAN>1bDKP+POi zPKm4aB52TAyCkuSr&VOKFmxL$qQ@;}?RqOc*oG7W_weDmo(cRXcuL!2+czQh36!`YW)is1(#>n2op6 zuTkPspMT7}cHyU=_<86JB88PISk@fY%x5kd*)T|}>R%8)s{x8t&hNfi|lewfNZRtOdudrAoOX>6H{ zZ+AQ=#qy&2_W1ykW5s&P?sNOF=xOY;IV5gJzi9Hz!P&17Ex;@%+uK*S)jPfw9&_;{U3(%5p8EyDf6vi~Sq z)0S_UfKD8qQcu&*pA-0+dt?TP}elLaA=wZW?rywoM*7+#2!%W@vTQaZ!Q8B`QF%C=DsgJ<7k2 zno-KP+<_m!rLlk(9>SDd8gH&K*f&WgeZr}*;nmFycg5E6+bh3&52oQDuW9KZj|w%*bbS_+?wpx)Pi zEr|KzJRCCUXmc7~YT`w*dDz}Bt-0CE@Eo$TayQ`@pd|MEwSUs?~GT&KU`xz%V#Hr*g8(_H7>OY|C0B}_~; zru~UlX&)CezQ{7B{t4{=6#*MBlbpT#eD_74^;1?_YIU-6>YGFA8wckP9h{oMPbrtz zk_@QG_jsptogU*FfdBAAeq%|k74}VM8dwcD#`{h%Fpk;{9#v-+S5Ky$ON4 zY)@7?taNS3V9hxVi$Wg5MdNX4K%Dg(KdpAmJKTm%bowg)r(ge+J^8R*-HZJDdE|y` z3ubPgp|kX0Hq4e;-(xekQtYaY`m(JOrK10@&aUE~-)N&M%@HgbOP1>T`a=*Stn}bh zxTtrqu~>XJYd6C!-*qb=e}AX?(T6bj>0OnE)sHs-GJ;kjggnHh%3%ttoD@SQg*5~8<$vQOTbb|%TZcTMgYZ=u^YCu7L-7S87 zIEEFxQ}pLu`J8^Ec&pr^6z&%EfFyU}oW=7GefLMmZg&R=3#HJ$zM7pT_O))z?Qwxf zz%e1F`NsHfPf$F(WfLEU9FhE6C&1t&ZVe6=#sPoC;oFnAVGseaCdiy;O;JIEI2e4+=9Ei2X~jI(FAw*#$AKEWIOYj zneXn|{Rj5fKKDFw`>CpX-}S1x`?@c&V^CcR6VIgL0Mjq)|e z9nhLlGSRFbQN_!-9h7)mnnj(f58Z6+At(RYk^L`Uti%FCGv~}8*rY6UGiyu?$pp5R z<=@|L$Hq=!5w8#cTa1IrhCjnp$x4S0b=+?GDAl)&W#cGOP}U_EWYDd0l8O!3BB5(; zl<`Z_n0^hDcG)gsj!?uhqzYHSA8@}i#RhF}i}a6II*j&(k&2l0+NE#MxF{4zqCjWi zqw}-J$4rI_Qefwoa#E#s4zPPeSw_fTifq2eL%f96cb4u(c^ z1zMZhCGo643>BMr&rC_4bZ#0>RR0ecdmfO;2s}2E0@&mc76r1nK|TuQR?ggsxLif$ zO!J#hfhi{dqFS?7M0O?8K;OzOSEuKZ@xTHh<1eEnh5Ou1mi$c9z2&wneRk`U1rt4s z2D)&LUV?jDP->+?IwzS^U(q`0UN3W$?z{rEY3eSu>e6C3_F!CIL= zWU)cYA}^d?w(*C6y#*55Qzq8L#L!I>JT@yFF9r$pv|4TT*zd zdPp6Z8wBJAZk|u+MMlAJR4y5<$~cYZn$h`*cS4|5*o8)fgaCkg-zAJe)7n{P%CuHn zp_P&>0S$7U#Cj_f`$R9K;?yJ*=>;MgZVa+b`@uwOElSjwQF_SMSYb#oC#I_O^X9Wy z$tq4cJpWapX{t)g9Tr9H4?x4q@4zs$O9a2LoO&3Uw6cLWYT95sMSU0VU94BJChE^I zru?FnK9Zs#PW_WrGA1xl-~*0)$Ni#l{I*VZ#xA~)TM^2ih*|CgaoHH>L@nFWP1u!~ zPY;T15$Y8y?^uO#MFCHYc*%4TqMM^hmK)_!HAfsnkywJ=KTi$`Epzm9Hp<}Pm%1uU zRZLmp!=_75*s@l<0C0?Sfpaw>duXsYRS1|!%>Gu>;Yh76aH~Y)k_W}i#n^FJLK)@3 zxGiB;TqFJ6DRrh>uyke}(VKAY8F3qM@n%~RcxH;CZr(5zT{0YFYX$zt9fO|`P%c$U zs0ADCnlc#t-+f_bLh7ESBo;)~W4)+sfh$kRxBi^ZS_GMN2%Y$5DbiTG5?0Buml=4L zNjvAACMF*nA8o6g5Zz<~tiC%i z`C~<+jNUs@)#(Z@RRM(f9ON}5G}_|Wx2^I*Aju%R2-mjR4E$e76IBHUk@~mJkM3Ed zJqfYzB|T98HJOc3QU|rZ(=Uq`XoqS!`i|r0FH6*DhQ3^vN-_GDjhLB&laGeN&`dNY zQePP+CDAgJ)3;N-3O>Ntn*WoGX+Qu+mYeD522I^qf+&zL%ryMTCsp4jQ`T9Hqk+1w z3ii1jEhiaakeR+SIh{o4o<>qj3xUXyooRt>L7qrm*cfegGM_mq`V>q@bEMm-er0bp z*}!kEE7?V;&@^T6-ozEo6cePEDh0CU4$5btfyR^qNf9OqsQlN+XW30`IE}!RlmAa@ z_Zp}71!GI=LRIpkIll2M#~)a?o<7&b`#F$}flReTQ8`T5kV=WssUc?W6fG2II4Dlu z4i1#80i*M5h?MEB8?}UX>tcmqez-+GINWPT)KD89GB>EwaG0O72y+A1rM-m z*HYMD@{QevUtcln6Wx2UmL`s$)olIOtP^%4(if&M1)@i# z2f^w>Z<|K*%#wKPbUxC-D@H2`kzVP^v;0tyP-VPs5*XQ&>-!9?MjDi$6H^$=KaS{2 zW$rA;I6|*;H#cef{cL_1)`AV=`iD?XVAv%1K1L{=J9WL^tj!chr6p#`IVgn9#XV>e z8FIangXg!qOZ#8^YY6+1P6u>OO#y5n)aYR6WEt&}d5Z8kdQ8~W6rONdBClY_PmxXr zyHVxRUXovWVw+S@+A88yqMHnfW9QRlQYU$~*ahZ@oRcC->tMw3D9-j*-qQ4~hP7zI zG-?H8N4B@WBFBtT0|cn%yDv=m!{_d_sBwy_{-I)v{!p=x(>eN7b#x4=>1Yt9C_pV5 z_2(Bg?nb-ql> z$5zfz&O!>6VrWiyO%?V^pKOO}cj9l`q`1}_MDN+f{?bLsA6D;hzix%=@NBr`6pxtu zs*(W9z+w50U-HW}Zm1`oDzct{c&_y}wXuG|3388lAB@CrybG^)awYyGu9F#DY)c1o zN3-P*?5k^$LReD>)T^X@THLGfL6G`_< zl-h|&MhjCNRw%SNaj2$M#@s8$RmGI?UPy69s#=mrGwUcRkKa`^gqgH9g|HhvYlg`LzD>J!Tr<`%^bWls=`hXm;x~%0$h;5!i`C zG;_12Bo!Q&EXF<=&gC)=_Y|opzL|cpk2XOauO<>M*WXVg`||L0H#Q#0pS4Au`Q6RC zNaq>7VwG}dt)^7TM|(+|DDbySMsI?zl4g_vu0wV_!&vM(;{=N8G$aS?>imUCb zRC^b89|bn(?L?J2Y@irGXt?tGyxK>Ui&N+91ZN2IU>ge7=IKc|p=9(YY{+}M?XBj^ zVGG_qnV+@(OR3DYloD~W%-sVJvbuEBrng6OQVXMx4@(2`EAvEp=BO+e<$`mn6Lwv# zHL_(q;m(N)obA17{pQz*eO2q(=j}=_4#%Nx@AbrN5&VAT*Yb}#%gg5YPlqv_J#TWq zv*#@FDC_{FtIuNx&7`n*YY-iY#w=vmoefFiP?b(GTkDib&n&P8qdNzG3YyXq#C4ub zWu)c0L=HQEYvZdIO4j+RA}JTB>^4t`?Xd}6cAm#5H$8-(uPBiWKqn`!<6{R|_;Z6xoS5m?!t|7R%zDej;g@jPV%68Wu3 zEm9+OxZU2kNn2vHpwf+o%6*)8XUGHz)*W^hXZ&rt2dkP2gELlCy*S3ftQzj;JJoKC zF7pB%q4q^-n5^C(v`qnvC_h{)ydlAaS5V9vk(H0Q;Z*3B4T3b4iIa|_hAg;# z+Ki;7Hx~VB@-x4M$V-d5f+|*u1=&e+Rg@TRWle52j`*N-skUmB398EgiZb)I~P^~E-* z$&RD_b;$<`MC|rmE}G(I%b>Fwp=oYijF8OZrrQb%zzN||W73UQon@BQNEXP}bNLFz zrIjpoU#y4n%WVe+W-^hmqfqS)-OIecu99hbUzRGe7U|XJVrlMfLyKWK#Mm=?XEeew~4zLVKiPsfk`kf3NR8+?X(4dgsxf)*qd)RsIf8jh^hQ$+WcaE}(>-pA$JU z1aM@$Of0)rL9flC(L>}TriR*c6o=Mv`h6s&F|i`cZ7bW76B8VjJw z^%uh1f7tYdkB3p#L)+VFbU=YB-y&-oa$M|6#bn@0pPYB&moH=vlDU&K`;@v&4?7O2 zRsT7M67oM~-Jvjdxvz)0XgMoZ1FbH{_uC?`+h)P1M{kLT0tuW8P$DLyd_4d!AhUHlUGA%>>1?gNtD+@#dDu;Du!C?1{SbNx zh01&D@9N+Gt%WdRKzHNg|2IwIBL=To@9Cj222>zH6&M0QpYX@hA9=+pW!xc&driW) zZ!?{1j-n=_Hd{)X*hr6Pt1^@r=l-Q0E>aqObCIZ9qfls8b@^R2O*4_en&}C8CR3Iu z;Ps-jtv&Z|@g?C$-5)J>@qVb{U%`|dD#w)coF$udT?3jX+`p8$M}L#RB7n7asf&H; z_1&YH3K`*Hx;(|^zxj|?k;j=C<gjUo()`z=B7v7jajAkwYS+!{T(Ud zUm0j&+2}ZHIPqHelA6)p(9@e|68ujZ|9O5nXmwDVKJc$^|FQrWa+g`^89zR|tu{@H z#J4eBHjub)_^Y>0)H_3G*}?nIc)|hQt!C*x{Y8KRHGQ59H^4|+Ka#9@cp8bbKpF$9 zRMj6Fq{4ySmV*%%u@Bv&-arAiW@ut4f1@*;#I;mNv7KBl0+(mE z8A4}P6>k0YTtE*yBfE`hS(cz82aS^HS)mUK#myvb_9NwQwJeS^ju|)=#H@x@oc^F5 zShBQQTsz>k0JUZ9?mYdYqDd=3XO+T&x`tO0;bg5NQ<}slY5nyUAAi*;f|OtbkJ)I)ZET2_Axlwk-k)zf(>^Maokr4&#g#Y~xP+B5gF}Wb?hrX~Y$B ziHV^7=1>ugy-Yr)*_FJgsMmE_@~BoxCPsvJbw7$~@`K)i&kG|+3L;s4X@6FU3&KPB zES)}CB4(rqyW`K@0$R-M4F8No2h-G5(v+{}xfQBhOCx{lXRcmoZFN# z*+GT)i5XLna}-3Ub#2`5wovp1#i#E|Go8XVSQR(`er|%s1 z$WvONy@}q|tnyc$A>p8#c!aw??RM<^U6BtpXF-IlcxYP;`E;Pd8fFEE+23B}^xdT5 z1tSs*U0&G~!rR%|Pfex~fM}_i4YHCi`-8`%SEYP%*a1n)RLv$cb663aMYPAI*>OJ( z&bUHoq2yf4b za%Cr&)yx~=T(#mFWk_1ERdL-GL{SNMm`=?6^_LtsD}L4*Jm68XJ^P~_Kdh$s9OjZN zD5FeHBU7aU%UFsRftJk=h>0avCQvXkdBOXyvN zS6Rv=K+u2EiBO`qFrmfmxV9KGN&o|SM}-;l;McMUb>2!PvJs^SRyqj~f)yb}{S*@U z=-ofj91~MzdJ8an{cTQmwB-9`@fmd#mUg%zAZp6fiKHgCCSOHK_eQF+Y*~*q3vRL? z-cFG!g;hbH9R=CqZ3ySZ6Z%{}@3@BTf8cStJi~Ys{!QW&xx)+mitmmViV;ZCVL)O~ zgRrY0r&K^`5h2f$;f;J)?L-KUlL_i#xWsB&s4CEuZ>Kq@ML?|Av=0qh8f*=l@W!?b zO{s!Joi$+n16xtfV_dRBcP!V50&aeoCo4_hEadHC0o%1kj@Sa)4mTcOUq5QerJ6X3 ziyiJ_OGPBtuo$u6>Tv7_CG#isCdM<_sg_fu;f3|+`o_dl<($m|l2OyUqq5ngB82^z zYWx5ct~3F4TP4RgMq4|TIDkdVkRGz4%Zkak%*8|paKvvlG`*N#Mmk{0L|oN$??C;TElOmwXgJnc4DQvbRYl)fm5hgDKiX_Lz<_ZdYpRUWQmoSHI#iuhw#c zeQe6^_30ea_x2OLuP0Y+LFRO&txjIa{kS%|%s>Ihl;rm#kKvXBUb%IbFZ8eXy;!%x z5V?U?D$e<$vo42gPLXf|zH15WGjoxOQ{uB7#n#sf_e*;G(ukfQ{M#eAqN4(~Y3ZvEiO!kuMxdu3W#3y) z(xU0ML|KUa5S10M3NV68E8R6{{J-P2iSF!so@YMy`ZSXd+fj2aZBbj_VoU^4BH1`0 z$$8Yy53s`(AybMG8~9SZy2p8LPfid4cXrmPVE#6{jYssBm%%CkD*d-DBsAG?bBBHV z&L*<7I(ELZm01KRNOFu3(mS}(AuAC9_{E%Hmmor&d}5yx?XLR4Dip=v`PjkCTXaYt zYEZiv)Z5D^)3B*PM6`ZxAs{d6-Ib>2O)Ac+gWW*_!}_kG%x1s=@a*UF`6>kkh=uaO za(8g>)MjFIx+isJ$CY0?ht_OA(a?CUkykti4jtI%>wq{mG3y|hxst;_k%6n8#WL31 z6(yZ}NUamd`HH}N76-rb3tq-aUrsJIgN5NYXRVCpSmV2++n&f$t0B|b`CpvO?FNM^*( z`9t;>!kc`p90Q0^I~}qL&3mB+E zv}egAeJ+3NMp6m=SytvYTvL@DJZ!vLvLr~ha&EBUPB2;|i#B@o3VYxJai(Y$v~|uZ zyVI_#!p{7sbI&Bmew#xFFNxREjli)3)SY)3{>I!Tg8tP%B&0tjK*EK^riK6ZwxsCL znG=KsiZ~Hm89CbAQ@kOhPOcq^nINPt{)oV98$@K>O>W4I-FK#5cE`tdeD@K)-#h*7 ziL)OIeoB@iwC=5bjwjCfxT-`rYaGl)&wYeROio6#r-2o_it zG9?FbSVH?hlvGS}-+Zn0S>RhH#vcTaA^7`}4MAM4EcJZe@7fWa#9o^i@|bGyfgWZg z5+Io%hd~-ySZYL=gcYTv_&C?Uz0{xZve72JaZ3TR?0sfld4(TJLXYebCq0-FkCl;B zH6RYCqs*8p4wZR0zWKnEPWvK+For` z?XJgmUc5c$v|ffR5#I;&l+>`>O?@S<{FQG8VDci=2J(WlQ_51>k6(cN!*aU5UE!w{ z@1d+&oc{N!?Cht`t;sS#E5nF~!;L|6by`5_o#0Bc&$wOP1P(vFWsB?XADi~q$Av}) z`_;O{6!||M`p>%?wWGJ744t}*a}M8b0T`L#`+No)PNc%~+7+QbD-Iqx@Eav)TJoRK zpP;`Th(o^>$E#r8wa34>{l0lRC&bP8n;=7n4xbZ&Xof@n9{VF#06NdFV6Sf{UC{Xb z=Tb7T@BXYJ2s_B|#x)ca@N0S7VFP}B|FbUmbKPq)(xIwrxPQLf7Bi@uy-gwFw`%$CzWqediwqE{Eq(vz zrxiw6N^pm+%*g(KAA!*p1@z{g6pt=vEnc|Z&*`0MxY2Wq*rX$oZN9lNIrVwro`bX zE6tv7yXYr|e;xUE3{~$^B@yNk9Y`zNT`U2(!)X^$NJyv=1X}+nZ&vZ&p?)f(LrT^&bc5`&hPoI(u9=2FDliQ${4SoLtpuo_(PZEL8_VX!KgBI3 zBw)zN%e$B`yt4SF?r?LYgp(4B=CB_nq`+@p(bUovsMB_-J4J?MC>V2Q_DkMqiUXco zF8G3f-LA&zGwtI-Jjp#T++Lq)KNu;g0SB!qbrpvkX1{19d$W)M@NAzihAEg`KIEck*5!bie@?~}2XVsynNE-^#jlJ#$Z+Nb3`}Q94cj+J+ard_enK;y3Qmkm?#nwiRTcbu*_7V~(I*qUh4WZv4-C>XMQE148 zPsX*24y^_UVqGyFgo(0L@%5WyNCR_QWY2lDLs%cjZdwFCe2&Hu#zlgK?@P2yt80BE zf6ti^PwMZ0j7@=)i3A!cXQ^3o~}biQ!M<-c>I}|@KO9-4w?5* z%0-c_pZEqkhY5wOm||Y~3G`^V#^|0$g@CvPWsMaJR%1tccP6&g_f=ZaI*k72SiU(I zqQ+D3-cr%;&O^6KKeFm)x2$qsry(+lQle2ZK3X+yUAEk?g}q-1Qp1suy&z zDOUw({36ch76AzL3QI~{Bw+bw;9uWaCq35ss3i*Z$~^I_j?M&s2bP^K&7rJwSiN}0sd%5nw& zJnAiowOHQo)?uSeETuz9T+P<~{Ww^KD&NwQ8+E6>eACFoY&_(HdZKOv-H>QPj`7|- z=>F@Q0kA=Gkh@$`(B-h$_is)Wuo_1`#I|Md5ebEBEZyE5sHIW+xv~dQ2&FI4Q0QcY$;dQ+V{2Z%3=dATUXv1fk-6 z%}4|p@#Ii}z(zlLSfOFj+VM|w1vE97LW9@#)SE&jUp1v%C1Ff5I6paC7@cX@pC><$ z8^c64u4w9>+83i)3!O3}j?yA7>cVpSB3o7?*Ud!HwC(GOyGBa)JE#Z|i8Jf`6ubfN zLrPKI7Q-M6gJoojVWG#NI5Zn5#tw8Xl>rlvGp75_Mi@I<11pj8u_I$^Z^Jk5B!G>#kwl#i4Be|g8{*7^@WNP4PM~vh!2S0*R$D~ zjvnc?4-7zYyMjtH_lWm;9lb@g67>U_-}!r2bBthrxAbULQ7lAllvB;C_><|1&5 zsB$5%B2^bRT6wW41*Zw!U2}C#3d&(+kC@n*MSprkbWi+NjjR~9Jl8eQeagE zthB;!{oHRhnZ$s_7Ri{2h0t>|PxFDsHPA)yV!ws6dFRD4fU#X=)S%HZ4)m`dcA+2@ zk4O5HG6bGOZWRosP8#rJiD16@0x$_GHyKBh&OZ3immX_{M?1ia)&#i`HQ6P4Dv^` zLQa1QB<)pV=L`)Lq7=qiJrTOt*vCQ=!|oteT7`giSixS~EjY}^B$c};D=I#52Z?wJ zb-+c@IX-ReZ#Dz3{`%qlz@a_kiwdO zy9$>Qqgj6n;!)7g65eT#`c^r+-fibNjMq}Wixyjc@C+wPplIoauV6rnXO_;4->3Wb z#lk^qlBK@McicS4K~g&$aD!DW!H`@!+PSkpJb}R*eEfIF8RzNl zo&_?Y92|bnfGANObemjy`rc(Ufap2)`+#o`>g1zq68X6+?3AxPk4=NlP;)>M%S7Hl z&_DM$rR?KPdraDlDzBk*gfl3VDqO+2!mX&r`f$xARD)Q}!N+wPa@n=wVs0b^Rdl?@ zaeG#S%dzVG2@VX!$NDN05KI8LS<-s6HSX&_3|uK6_bl*xpTEM_g<~~`S>l?mxrd>! z)amXXt?SGn$sMkf1itJD>_NrIwyyjBunGr*MC996!_FO){}wQYGApO%)5~`8*S8bZc{BaP&jynRG_d{BeZy<1iuv*9S%5oKBb# zrKlVxsH{{ARTSj^O?okdYdk&0)Ynbx{^&;(8n=4@LU!XV1+cLI9+@jyhgX6!h19HZ z%b}IscnE*UGE7AxC?>ORUchl_jDixh{LSyfAxF>7ig`R_2flWGN>KtnxS%)DsCi-A z2@%@wZh?R-dDTyW7aLBUH+Ch$z|sC=!$LI9dA!pD+LOxL9d0hz*PbVFSK4}tJTz{P z8^v@{(}ga)9seIP-3eTXPx8WGC{%0VL?4N3Gr`)sxEBw}ZxN8Cmz?Jp_oR@bg?Q>X zX;D4A>E`ACjqk*4ep-RT6&o>TR>BYCl)3bc;F^eZ_SxNY+xolS<)=Z<=G>LWGcY-7 z0<58pWp=+!R?|c-{wm7Ggf4Mm!z;PiT%5q3HVv1g#s(t?9|(z+Z}T74>&&%a08#wZ zQ&;7ppw4r(IgI;Vs`=9*a{G$9443S}<7I*de`_NAdupv{Z)^ZZLpXbq6D0O`*mJ9L zFpnv&M`zAgvesv44#hMQ>%GMfMS6>we3@}ORX;kY?dn?8*Wpe7mMUF|n2iW%LAq%P zL*Zd(H!215La(B|BPTJ0l|DkD28@i;7aILMq7GyVe)-bgHC_!8k_Fb_vOReG_s1xF zg}ofA8Wg>GpSKpPuPr=n>=&Pe2Z9$GwZP=#10A*xzGguba`G2U2fnC;hvuuC_0l^5 zJ5(zHaPj8lm>oe?M&ng1z@B3lqiJU_%wZP9#5Fo>g+Pmy#a#s?B3 zQsSQhe+h(c!)ZayU-(PL=q{0>#mEpbdFCy!dZOMQzzPQpZO=tYmZ9h7#GYK~$sRnE z0NTm11o67}3CQ=k2dIB;1nCixE|p**i4_I?_$412aXf)ydPzH*sZQDb28NA;xC_bo zHN7`|r9JN{ehBPh;gnt=?2XQ_)(L&C{Z>R18mA;UFqJIl2F;hRm{{N?=qq;r?oJcx z^v}|lMDr8<86eHdNF@Db*hzzI^0YjuMbxO9KL>!5dD$v|+#?JMf0t5hPe zRvHBM-$@8`etZ|&!sD~`>o4%wga>id@3>b|{25upu)?YI@eZ8d#rs$#r4u_W$IZul zAe;oU*c|)n7Q2tvgjclG$^g+avS+H?A9uTxsCpLl3+2YoHRp?abf9$W#b5_qn~1?} zk}u+baXaH(*QD3ERA+rlW^=!@>xZfD9+XZCebf^hU`8k4jMZClC>x##6*|U>YD%bm zwrOc2&^MZQH07ChXx1<83MfJA7k>>7{3|9%7_FcW^6ldwOL1tmA-QQa8KP4&0A3|5 zS2r4<-Brm%{4HDc>c{fY#e=FMQpuBbSY>21BKm^B&~(1ikFB&Sx-B+Uvc-ABpFCZ9 zPK+NX&y=<0T{m5kHOPp+EWi&{$=#h%r&n;3J1?^JVOXnB5GiHvG|#E9U7V$O%TZvVJK9ACS)_^kqLlm2z%`U3MbG z%t4LP;IEtICkDG;`#!5WE(VPOsj!tdRaSI@Fmi5S7;<>>eJLqm+2<0XALbGwiMBQZ*P`CxxP ztU#_x{eoi$c3ip^x0ngyXD)XxflM>##meZ8?BvG&uu!Y}tJ8>3yd%IQdOu{{f)3;v zy=Ckpr9qC0hV`<5oc2yL_ydOG5mm(#8$;nracmu;3EGio24pXsh|TU_79b9iWj(}q zAKNRHy^5hKD8ZDDk{mUHsv%FUe#rOvbE-zx_fktHgP$gSNfD+86JuwYEPXGMJ6?M` z*tq%@PBOx&M z>TOESw<)4?2~S7VShtCezm?iz-cf-&KGg0=|05MZvrTQJq=-o2)@kIaK!a*W{@>2t zCvv#05w~S_+tZ>;gQwBouG3d8u-(I8?BDJ@%oF_%XKlv(84@3Bw26`|cBd>T62IfR zv&|?%_n(jjfqA1?%Ge2SUgP$?c1FjG`@@baqiZ;Y2@GARuQ_miN`QNv_tw7he(;Hg z5N6sU0yR7Zo{U`eh9j9~We~-FQsN^d+^FYV2K%*FX6RJn8dDZ1EpT-Ce^YED;KSPa z43SMcO26^P)Kr;LflQm^cWB=Q1l`!XOitw+H7}vGvHnMe(KyAKMxnnjVWLYF1s_WsO>t(KM#Ab8d%%%u1=Y!OeQV=ZQ9q`!Gai}#{t=96ctulO zZ+>`>8C|EJ3f#O5Z+I@n%kgpgaHp9ZrQIM7nz`jdVKLYuGke3pFmM*XglN~v4w1H9 zs4NJEf1A1^s&EM)@7_2UZnRvj=pMt{%JOTr(m+Vb;wzn8{N?g z8*;IFvX zwkI^4e+oHhHKh0IG~2(j;O%ToIYnI+by3EB^79IL+pvGR*IK3khW;Vj9TPe?p18AP zdn!2bLl0Av1g7;fuT8E~WOai&W%3v=9 zlq>s~E2PW%B8v~9^Q(@|#JmwDVRgC>zSw@=-R|n}e#^Z8rRHljeT$FUFcgD!c@EpQ zCi?pN@^ur3#o^$Btj+to8RwD|;0LZ;V+#u|dttUf{Oa>x6}guNzn{-`R|739ZPVq> z({AJHq-MRL| zOh>7et8+cuEaSVKY=SBY5VmhL_R0TC@Iv!Oz2cv!c5VV8p)-{nn%OP%!tbd$O?*+= zGD%{|!t`lDx4h8!pvhsxxpaPSVEKl}_~c`v>K1+4DX^WnqWY+~Cg?S)G((Idt(F(N zJ>?_75T-(t0!RKkUe&CZ;B_{4VSfOu7+XRHdgN;GfYp#Ss^#yDXyuaoiv9Yz6krTd z2kd%ox|Z9rfaT*UE+Ow6lBT-O%?#MKxp=NEM%^&~)*^tf4*R&AlZ!r1M_Qg&Gz4?S}`!}=jDzUU91@pT{^XE`iZVw zWrce^e&=<`;p*y;Du5>%rv?e7@=bfEq^_S8S5+qc3s)e9y84Ym=BYSWS;RD1W}dwh zVu@QbPvf`!?gm<2#sd|G=$bWRE>yAAxg*Q<+y-r(#YM;X8cA1IA$pdtSuYmV zVv-0hcPykWt@#H!2d*93-IvwOF)SJ4SNYt~HiN@HW_T*sjMmc z;VQ@dKBt5>xUZbqRRq>``=#FkYO#q;*X(y`o|vcR1;Wc_&WtM9MYLtH5%($#9ZGNL zH8g9@PYv)>5DjU!0;}k4GWg4?e(f&ym9@9jhgxjSJn?Wc6AFzfZ*lN8zS8UGboNgC zHhyzk&p;TPCCN8E^pJyb^eemX_NJe+jxIpEuBj5uYK*3NYb|fEO4!X;yF9e&ZX$c! zp6_x^Gew9sTF>VdbhY6esWTJmtSm4*W0yvNv)!5|4ZQLObt!J_5adp~7YF@veaj-m zbKZ_5En>RrrCp{a?{O`ew{q)rvZ32da=Il*HTz=34O!4smDejFC#D2vyW3Mulm8ccqY(ThCdC&kH<}30MV?Hk4TM=- zO#{BuMCm>=&Mre+?k&U^$2lF7f9F4U&(>+A6}E?M*y>4lW(2<3)$$%c?X%-ZfTz2k z^L!>Ic;^^W_5YAR8@C{CO!l>Yp~tCY^F<08=_wqfp+V zai-K;@?jaywn>Q^?;(q3tzr^E->yA5m($R6M*T`hcS^GM5ZS9_f>;8W7_V;o>_E@L zHLF4cXmoX{qmE=ni-*W>H89OAuCgLIYYPatmVKRF?;H1`!KML4 zE?cQAr!^SA$vl?3KiII~^i~m7Cv$?_auMX7ZIGEU?>-6dRux4)UcHf4019w@IDN{> z9^INW?PuJ@37GI3uY1D1oPH^yMD^{bpeerEpBlt_GSTgmoq@GO3epq8f25Rh+V*oW$5oVsfX1yv2`?8BoCeo0~y;_x{CNb7$GMIpDk5FU~WQoY{a&ix$9U?%gs2vjKnX&UUC;}g#};$wJ)P67#Klp+-W zg?JHhSP5af(TE8<@)MLmNy#Q!_fgfPzgn>0Q|BGvP=SEtS|WhVNo#cpEEZV{>gN`c zahRR?)b3Na{4NbM_=bF588{(W6o5CLYo!#HcR6MkL*%MC05HP>5*rijtLDhhlNCIu zhMu=l<0!N~G;xFLd2x354BjB;P@FTYJwjFYXsC%(pP*_MsbcTz`1sW2-F+MJKF7tZP&x3;aBI1kQ+5P-D7Y5%fJ3p$J%0t@=$O1o2kn zZ_X$k*G8toM(TqQTec~_wEN9Lt*!LLhW69g-~CxnotR5I6xfnWh76>#wVLPZZ9h6eE?)N~`%MszgFWZi z6Aw(S!gb>yO-yqyUHb|1oCqszmgqm*$%XlHZQ8yaFWKwP;8L&7C~YSiHW=vk8v3v< z?|+_p$usK{Y!)UJv#fE*v%ug=UXoM8Rg@fXe}@dUpVKGUqK1&vd37?e&OPDlW?3B*hu4fu7|_66nWvz8taS%uWUAJ0@<%6<{L9dV z`+vjfDWNQNBY+Pcb0?BbDut`z3Sc?xrP)fZ30*PYpTE6xNXvpSI%QtB!S|wkeB939 zs`Z3#se0>fmIPTrGl<@3$epTx1U75?4Q2e`N~*MOA6UN+>d)ajVe|R9JmY3gSMHP7 z>2{$lEi%+zTU;@R16|zwOKC-|Uxq%3h0gpIblPo>O3i&nyejT%d7S-)xwY#9C9pJ{ zE8)Sbvnfv|x>+mMu(`@kC;kf908yNr*Nk1|)glr5-eJ2Okoi8Tp(Xf7b>L}MyLMlR19(pwjO9Va>geKcW8l|$dNw1* zhAV=CU!7n4!N)aU46zn7;IX?JRRQOE)8|VHN#mewMY6kQch8~^j(=;qyR2$JIoHVja zK8Fq)nKD52H#}qw0n@49^x?OJ1DX$+4PJu7>jzJ7XhC0+Acj!}U~J>kXA^`>TwIdh zBNr389<|;oAK)UDT(G3;o_+HIANI#VaT#M&0G2Mz4(M4cWZyCg|22nau(Jga~TXI9?NoKw~_SN&|;HiTj z?rCv_w6aMXx!5>|@-Q_BdWdYUGsHjNJ9!*z#}MbN-@EZ9xh3I%p@i>CwxZn*!=7MW zT9c6BZKD2_K*{3b|F}B4CC!)o>|DdW1}@5gphO= z$xr+M{v|%z*6O|6T#}hswZN+FQd{?bv31sQZFEiFZ;|5e?(Qw_?ykk%iU)UhD@BV- zkwS3^!KHX{hfpkNad+Na*R9Xlm7^P8EN*S((9axi|Q|D5}vhQZI; zMi%e&;?d))Xt6@)-JjJa-a;J>O##nr1nYOb$5VLA2>*2SS|t!^rA*Rd-j=IP#|y?H z55Kg+1J}dJopPm|!+pZuf#Y!Esh|upUvkx@R(5WW%4RZxV~+wptrVP%T^xv3zL+>( zQ8d{1Gx3`LXa7Sj3#tDzcaEA~v13-^8aGmrh48atTx9b6^hc<2uU&2uq+{<>8n5Fk z0F3i&2-4lckt^SaHdh%vOujL;o-H#t!Xti!tv z)KOO=mDOEN+lrEq29}B4dy{TJdxMhL7t0CDOZzPEbj#?;nY*IUGJ?H^j`FAxnokR;OXcpipy@e(vWsp951oICQAc))``u; z2d<%t?0IQVws*&ms{u70vxZ$*sqQCiTSc>EFSQQpPP>#QDOleR4!ApikYfl8e#;@i zypc9KSmwOok|TG#NiDjhzu#e*BIOsC_X*zc5zzlQats3vO4s?$k{;n@+M_pJl%H}~ zO~T?1IDCVSOKCxVm@URGe3IWPd*G&72dw3lo_dc<{RfE-0&K9kGkpxr@ z$oe!Q^&@AQYAQ_n#cCbsk@Lw1TFMyNld}Y@+kK9N=JWx{xx%9I0c{eVeX3J4*V@p4 zlT{#af{k<~Fx%`}ID1iNO4D&ID_=gCp~bAHQo*lIH2`0 z46K|9BszV7>h`7nUgr!a>H!pv!7JYg>X}=9vn3}3CirK9uFEBY^88un)!l9ScEQL1 z|Jwu-_1f2Wf%ofi<3Ud<<+MQHR*yo?-^jIRTw46|SRqy|M;>g|z>`n0Zorn2p}8C{ z2yZ?aPYi25j&Eb3if#`A#z$3((q=8xGvJ+LVO;Pz5dbbjbaQcWq5OPE;geaz+%`bS zgF4UO^YTO@J#+ofx1BAXGF`RaE&#gWX|4R8Q?qy7B@ntPRd0r5a6s!m8L{bqmfoq5 zH7tGm*uf$__UWCT(!%7D$k1;eKBu%g#{tq75n;Bx$LzrCy!quBDQ_8~u^;n}Y#tL% zWz73Ex9tavEY|nW^ewgc^Is&i@pMcyd>?=DR1&C^)Ldd~N$)pTWIVkH5C?mwBnv9( zySWJL87R)CXFWRZ-)9_;>bEVjk3bM?U#4b5sgoefbrEugVk|)u$MVC+hk(HF!t*m= zZh0;q3-i9HTOyCz_<+f>lE=C&ZvD*pzg&;eKN@%F4*hAl!%St+}(FQ#4i=+NxY}b|7nZ5DdEgjZP zwnu0%N@=M_L*hWCR(c)Fe3`Gwv>0@+bd%TmL+?vXzBj@jiv0eSHsvPUb%DiG2fypB zD*>SUj85exielnta2cQJglmncRu*eKctz;{l)!35Q37|SZqU#sfn0m_GTiKkObyPE ziPY=rV$C}%5nKap%32fQM)77y(Jq-0zgvEJ4hNuvYRcU!8${+Cc0Ug7_HLYiea>h61^k;moavG3pPLOn zytg8^E){V@eDlQjoUuK}Q6ciNXQ=?|B zFU}k`iNUUs5*T3ERce?Dx!FZqPox44O1R1L)yi$Zd_(3H1BM)ivvCjBCi?#=u|L~R z^V&{ZeVK|PXXpBQs_(GWCi?j18U{Z(1#H59&J%3G?2uujr%i(P-m-gd=}$pjBY=Ip zDmq4Vf13XyDeC@hNl@KW%n4DIJv4CO%bPq;>iZAnZmTC|tMZj+dKpF=e{Hnw1Usp7 zz#k8JgTF+&U<;_e-v9F5Q9Bh75AVnQGO3X+Y_s(u|Ds=tNUuAo?oh07hHjlrdT45N z*JjD@`Y-`qfZ=EyLgjbSd-&+<%ZI_BHGV*jsNklnF&js9zO1J$LG*1i_K(V)@oOy< ziZ-^k53+z9|C+U>b0Q^_%+)SB1^8QS+XCy%MGF)74y?5g_$5C(p+gQ&awe>jxCr_; z82M*VqUWdN#=Df&q6ss$o!+60sgKuahT*1eKROd0gIeBLPt%dVUf;V>mWU51-*jlp zc>J-yF(qZ$2m`Z!d)OG!Up_`dRc`ugnf;`N_uAP#)7vms$0 zNFK6@unO_hLroP>Q1--3@Hr5(Ru{W_f^p|vlaX5ILajC}A!XZ~_LTj!$}!!fwj-{@ zp;g%!#{&7h`oF282Cf38ElR$+Zw)BV@_4br#Asl_^jGntcR}&bTGv?Y%FKz~@u7Fj zVsZX6@MJS1ms{jJ1oRnt*JNcPd7qk7Gg_*&4Q(%H9+ZnBj;J`S8o$Tq#WAvft}*yd zVC8d)ek1%LUP<3J@c^E{#3>1l0e|w@v5;8(E$W4 zRkiZ+N@`4F!JLsT7)n={;}7u>`(CD-J`mQ?6xE5_GInQF$tx>Hh)A(^+S?{$T(B)>mqL<#4R;G5Fj|MpD_Cu_R0B2WeW zykXE0&C!!=fDd|6ULZ=e%v1b7rW8$jWAO88B~_nNl>_37-@9qkKxOsus~^R2nL#2s zm)-YSu(zyb_gxO89935O4lFmXGZJgCmv&E^9>3D{?0j!Y4PU?KU5ej7eRcF6-|Z4e zJ#c=_;4^(p==?>0S5OHmZ-^MnBaRNb$LT4ul_RLZ6JSbh?F*AfG+$#_PATP-@j8m=v4Bzm(lLDX;o7i z@yE-381iQmfmuI4ftzyOmY1OShCw0)2oCg8lXwO%_gY3So|m1H&rEp}Uh77m=1qRZ zGo0@x;hQ+K313hEy`g(1tq5au$%Qu($#`2qMkoB&o0O_-;+-@JJcQEqJGN3+*n!u$ zLMvtGcWPC>uJ!!u)u)HMQ#G?k1tOVif0z$MS4L^%SUe={Ca8g#)a6Teo7yIyDvD~3 zVMx3^>t)j_7zTEooaSl>=zi5+oRCxt{{zZwfk_JLN^O%&&_(t>y+*y;A<1wth#8w3 z3oWDV^+n;oa&3!S`=7~gpMyos-Eyk7ZT0=@B93#Im%EQyD%aw#VL=G!bsT$ZdSW)V z14d!cG5Nf*k_7V+mG%$?FW=%eR~cNDCJOu^sPnZ)q9A)ph{AY3#BAkhz@fh$kUk{y z{Vt9@JA2}6I!@;3{-yL+?-5EJhJ18~I!-qj zkl$SE)#x=Vyr2w45GZ@CO%LoXhV4r)5F*isUSY4=d_|StUqks^*}ka!TO9DOxaVL* zj|u~DX91q{VHuA&*f+xxUNUMtsom}+r;7WzEFHGopUG4(`8heweS=RSd$nw))stOo zU2r7hcFgxFGydbvi#;**8tdb-G55UZ2DY>oBTy(&y0d&K*bRIAae1x7Xt8%USX;pU zef_PNj_bZ&`O)l*(0lshHN3;i`TH8ES}+L#-+iNtojmYB zSYbA=)eZAVN7?PbLrnke;I#n&QVrdazpBu0D?y@+gV0Y^%M}0J%v`DpN=Gppu_0KFf zI#s_Jo!_)_ z=NO?Wgwkyi*bO?&LW*Tadc6}Vl_Js=4wsQOXQxIu07M+}YWpCUJ(a zTrs4x5)bq*cCly*=@9Q>rN7ALDu&h*xk+tb^%^b_G*w`z^-1)sQ7{Oo7JkbaaSrCb zypUbKHLd~iMS$7AGqc~Otvl>?*BK5cf1C!3fb*Z!L8 zxONd<)gLoVPOCf4tR~u7_!aDJ_Y@7ad~3l3F3)w*nK${{!v`#BY8=*zwir>6T9k zYgsf~_qeB%4rjS8=mbITU-HVMZywcF{G)W8ae@v-4A4UpAIW@pD6nk*f|$2QRYv@t zDS_ROVHNi5E-%&y4yMX(T^^blZq~vNzl>YwL;;3_V}GX^Hi*HFQJC~C5?&T4tKDqJ}#l-e^CJf7zp zsjzXUZ!~zsp?(Q^(8vo0&q|rz+2uUFQStu#_2<U__ zQ-<9rLFN16N5!%D;Z5=3~@>5v;6mGnQpNyHahO7q+9s~*9N z*cui~br_;q4}RHX&$f*tPJbub90=9=UseB36LP`zJ^Jv{P74%p7jsD@z08IRq(S$P zg|?7>vG%@&;!!7F5S#gUhP80#9dl-*kncr;YEV@*eall_oPSxS__9@{z3nU|z`<&7 zBJ!!d0nDpPF2CVdUf!*9nfzzT_TfwLtEr~%<|s9*uMc2Vl8A-Wed$hbR$qHz9N{$} z|2PGGWtbkA8h5+Lr#F}kzF`7#iQOn;$bAD>LfxpZ5f0C&zXFGwa;X()>_{AI)LZnkJky3blv-4?Y-@!>G;;thyx+sFZ9haQ&4S`UONHaQrm=i{*o zqx*$qO9(3WuXp%4dffFd)lp zxL_yR2WXx6Ttgb6mmvJTn-J$w9+kccOYf7|N?yGu%;f_{Mup+`jILvjji9NZvectiA=TGQ}wc z_TTcdK{T{DTuvWz$(&?=2iV|~xdim#94g-k^WroTP7+M^uh214vx<%FCgIrXvjF@_ zb{l3q5>w^6A)SXpZfx_=Oh&AA(FT>+o^jbibvFU5fnLP;5m5QX&8z>{A>jC)sDy=~ z&H}Ho`DM$eB&dV1hE4oO5g8Xbw?9y)xN#sJ1zo%-B?b;L3g;WASxdXCP&FY|1VfAs zPYFGC1rw}pCW3dFY6L#UCWfrM9X+cq&G!f*;JQE^QJ$3^-ZmGs!sPQgK;I=EW-D+v@CIsEwpV zNgsM@q+ce&5M?RrGq}))o&LwQGZ_6tls0YwU)n(~pY1p8?bKLCSi1`E6 z?e8w=KmR)?jtS>v$+n%@v2y6xjZ%8kXopuwTAvOd5i^cqn$yzLI=_F)GOr;^ecMg@ z*V0uzucQfM<^RlbtBb3#B&5w&_e4Er$50L3#C)C`G3;%LzUhA}M!w%jh+}A@d_|=q zi}S6r8#fw(Go-x*+2#0E*JO^;gtn`%dIkd<3mPpjryB!B|UoPP| za``7`#}7t_+z)N0T7&S%29(tQ!1MANZ?xL;cCT~OTPzN;sWlUOT z=L0wS$sCJjZ3W9_NsEw=N^J&M=CW#U$(!1k&38k1$S8+z(RdRgwC8g&45SreC>^rw zXU))4+_`yI3u^FQVy?zxf}yuyCshk z!fM!<{_i|R`g7_E+PX)PkCvifa~kJbO0IXBTLZingDekr@>titNM%QiaX5_%Tnr!2qWlIY4|;->;q4`r+Kuh0S={>8I}AM9 zGq1Ez63RSrj`2iH?wdFC+0HSF(uL`qZ*n3Rp#Pn_XyNl@?-+=VD)q%s&+kWsi**3K z&GnXBmS^`F((8o)S27$<(XRhCe*kqDmBP<(6n2xzMDX@!VoZ(;n+|{6TQ=iQMe> z>#A<$-25@?@}OVQ&7_2NR_fxWgA4E{@u-igH&t3s1nY$j4dzfg}Ry38%2U@~EiO z)cDSR3V4t0Ksix!R>UT#Cag=rah;bTFgN)h^&5KlsSC@)P?_2FB$Y0Q<40m8-O=*s z(W_audunKh^pJ~JRXC-bNmkgOK+^?6ZM1XSL)Sb<7xIVsuk1 z5|Q~vf0i~iI|XrSrFCjR49>SLtIw_#g8W3W`kVbfH>Ex%a+|@?P1t2G2AFYbQmOKc z@K+Y9mKK%anSFx{xJV1;7bEQ!SfxzVrD2x&Wpqxq1Re$6fNpdBCj(C#aCbUyPdYmm zPJI_v{s4m}FMkNcQhL(jh!*#xP<2Tn+N}gctDWv(N>Imk27E}_8%&p$5|VO`q&%3) zN}Is9&+4a@jOPiOy^BTVsgmUXB5!8~Hg)`@CQYy%#Sh_^twnc=H1Re+AXp)W!81Q7 zi#fsp%_EW|DoBcBIDvgnlaV*^-clR3oMBT7Zkvuy$)|=dUyrJ(Zl@j2+N>_!_J7r< zU8ZTkzps<)m#`ylq90H&PgA9(!R8*^W(6c zJY)NAD=8i>iy-X((f2_gfa1$9jez;~^Ly>KSZCK>DN0NDA3nzZ_y5WBR*JRKUQ(W2qMbMNiJ~2jiE6LkYHpbxPNr2 z#d??0-stY7e`ar_lytctasn+h#?r)V27E2sh)B*qX zqPth@0ra*uI69Osa4qw^>J{=jAohasqV-fO_B6WesWiD#s2z+4gB%(i-=pg$fY$As zV*$X8ds2HL`~7NyCiYqs91k1{hIP67F7;A{jt|_Oley((ow=!2x*#jeV_W~ z$WA}hOSUixflP0+%ekEzSnhUsZ?d{l`Q`GW$7-;0T1VjGi)5L~5&}i2!)Q3dLcV13 zo_TqYHx+1&aQ3|Onl4M7($tWegP9w5c#jsmQ8F zg>fXUDWSP9esO-7PO(2KINQJ>BpFuny?4|_d)y%P?4*|Gq@h~2&fCvq&CCLT_{T0U`H zaGppz*FuGykua~sV%=IsuA<7{yFAIarcwj|?W4!SKksVNR#}>Dz8bbTL^+HeYZi5^ zgxA(Qx1xc_9>}w~DyxdeF&SW9lq2DI(+sH}uT@5mD_HfsQ0ZBSC*%05(^nO!8kksp z0zY#`rmV|?@B*VH_(Bgg4Nn&CxenN%!)}0zrJgW=RBB3A$u1o z7sC~i{uI)FTmg1A_e=#SEh1<}9`*iwV;I%Or;wn_H}xs{`~DmuSs?|BH_q4H&!F?g z^wncfQ&xN?G*VX26fWt)f`CL)B{ErHT~tDJ@ZP(wSSmyR<)EWFXYWX0XF(1~x%Ve` z^&U^3VLf%U+N~|h!4$MMs3RM56!1#C>jrClT7*O`^$T@td>j`VgbQjvu zeMoqqo?3(_ey-DOL72-O5q%#QpFubQaP}UwKWYga#IQ}UPq7tVhQk$fRuuqJ?L<2i z2H2Ran1RD3sK!81jOb>u?}*fr-cVI#r}Iuz>V zX$1q@9HpTtz`-5g4tUUm;C+fW`*W@U8@8@^L0x&DiHvQ(ccR0cNhFpjeO)DMAeuHr|VC8ctM`{0*#jW=g@AI6U5d9 z_b(T=2@8pc&>F5=PBTS>+ZCP^X+kOpUzDIc&5&Yocugw8Ru`lLxiJ9>#DRaabX2-U zhDeH2ogLBC85*5r3_$i(PYPb%zeHC#Y}i0_b(MZI;rf)dZs4_Ql}>BwVtQH^6;7b` zc5FKGSJ2w(N2(ImkyTa*<1OirqT4kRD!{jLWzK)8o#U)j^pkP6)v*-PpTSR}pv=lt z$~e}c?%*eW##;sU+_3c{iueye_1x zgpTQ_HNW950Cb9_Wf7N&P#KC8XqrOj3dlod3~2}(qM#z2&;dA!Y#Uthc7!P07v+>wb1Ml+T#ZPiKJ>)* zJ0OT-$0vfCIFFYXapAku3Py_+Ge^ z*Q`Wh9{Hes$Ty-$BI8?y;DtTO;FNeO=xHxNt#BOY+x$ni<$AbJ>UWQA1{dTtkL#wd zXVbk>!f<*zokPXm9*7^vpV_KDa4gD0TP(Nl{=Vcm>WrZ4en@})E#m!n@BDIU`sA$$ z3hXi#Pc)$_#8gN8%YGy$_c%{n$nx)OcOM@#yV~Tc)oLA%|7HFIDP7edH==EoE5g)g z_Z*TbhCArc@Bey@{yJHh90t0OkOv*E{&jG;)W*a}HQo#P%fosQ(JPhlLruPr+{StU z(cNwLP1SS|Wc9SeX}<8EC;#=!lURRUKyFZ1N(WJx0ud|#dJgA6>=1CC#;8)VD!>pv z4NQxAnZNt_Xd3_eYz(5dOc0@VEW@YeXBp)UJDwo8BaCPvw8#+5b4uqy>=? z@YujpO`P>#L-_Y0K0%o3kemDZX5#Pvy1jBl*4l0#0OgfV7_AHE($S${EW@qWb(s^K zUmBO(DFb4DPhRR|F&?C|x|*R@Ph;{JYsthVZT*-=!5h=v>iP|fGNypg_vlNOl-<|B zqS*Y*>yIg`C}fg>{~6*WI*iC+L->I}Aqi3eGAQq)kDD;uWyZ#7x!e|E%>RhVNnIZ{ zIlAt&Ax$=iDnSWVYmsLy#NHnf2BYl8F*SkLcK8~xp6_?FFJH2_o*rfff>N;HB4uog zBpSj=^O*msO@vxQdd>*jy;<1ry~B`5)Y&q`4`>>sM&0ec&aFdKz{Q1ah660M8)N8H z=y%Mo{=M@5Z7JX|D^eoxaw9^eL&_zcqp{t<{c11?Fq;i(YHuQJH}GHr`Na9qz;r;F z9IE2+_qXQQ$i;H$0Ej^dEw{SJ*y!<8(V{Y;H}X=*mk%D%xJB0>@h^aUhm7Rc6_) zBR?Eghl2UxA6)z2g*e2@k0G2M=BRl5W96cRuEA3>w@R0yk>c5JsG`ARd`x{{>0cs* zm=&pr%~G4b;)Z3zarm&@SfEpqHa#~|5vi6} zI?Vm!{tdCmG^j&}p-KU)efgfVES2oP40djtarD=am7d)(&(R3m2K>K)KN z0-PY%K}S`wp^6|PHG*}EK&!{KJm(cwNik%9&qe($<`XQt!NFzr4t2tlg3fBafDHXN zri-%Ft{vqrI`ac&D|`06eD*y=rhSHR9{Y%<6E5-SO}LFBSf?R&hbPIDkaV zZdsIQ#y-ZvTSlCVbn@(j9?9@tn$-z4%3UnjvHiExyYdalPHJQE@f+`cSTFfh_e(@) z9NFX^HHr<2w7+eC5r(w^CflA1R=-VnoH|6cHz~mpWaSfw7>ehGS@ZZ6Nh~@=2#3hi zXis2uzqS4~b=WFZipl(Vh(tZKfh*#9k%$U7blX4Z7d;?utmY+L(Y} zIfFv-!0@kjdV8~al#3sF{ypN$bj?srk*<1}eDaJz<>o?)#` zJr>ko%ga@ytZvjl>Vbaobn=8DI)>YDA1A3I_@9 zBZ1aM|D^@!_I0A%K&?^zna*pA>YPbE>Jd@T!T0Mst?;*+X#|i;kp|~CKkhF}`A-S0)yY(1DXX2Q zD=J|XUZ1B9$oDdz6J-!S>HUOz!MQfy+W1P`W4?O}{&A{k_=fN{zd4tjYy9LdaC7?mX1rIX2>in7Cm3fqYLfhkA5`BE#XX z`TFkHN@sq_PJ!R)Atqs$^P@1cob-|lsa$fdTg-;GyaKa$Vm$3k%H*oyCd-ho(^FGQ z5*M5$BMW`lnh%p?N!m9t;V5{V8V#fzL@Fm{6xamNl&yJ`0X`Mq5>sZAi#i59sk^qf zOT%BUr4AP@D(k5njXaYa3!Vtfrhdz~LfxWKy*oc{KEklG zK!Git9v&f}MXM0q_SR8Os(zcKw~qDBllaoE;E$#7wk%1{LLp*JJlPMzfgpFkm+#jvZ?d2#mH36t12*{xFV43gs_;N`ngm4+0#EE2=FQ(= zlBgOjvx-`&Pgq*qrk85!&#fnpw+d{jiuR6bn;9=*bmXkR!$q0&hq@mSNc{~z$Rb-Q zFz?V4P#!_FqSD#koU|m3F!U%?nn_fJ^01mWNu$TEyX_t!QVg`B58uLYgetb)Ia^VR zk>|2<3aB}&K@@U*8K(Y%Eq$v0fGv{bg;v$VsBHk8HJJd*25 zo6 z655UzGD3#}XpQ{!t&{K=N&YsF9wx#tHEf*Wmo*$A7WVFW$uY9XO$ z$`P3{L@cGZ6wv-^tZ8`pH|Ws)7wAYVR>}NgsQC`Ku7JQ1Zj%sM6Gd#FD}{n%q($$& zyBr42Ki}hV>yQVfLv*FqQ3wYsTD%1iPQgstFH7GwiWZacCStAJDTS4sPd|bNLb_m4 zQ`2_Ie(34(^`TlpTDRT1&y?i_TE`5n5n(jr{Nb4tkd<4=mFi@sSiYNB<;VUZfv-7{C)%C5rZV;|oCO7a-iTt>A;K1blwLF^p-O3FtAa))!5RI+h#VMC!8Q zxjybJ7EG)M%3obq!MY%};ol8PLjHu2ljxf?CRezH_37)jaG;kQ5%>?s4&vp@8+-x1 zE_me70)XPR>n|NVtWs?XE$Bh{Al|EfnV86mAM_)ud&{Q-s(NvH{EE2YOVgLf68DoaR}P@quaW2Ta*E?ntHnLJS(YS%%w^Y2^q zWuqWV>xDn?r5S-=jI1j+l+vXSl1n5l3V6X<>01qW4Q(uV76-r7@sso+;F(8uA_OR_ zh|BG(vTlaFDUyT#ZZis5xc=V0eX3#m)iJpb^ga87j`%zjVt!4{(LAsD>i2;G*kAyN zYbs>^8TF%#_+de(8Nx<_b+0DPS@k63yFLK%qToXYDmO^#*71-L>`qAvsZUdI>C`z} zMe13aJm12Q7_IZ4n(=Gu>*E=g9M;UD3Ah7TPE~*RnLfZ`@gUFV_DwM<=QnQ`{gORj zEK==llub&pUk+OOb0Rc!I0Hg0qBv#@x3V0HXEgzcjz=_fY37Ic@l; zbE-A*sM>6$pYY7N6E!AdHQTkM<2?Ss5`3Bd;YGNAr_$h}GTE|p2j#4IT6_EH6^ zu?B{i*h)JuAJ=g#ii>-S5g6QPa3Sw37d%-cSDp=1HJhrFJ^Zp#J$&4Mho{&14>Iv@ z+xm}>lT``g;vC%<3MW3C;0(cgxT z-E9FpX8&a{{2nxO#@@dCn1o#VeI6ohV-KUE67CbK(Y;wF_ zaO`+@-#B#R*ZR#@on4#0UIU$#0ww-$*wBIng$en>+bpK{$R>UVr7W85L1-!9#jcy? zyUN;!O8>*qPrCotslsB2U=ffL;g3^!vnSA8_ z`1;ehp5k%A>wQ7VAHi4{EbwCIiRUB|^{1@=_0azQxbcG7$^dv2+bVtCSMe!g!pfSY z0F|mk#Gd-a(sUmOU7lnWoW)-6C&$FPALeT_AQ|%PB4zfKm^+V`F7plle8f{!mxqT5 z?XcFqr9<0I{=tC(xh89C-nX@jtwN}$HNZ(dC5)TH)Bzn0Ze^ zNw-kHc3Am83RJcjzCIYQUmM@`5p;!!J(P%jdHpfBioi{xt@roK(ldq|I>uZ74_>n^ z(%g=bQV-WTS-Ylp&wpmTFtpksQCP)-I2O)hGV-WhFgAQlx+?dl#*Ssqb;F=Z$@l4o zyqaKg0hsR#5NcvHM60npIZ*+!HVEyJfA5?R@K=PiOcLFR>xAbWByHXY__RD-1o`<> zI7r5GCupE8`u}L>nqe^Nz0V(ZilxL*e~0wzheX^NGwbpM&DjKeyfEdpnH!3A0gSYB z6>KGQs18D1cWub^<~te6&#k&02P{~d=zLM%rQ`Rd+AfP8M z%FDbNKgC?c%hIRvK(7cVR^g{Khi{^kHJgt)f;CtuYO?qk{mxvUQXGAKoAVO=hvz1Q zJx(}m*^RV=Si(>%ldoX?<#_$~lfR~0_d!k<5;RhJRmx!;7=>8-k7&|gOv%+0(MXom zfz}+J!Q9okqLt|Zy^pg<>Eh?~W2|sUQ0`uX==^;pD!Bfr{3y#Fg=ad3a^jbCO@{rgr=SZCPC+Fm(ytPIZAvE;(n>{*zHqzxRln;d{H#$zGKmi=Jsidj6UI;w4bn+j^}lt$hWXoEo#j$dPR(>i%b*nL z*NwkT+@(QtdhnD=Eg!DJ54Y(>l7y2*Q=L{04=KxW)kj_b#;a*16aSW2nNNIwl|>RN zV-8Fojj75Pb$LYWN5gBaO(_*PGlEcvID|VZ-6Ll;xtfbGGUc zx-nBU>2bK>q`O>C>QQdGuBotX{1WpRnAvZ~l52U}OK6S~(I|_xdLn=G?fbo5L+XJ+ zK}%L&A+49ou?dZnvjan=JG6zhTHk!bRWaWoL=>^4yME$}hV87HMki+FlZF^h;UmB) z2}jdQeVN~nKwstInj?Oq4QZZzWl#LPAo6bieOqEQedYaTIO_5*^=1*XkFMc}qHmR@ zBY<_Rp)G$nd-|y2HWZM$oX9>#(78Oa9>TdT6cP}7^h^Alyxs)2ac)UqeK9sZOK%&C`2UIBSF~cfe^aQB{zUni5J)7Y>4}+*zIJu4+u6f0zS}R9 zvTbQ2Jh55TY;grDg2X(+e(70h0r8G+Djl^D519n-Y)Y0Y6xfg3mAalx3Qg+(&`Qt$ z020s5KCv}0O1XT3=<+{Rgg=xRiXIl_)FMfR3zzbkzdvT#OR$2PBhWj6|AokGO~ z7uN9ZG#n4av*Ae^IMd(g-l~IkawSiS^3|Ufh_%9;o5y=4Z;Vr~UpUqg-zU=2_r>%w zdvO>>Ysd&(BBJy0Ld3V_pG%|gBw`a&G47eABIrPao(owd4bsA4`7<$`|H2HF&IjD5 zLe0OT2rA`$PiB79FBz^)K9q_q$Y9p^%B%SEN9d4!<5CfXG*OP^7KADs)OBqkofI@r zFd=M*Czs`bXhGc$dl*r|N&Q`%{UPCu*4!|3tJk{v7wB^k`mUy)R? z2IwNY{2onMo}W1hvMW-gNtpSZj{e4YQM~1UV?2S3kbr^wP|JgbnN)uf%x&p^7EN z^o>(QaXaeWXE|$)AiIpcOG-_cYxtGW8;hp34!~i(2?5m_I|7$~(gBoS1jtOTd{K2j zoe88e**oFW$WS6FWPXTsQAXf*O5#6olc(Na_FCT!&{`?G#pI&MsOZs0`mlR;Jw2hT zd|EQc6ZK{AV0KCs3x&bLik%Z*8dpil!tFS4PA@zB!qMDTH4NQ>*vB66F<~6(G8NF? zbXkmHyAU}5@Ga|%TxY%8j{+$1N1+y#lt7uAG&v1I{|PboA9ylK$7%+z__Q6%t5TIE zx)>G`E($KuaxgZ&#&PbGpp$u|hq#RM1jJm&?sys8n%&R0;?xD1CT^HD&0FY@lpBu8 z<>M&3-M-LyO6Mcxm{fnh_DFg5@r)tQM<3n-T0-yY;IYtQkY<|R>G!qpo!)~)P4^o9` z?KhIDZBxg3Smlf!qd~m99h>w~HCE&_MOW)`YAC;h~ulHl+WLuCoklYwOl_ zaV_pH#a)WK7b))U6iR^L?(R}bad&MgA!w0e#T^o~IK{Q-m+sy7J?H%Re&tFCD{E!W zHD|^%?q`hH_8x^ara}F%@04Nu5eCgF&;47p0eGW=Iefe#uTa^HyQ~Qq-+-!D<3MBOs)hGdd>#^G zxS>;~ir=E*!oRN0(Q5J{0+Z%)i^N}{s`j>i!iwJ`e>vS4@qxRQPtmZeVXYv2J^`BF z^!pDpw6-6fdQyi$Fva7&MQWINbab1TU?n6$zLkkh-ttBBs9ySF$7>${{OdFs_NT2u z#qU}rLvX$=Kt^^$Die?Yt3F1G^}99)=9s?+3Fq41`8x;~4;RckbzC0R`RX$PAM|PW z(o_#(T47B$EgNqS)?=~M>VNM&xN6le9r^h1K4}%!8Sf>zeM)Qz%9`K0wt5C1H9((s zPx7E?wWha3L4~eoJgI$7TDNerbEK|f2+nYgEny!_a58hLw|ep9h7B}Eu6w>9b}?x1 zP+J;pFckfSU}G8*^^}Kc1z`-5!piY;%5q{TUoq2DTh0&3;3qL_h%(&GDtxcvhSeKp zgmOzOFiF3vP~0%kKxY)oq!WEw{AJOu=dwRi(0y0a-McGs1Fmslp1VG#F#KBjHx0~$ zI4>N*1gxQbDSRNjPY^`vK`H&qfH7HL4nnqZVx^Fo$PDobKD zH_4)vyZXVz*$WFRUcN~EA3LR=Vn}g4=#&S_frmFWi6^!|d> z1hm2cLPl(Zb??;jX-2zWr+3wlj4$mq?cIcvOS3?Yb->y8A!uJw1tgQ-gYhafaZ5X- zUW*X@qfYwV>8)kfd_NA2VwhYI{`~p7RJz696eN|Ml~p!%Uy4Q`b&#M5r?hIcUSbKYq0kY67jaT&Q%r=pi_(OGuJj1Pk6(2bZ=8dg zR_vx&e})cu1;dISn73|UcceS{&dcQ7XC2e+2vie-S1zY~$@&MfxxF?L)K_&JU(6yb)@=B2vtF(E z-#F7d_M4G3EFGxDN_jy~JfQ_H(+*zsjgeJkjTU4D*(ICpaVu@(Su=assuxx3pO6y1 z&f~ewSYU8xRkS_U8{xq1^V0&9Q0!>d8Ej1*MKhcvA5nnCS-lxeMm{9GMIUzaF&=TI z@IIxM2NRY~zGnm7)jM-)02*|%d7@7hw!QXv9HKA_zH87Yx;t#Ng^;Wm;TG|l;IVD| zu;MLMswcRK|0Uo)@+nx!+muj5B-Pk>M=rgarz%^$-{|f;Yy|VWRRFKGz9WVSrGVGm zx4mlR@~m7j`6;V{c}?7OkY`wkz3sKk^}~Hym#^>UupaG8@#-`91VQB1(8dLFoDv6{^$CkMKF*|$}Kjr<-uPa z+&L9|d`zs73Tf{Heqm^vxK4h)O>R))OkLLRJkUJ=K>P^mWr21^z>b@KPgAHTX8Jjntg`ae>Id(T7)4d%(6Z3aH zg29%FrTU|p{$QpYpL3zV&6g&R7BRWjuPaoMP@_t#sDW5qu!s$9ea3*m4zkOYGXP`? zRB3C`xvY%QXTs3i_7h?S0Ru4II4$@-(?5>+6i%xOCy)_-WJO-AL5=dXiR|dIb9xJ7 zzpnTM1_^q(E`C;+T#QNN8WLxUy;DUKY5^Sz#^(;t1sLnA` zV>tGV$w#5b7_<+g~*WuQ{^3sdA37UVJw)2NWQhp zqC|UdSihB7i6HCFtJ#vJ0Kl%7M0uoP!y?jyRw6&bJ*V5Uoz%IK?4f%)?PqB@)kpqhT#Kifv@us44YCVEK#3JQyD{AMt5%L zIp?g(%%vYz_UH|9K7_$dBjA-2$w)e2ehHt}9F2Q*1E5z#FY3sK#at*T#6p+z_uZVVs5{V%<9EVDnpWN*+Q=$$N!PFI zKfxAg_pSdSD};^MU{tVO82_LPggizrrWf7Sm5)=TQUUYhz&f4WfdN!-<` zXD?FjuM0!8@-$I9*l}9jf%l�+1Vm9}mrm)+?p|;#c4L6r~=I#RAAXDg2mE=r~>o zTAfz4P2&69iOA~B^3Hm3jozbq{EX61Sa@R_p2#g>%6EBrFE8#h6zETaT4lzJXa@KL zX92%dk|qE1<@k*Mp|OPdTFL7Fep$?jUa*(+z)^pni_QG`57iuX&;KFWyraH#ra)Zo zTU^?|U{y!`H$C8r6*>`g{e_iM0UYz+!x6yy+-aPHl=$3W zuZ#Pl?Djq5b^ltESSt%!18s^o-0j{&_(m>DNu3UC-5z-?U=Ri?vZw2Hi6|kTFd&5frC#$$G+^z z?RI&6*AIkZH}-^Wy-aT~nzQ3?d-TM+yU!%6G7Bhy#@*z>hbv@ew`;+6_hw}nI63LC zNY6ipzdkcW%(k_&2cQy%rNvHW|5ft+~6rntkPP#NqtxF?19X$2a>Hb7#8x;l;z*8}%Wk%PxCZJLCtH zQ;@`y>}`p;gcvWsI}j4I-hSqFX8CZep0^NB{oD2+{_pfWm4l6|7sd|R8-t!VBos%0 zVU8%QPP^XL50|edy%9WSLZWQfIK9$DUxDQO@FLcqN)&)FkA$qZ(%Cwo3rsTk@fqb` zz3=1RPlU#$T%0(nmBPM`6*)g4h4vJv#DNKf9+W)4I(@Ak4fPuO)KN*+l|bY3;_244 zWB;xHxz5m$Mi;hAuzH*y_l6&1+j1{Pw`3H9_iz8xaqKMzWCXx1pU=i4XY*#>ZIUFLH=Ae@=`y7)-|^|@DwouAJ$1{Y0Zh{U5rz|F}c!ov^jcD%}+z_n2CO{6zg z$y?WtT!GZ9dER?PePRJ5-S@p4OW4?8ZIDLdKWtm5Jqm0x>9-vG|KYYXka{!o45qy3 zi?&3XVt?I_&M+XxI_5ltePq%YS#F_-nt!niyPZw5SsYoq`{$I8GZqU72ABzG*54^S zq}6x4xgg7%v;@P^t)&Ix!xBGiQ|Jhpb=D4Q5wXdy8CD*G%;}QITXTEnkg_ufc5$9k z=<572CoP3Sb*?6Lho()eN+2*D55--CAf@%{Wt*9lH+-X%Y!9B!lC5XsTK}{7m!+b_)Epiw^@4{lVWO1CQ>~j%A+H)0 zsG&`^5(b5E2h#96F*$=xS^#z05f5UQsOHM;)mb2XL?{QLY14jAUBCV!<6R5ot{Dz( zF5@wk-p5hT^QriOuQVWd=TCHOMwm9#e>B;>>Ahu>8Lx#LD%-R|2S0~$1dNx+$*$sg z#jnYaEM4kN#wv(@J9a53;m#qOG{Z`8{E#R44NNlINxpZtg;4uvxUmakcp&U-)HPO{ z8`aXPf2W=pJvt0>t(h6EXbWv62Hv+VbOM4kWJ5v0yU`W_m+7~p#Fdh#L>=4uPgoMa zH`RLptI;O`iqXNR!{!5!1hSL+Rd37N^>~_m)}U>U+sw`GKH^d!WP^U~-cCpIocwkh zMCuTH?SIx;;71;`n=T0-ur=l*d-3x&xPIu!d(-rmIY==Lvar!Pz;u=)X;=LCjr
`sqZgdVQ)h`5v!Ta<1wJS&&9X0! zU&o}w!C`MBdlfy!5V8FDqcnYuMO4(9bDn;&k25r>Da)@^LebWLlyE>-xkgx<)x$TS z(LbHg6Zlir-`x&EV=Me8KZ0MNhNCZF)tegQXwsK=u6Q4p{H$=j+$YV|V}KNon?$x1 z*jX3-s#s%XMz-an<>y&F^Uy(tUs-+lHU%&gFq*2lYL~egN30P*1A^&WmuxS!mCDC( zd>Q5P!iYs=j%7=p_^~#J!g(fpR51i&ZTfrW_%dQ_4MgSXJeY+QW%~yiAx9QS>$yO- zL1R%ol*!-%9NG~?cm>OQ%8`g4H+=1oGc^%0 zrAc4RQNE(Dqy}_2rL?u;wGJl4qgPYATqOFUKd1rPa-Wj8uNhd>-V|XDRz{Ugw{@?g zqSKyqxYb|1*2=d!ok0SBK;gKk#K=y!|0U^OC|`=i031wB7+YGLN ziwh1Xe7CQwZc9w~T6V9OvGZ54Hm+TQ&bL~8M7#E>{e8~TCHZ$qJS5`&$B zJ9a&9H6IsFMNdFmN^J#?W*y*r8nXvddwD?UAHarSRjy_MFhbQj?O}M#i8TC10j@;fs4txYO5mH$lFD-;z^IjC$w6!TGuA3sy`t_(U7GM>ZbnW^me9wY2`4h_Hp z@8wpaW8rEKvY)5YBS$oEHY%&$UT3DHvvG`*&!;eScF zG?+z*pN`m`E;k}n104^rE=2f1lrIJ?RQyM03XS_t5@zjlqZCBOB`>rsX4P%R8L9qZ(%#r)iZLxr<1Ha&^%0ut%UQFwCx#KX1>(BG?`f~md#9M%b=DtS{T7@E&kih43` zdRU$dVLC~^kd3*5&VT;hvx)%a&fTBJX95Y8K(S7qh#pEG$!`;>!Dj20Zvbo^_r^hZ z0}X+AC!nvI;K2BATsYvz?}9%?Yp4f_&4hNK)s0*BKNxgf{`g3DN#W>PyPxm7gRPm* zDd@|f0N>Hbt$Aj}D@TQg7Nd4`pbDbRXrKwhU0ShUEU^71@J;^!caItVWQR83S3nDA2<<+*rv&?p;%%fH)4VDe+CCo2))ye}QX!rn$by!9VPd!?1l^|Hu21HZ+ z0oZ!~0&GH8o}P-V=cUkkfr}rxki(Iq;R(d3B*${89WMGUA!CTjc|lo{Mq0wCj+Pyy82#` z?!?%gfke+~zL(mC4pb8M@aoY`1uQ=9Tm~HvLX*7sG_h{nePw)eW7ju$mUME5GnyO( zPaQyix?kYKwt9Bf9j+d59o{JUXchqRM+rE^r~ZCJ4u8}TfLd^uBbh|)&R+%Xy89^^ z5&Wp$O^ri6C~Eddo7H`X_8{y3ts?mRWYd)T+3VrJKA3F0Vl`6#IU*_f=}!G;;-T*Z`-ZPO=q(ZYZ3XgATK@{)b1*q=d8H<)3xbnINFDEpiSok z+kcUl2gnuxMLe}e(E0D+@|T8SrJ3S33VwiB?z_rDlK7Qe1>7GEr zc6N}%F@zJRaM!A$QJq>kH62M^!B8Qp^ohtmrz0LN;+;yLeu;|*mXBstv!Sg~`2eA( zT_}IjFyr7l;_?0z!&+nU1vi~@cfF1;isWRYP@k9ubiSPX2cqM)5%NaY$f^|B-u@2T zw>`K1_O$C6;!Q}5k{KI9!m+10 z-(m^EWHXBDz)I;_LYc=rx2xuId|5s-kS@ySgBj$finXW0(LRUZGt=7> z$x9agaOA_pVC?6;5=qA37@{q&JLuYDY;wsQd%!bxCvIN_^21QWGyH0)=j{jYN1E09 zGa~M_h?D0Wt_@J_^KUY+{5@{RcAaGCn%@!E`j_r&0eRPga^h-lwtvj6gHkYBn&T9f zFH_Kh1B}ApyCdjS6Rga;O{IuoHt`?0?}JL0@wV5Jpy59@K%GQl?plN}*IG_7e$upB z879m{EZr!|cpTH3%#N!-^w2GFENwSBy0)sqgCQdRYUVL_4+UmI<7S5GS}BOE-uYDp zdjI`aX8#z0m{TF;d^I2LH$=mg;T98CtvN^xy<*hx@v^0z0^Q=i4u_$+zDg2#)^$kv zFrmBHX%twUnNX#529trh_>w-{S@q5Pd+anBo%4=n z%vw|{nf}LfsKt1!X-|4?WT)^$xqZ-a$sNe6_tSZDW{mVp+AbrFeEq<`D zc=%_iD#l55;apiuZe!~^3i}#{WcR{?M(jYR+RTpq9QzF8BLncQ2rLM>IxMvtl!$jq zba?qjvjx6wd!*NO3nc-ISI<2WYqDh_NB65uT~DK@Pjh~|zq{>M_VHb|na}bgeV_N{ z7@k@s^{*e@)Bw~WlAP5HT)QoFAU>2f*aDyMGU2ZJziP@nOpU1t*aSUFI zS8yuA#{EZ=RzbkPpFfcb79MoL-srl93dI0_t(AW~x~B?vH2?eVd~3wz7G0#WeCzv~ zqJJjNKX$Pg*oTb}{|EK^qyRgl{c=F{#b52A5T}(75oYY+W7d`Zj~v1OM|pZzSif5J z_l#jx529g4KgdMjG0frPa*Nz?{-Ghjr$kp=>t6>)=tn|$^M_61vx?xm3ZF42vy0DU zAIV1hq%DG8PWSE4t=w#N)jt`mvH0DH2QP9co7Bmz{`Lc@C-d&?JtBsVAa{gi*1uJr zhOgV9TWuBi1#munWo&HzeEDka@qOpv`0&xku1f|FQ;>54k+d=F-QkXMmIkpFSm?yeI%Kl;{ z;s^;<>_YDvO7-ipHomw{qc4<=@^NbD=yM{9O|}285!kou|EtE)=$*jcN&^^moI=-p90n6{NEPiBQ3EmB_B8F$c-^G%aHj|yv>D&LBfzyJ z{KH##;tWzH2*_r6r_wiKMD5$S_d&}zaIk>M{7q;~pSioSCVedW9<8j!Ax>Z-`!HMqeY1VY z$>IO9sZDMTu)w$+n2vE;f_0GWb()UDyV=BrdqEYnS(Fng69-f4bshYbzOu(Wyip^c z++sPe{$6P48m0kYUYO7`*Kn6B6UW1E27VElM4qkumWEtTJ9#CJ_@1H|=hdW=XXCU* zEKwaP?aR76H~7F0bi?GAlNQS`B}8W^sru~HjZoo?tEoV1ALXGXP|^%jUr zYLxJ(4-6c9Nu}!=`wesZxHW{5VqckN^&7Web2!34po8eoFbBq2sde`m(G4^K{q@tvV=V64M|k0hOhAIV7G)S5)eYNd)LDT|ho-pSS-qt^uc~ z4#j{obp=<2F1%uB4TjM7+3<8&$^&c#1(y+1A*io0FQbxehz=p;xN$vFD8#Wk0HkGJ zdA(+mp=9WbA{SZni4vhG&D`F!`?T-{M!KE8{wPz8utB6{X{i~tR`!9SpD~V<0wNxd zFGx@#()t^QI8wjzE3TxozUwR4L8h7fKssHlfK8c+PlH+65VFVmn{xl+8qTs+?@KbD zR}R#yIkR8FFTV(c8B;GS>ChNR?WJGr;E&lGLauOHCIapTj4fbIeHfaSxisS9F=2w?vaW zJkvd}-(4C))=;4sdh7w1=tl&=Qh)2(c3f+&xLfhB;xTnt-EP`7x|s9fY)Z0#`~uD$c=q?9Y#9lW*E~-(ueU3 zx@qH*UN4`Fr$h4GEU^{o`eE1<34Gk+N+4J1re1c5=`h#kG<>{xOm3(*mDNt~o>OTq z7Oobz{j=y@zBz}^XPDto3EuF342K?<#J8~zs@^Frivi`b$aJ@`)>Gsobvi*CF3ccY zqlLi~&}kaZd1A4`h~py4OGD@6KeVZ>NblmuXn2;YNLrS;LtnX9wy!{``LyF9vJ|~j zil19;i4jz`GqC`@(}*SB@8CU0H1=>?8^(Zu!Qab%JaoGeZ1z+h!a<=Sk&VN<_(@R& z2JNyVyfhYC;P+ZlCM}{LYbR15E5F+Z3gw9y_Ykml_Hdk|vKF{yIPY1r3D^aDTBGA1 zRB%@Rk)CG2#=P|ixmbPU%%GVb6(^)q>?F)j-F|ZI@$%`&K+|-mr2{ETchlzbNrxQT zbJZ9Tq>8VxScROYH{fJ*eSol$C_Kh{hlm`HcAWWo$>Un>a|RV^xtwq&N|W*RxW}*a zt;5Gu2qhXP0_52ENztL|r#D)u5GK?XzYqUi+wu`%-HxZDSkEo-m+;|bW+rWMc@sm` zdHm%leAHP9Q{;s)wb~e?q@FV|Lm%FckuD>ow&CwHY?A7Zm)g7`MfF5jOp1byWf%|p zRcowchYsH0&za+Ed0xdTRd^ATfdPp(uPsw@`rYnVXkMxq$mx(LRbJJjP68m&RJn71 zs3_6e!N<*;k5u%>?|y(=Gt~#|Sy@?xiTnu-$TnC-hp@7XSD3 zR_F^S6W4Z;-|Zx&#pUV<4e2-^WTRiLeD#%UC2|cZ50|}&&-Gs5ER}#&pm1L+r?5i* zVh5GyOD-l+YMLG7cFLAS+cLp5jDTiUWd(-AG8CAkz0>d&eYAI2shK*8uik9lfnm|* z#M+h7XY`5>8$DkdokX%?2bpo*?N^L$_0va!m`3wtMg}YK6E&I7g@G!qHn7!QwP;JmsicZ#On>qrfh4pv5H-hu)EW1DxEss6|%xC^-Xg<% zh133s3tEWEHu8~4toE^gz^T~er#PR6TDhW!rRRxNNVri$>An@V2lv5HSpGge85ssy zx%>nPVjk*GDdL2>{=yuVF91f-v@gK&Mhe5C+*{V!M0QsTOHBPx9^P{49@P@QNg9W} zdHxD%!v+2si`6t(`pZbgsp0V=aBX$XmoV?*=&Q1>dS_W#MgC2oV@Olt%Znk)+cYRu zL(_%*uy5U-p<2_p2np`H$dmosY{aC}x*xgd z(jlP13}YKa5;%nrTnof5X86m`;^ah1&EKI#L#!lkuo~1fmX^AR)NU@b4bgCy;J?@x zQ3ooiS;1E4ES|&=Tdw_frFr7t2(OT>4Z=>vs>GfCKt%4Vy9t;qw zRkp^R0D^K?mL{e3RO>$VR@5sChqT{gPU&8!WE>!D&(m)2FMAW=d2`fXD^5Z60Mx^E zZu+^o2iC(l2bScv@znJT(98_pg1E!iAbo3~EI^#@4Pot~y#9EMEn>RWdj?R3SA%vN zzIsi*It%CnOUm-=&&$(SQvziN)D{VB?}q)iz23yGvVnf=#IhjLRUhTpQ7-(41(^MW zEtO`j37Cs;6R=GDE^)EBgMSzY^luHCC}6lrH@?rmrTg%;>=M?Qr=6jBa7*usaO7*( zHUDfC?$7`U7(!v?<|{XUw7dBR9BAT>ej(J?7g`?;pj`B^jYw}iA_c?83 zouqWGoP48#o5MhBKvftJ=Bd^}HAn(iESk&gN^QlT~9G+=vl# z<|n1ICg*)B%C=+)P!$ELn8~5pp!n=djQK6mFU|@AAjySe0KbAHW*!~^VH>3W488XG z2{UryB-dF+9delfA9Gnz;v1*O0%--!D@?(2Dkp$8^$xT+g(0IJtQ);H^#~@?7d;Dl zZD*9{Igt#%Jdb-IM2P@Rz^8O4HDXgB^KZnAgTR00_Yg1e2de1N%E_EvM;M=rkJN!? zSx4UXU}k2ReWeBcxHB5&m}A>z&@qs@U6@Jx=njZQtf=H*#o)%~Y4KIc@I@5X1$NMyav&tQUBK(Y~h@dW1EFW#nW-!tIYXC4wegP}3m|`*~r5y|DT5&ol|J zMBHmlr}*$Tr&*wWDBjpr^M-_w)F#U5XrqS*zx0JIdGtD8Nfipnh8HQ&!HKrV)Ufzq zFF{qMeVDU;^#0JuJf_mF6T>-5_6Gk`{f!z6q2QrqJKtRrRJ;>KanZ@X%8YtDF}!a}%g<0mCRi zMf%T*pKfovwlZreH5HvXFu3AyoucRXKsF_V0U`(61KvrSjyKD-!}i$pqcjOX&{?D4 zFqrFYWqv!03jbs4XwDH`%+I;dYo67E$cCRXoHb|VpV=iC*zc2eExZ7-(@SMK3Qj3C_;NQz2>B(?_5{}1u*O_u5!!- z(z$k355UFc(gWFC^l*H-3lQb_L^1WFT3T}2cUT}-&9L_;X{$ebeX4O?!kQ1KR!f3c zJyk|fW`ta&C)t>EhjxNqrhp%0PE`G6X)U)A8c-TAyf9p+XlJPZb86ZQDcM=9kK_00 zCy$dqBhB@I_YGwKpS7DL)I0GTQqjFkCnamlfI|6%L16e{XTssjs9v=WargxR!0Hnx zbdy+KD#xboE!4c3{%}n4mM_)`uqKP+M*VjOfIX}Wz@KbHs&I_pm3gM6ALp2ySem%- z;sCa^X})b;YZu;iEmNR}&i}YfRXy3fTc3Ev2s)vQYsLS&EWsXJ@k6v>P7Sew=pZ__7KUq$V_P0FOk?R2s@e`z# zF?g!ybWD;Q{$}8;y~@AzqL5((5S6~=Vqbc_^kSDha1F--cpsSKM#c5df(yH{Q#jnh z3(ai`8AcDpG{33=GHX?VK z44!*anwLy><>BPA@35$~RNZzp!1QJ&ZM zWl4#0nPr7+Dt}=bTEJI_s~X>eUr@@c{2VFA@%<%rMF^8`gb>Op>E0sLDO|~iIeJMU z@c*Rf)mBgbsR|BM)zl-&!INLH3_}}?PwlVQ^O##KW@iB9CT`Z^OZqXJqBWL}!eKw|?S>uKe6B6!pnI*PNGl@r0ox9@-&yEt5 zO9#YElz0Atq{6<)|F*uH&yyjpy021IC@6s+Xpz#(>~An1VR=nc8h`C7Hs}VvG%iYK zA-nMij?^pP3sjCa-6<_y3`6oxe5ZBRE+&6C)xp8DXajGpL&Gz1-=Zm9flfDA!mEsK zYb$F9rwTQa)aMp5A4gd{{wcJ~@)(#}yvI1oZRyKnBSfE$V4`1ih|D}ApH=Q8($zaj z!0go98*((LL`VG7mt5q>#+jCEc(KX+$Nk%wbW}fIs!M%OB)KI^Eo9Z1@7=$bLY{AH z1|{qOC{w;e+HDgN3N{$WoFPqN%pp1ojpc6#0|i+qYc#XJfnEfTYC3(>L_ijuo5-)> z@nY&SI$Hg{X?7KKY_u#%{DiW2)m;$+cO?D-fmL(*#YPK@vbL1t_BD?e!TVOX^VZCU zx&rk_?Njmh&oikI+UQ+$5};I$1XIzb1(8np;+qgFDN-?adyn@i`g{-kz5c?~zugCy zVJR;swfC_de{P^a4aeu(K}jnL#T80e=JWPDmxTAiZP4F^%OxS?;G98C0K-C!!)E^5 z*eiH>u1h`{X}K7)-dB#Wn$(S~WGi1V-6;|Dov@ z^H3=}ga@&?R~H-EaR-f`FmjiYv2Qz&kEF{fstHz}=8 z-vX;nxS+U5otT60u_~{CZ){Bjuz?AoA*k<3WYJRD`sLwwQQ5FD(3(FA7oFKBB1m6! zhGq*eZ;?3Q$s?AeAni1UoZhSrYDn+U2e7OsLyR`zMC}(*%pJPBI$Kn91MfHMp&PqY+3Nks^#zBl0Co_?M`|O`8Sr{HiNGsY&Isc# zflg2p*nC^G#jZsR%*&x{+3}?~A%2PkvXL#uN^O23sm2s`p2kU~KrbzeeHFi6s3^_O zYxtQY>BAiEhanzCt#tL3Rf=~q;KZ_vr&S$*yDZje>h0`2s%N}uSl&})tAd>uDG+$J3e%j;OWRg6n*)F!f{!Df& zu%2qT?yYEnOw8=e#Hz{k^3c5fiFVV)fF$DD%#H*pn6eZ@ko*M`tcps9gp-59nnwsR zd%Nlccp1@)TjuS{;@zMd;TzwIf-;@0wWn)Po44fCS@LV}fJ z?3|;Wv9>P;1pTvHwGzXrUGJ5p3V809X|gc)eKRO$u1oKgqLW2PP0Us@6#M`|C+aMG z7Ym9cKnSCQL*p-PdAagp=F4Oprh9-gp_$qFy@f9DZQjD#iA4n(omvzD0VWACvRZuc zJJi^iS%~qw;kacA+#Mbb)aLR;w`*GbLIMoo3(Uw*a&O%&m-#vau;JqRO=sWSls>3c z9+M8j!C(_M6|wm>HSxBkD8Y}*AJ>LZ!4I?E2zFT2QPcK6*(d*YtNj|9n4#N$ds z(=k$*+fDef&|oSjeRcxW2mx9a3APKKB#|H!jx(p?>de|IE>j&ikR(0s6PKcr3k$N}`%0o@UR z#f>BvxP{{t!q6tZv;bXdeOp5pgDOL|4kzZ4AL%IDcUe$QH}Sh4 zv;ERZxmDSdh2;NTv@Ys`*`y+!1Qu%Wqvv{Urm=1$7J zF{s75{RVELpgr~*6CFrA;%7-R5)E6cGxnSA5KlyUs&z2m9Uy8OBjZCzg)*X7)Y2F3 zYw-+18NIUArGO4cIR(sFK4rq51VzNvgceFmKb8@r-iq1$S5cLIqvhe;Y?^+ACMUtY zplA5?h9{?o`zC22rT#Zbb~=qv9797IFJz?!*>_0X(L|5$X-A!O2s~sXUlCQ_p+(6S znBNI_3vbF5lV-f7WvLF>U9CKTJdPfi?e5*b3yo7{)(Z&83*+r|wJv>%sD97=Oj+5um7ykyc-{Rk~bjD0>u3mk4#5V79b*Ez7Nnnr#m zOTMK%dWi?rg)>Ond$}$+bN0| z%!s4 z-J;d~W;TQaTsFHJiM;b=vmv@wG_affF;o%zrx7?`nsiIqhgmbVIw`oc&^;F7j8*vUl6Nw z#$u>55~@J|xjwSd_?1}<=kesAIJ_uQ9F^z$LM+pM_oz^U1(cNWp`$9w8ilQatnG+p zRSV4>pn2~JQm=>1vR+J79Z<9=m6VK4n#jbd;|;eBT(Lxt)V>)2{=?i50)pD6Y!<@+ug|UK!q&C2BsoYyYZUbh;%ZjU4LPDI zwlJOqzReoVYqp5-ZdvLFmo`b+13Cxm%fJLqSDGU{&mBundc?B$sOfLm@CW*^A}Dj+ zDRKAd1rLaFLC+7nl9B;Ys9ffMw`*!*IKQR5mrguFpF{}^k?4mvJU@;&XC}rdfyL=R z-e*HpwgpO?mm+DcsjABHD<^c~Yf!7v(s?aYf0Z)w`~Fd(#?ap!0+Qy7_?1nYZ<>Ma zT`Brs5=l9E>1utWdYZXOHmoK~oL&%a&Rt!#0NeG=>yv((ZF{}W`K%NYzzn)~V#WYFUmTO~H^X@Kk)pNhc+rD;qY(^n@zahy2LRsuW0yE5MROB$_ zkHfdBgLShQk$xC`W*AVGNh%FM5M~)uY&y*-A`p%Qh47J{+Sc+(#l=p0al8uqOyKaP zGVG6P^@Hi0tB7zPWBZzU;Q0GZCeZ&93U120CZ$wissywNm=vNE&kf&Y0&B`~zr3^N zJc-FmzE8iX_j^Jcs*}MNjoHDprj-2vmpiH;PH;-D1$a5@9{WNnXAyM(QyWdLC1M7P z2CIWPUrJophrL0Z9uynuSL~%kc&#yr&{i0A@@6zBf*BK6UZT1CyP**;OMMlyr==*; z%GM5vG4^&<-mKZZ)D5afP^Y0fbJdJvuaxMRHuCP;16S75K?4=j><5yz%g~fVH33_yt`^wK}np^L2l$jIa;R|eCBvUctWobea3))xUW(O zNC5^WLeoTf%V=D*)6@|xf%oUJF54>otq`oPiVIo%>wr1>2YC zY-1e5nt6Ijon);Yk{FMJ>`9K_#eQ?|>%emW^&tl?;{omkGWFEE#soB&e75mv?RyL3 z)hr6HHCjd4j=(UEC~~=;i~wi*!%uzi*T&$m@LslpM!rwm~{=bZ8bDl9*B+}UaLHo3Nq+-tz< z$#i2=##7d0S0XOBb51#>VGDa>IH60cba`Je_jcJ8z|b@Uoe&&n6(oGiaYL`$PS*r6Vg%9-Hb`F^rEAFVGHNrqp z*tS-+khx3fH%Pj%F8ycNMCt3GHGj6+?0%F~$Y7PA2N8v7nx#QlP2Ozq4Res{k6nf| zXnsVh=8>8sM!l;8LTJARUcO6a^qj1~dXNarh6>;-2P63Ad_(t(30-cNAXMDSB(P)p zqH!fR&W5Om-VXi<^-Jl0_ioONjvxs_j z&D3@__M^>eN>6Xs+*~vNqh950>3An@Fq$AUOoej)nOSR2`L=u zav`7^+N_-)>NIsJ)e&3Q_@${q4P(Cka%!Et!J=Ku`zq*)}^ffYXy z7dmqMaI+i}$E<#2{8B@1|2;|ViCaD2>A*(_%s^dV;6#%9B@N+b!^y3K{AN~a1ij<~ zO|rcW%2wu3;0NM1_9TPZE~9D(uAh;IQK1i{UK5*s&;E_+-P<-VH%??4!8tpr$ zQnTcKbe>X<0|y1F$e~2?5e}BgjLi4M4yDnWQK;>x;$;k?xOXAb;ijsdQN0 z{lWj04<+c}<9qbmb=%jDxGNhxYDeiqeL$w$Nj&fg-`Zu?6NfpSaw) zxl*1dkkg(4A-fz=TIF0C;v~vwqxp$dhddBk8&kTaGG8$@K)a7j{;L#^V`9?Rv2&HN zT8v>hh9`6<v0%Y|w?s89CyvfQ`~N73q58B^gZzZp>cP2 z$?NQW_POWW`^H$KkzQl@oU7`qzp76D;III6wtoziKe!N&QlhCvHrjBnQ1j63(}LgH zm~H{5G0dwW$s^tvG)BlD&`I+LbkZ!GZRmc?KN0_ZB`%(1Ox?8Re}z`y{6tP4`Sd5) z^Dm?4Uq9I8)~`~Q3A|FfhW^iYf2~4F)4iQh9Qx-i_rKxldN1U=+unCn7=JM9|M@z2 zn04^)_|kvxGOjyPK6v;fvOtRb|NHfuRHKCdy{o@=5Q34|4M~@D$Ua~3S_|+t_kQ!Q z4O75`biE#RMc5cn6gPmC{~?*dSohYD_5+}o$e??K=a;qLFAPT<{A2%|=Xcf-9+XAlZjQj)MU`O(NALX9K4thaWZn>*6c9EUF+b}9N?6#5my}^ zo&q?pA1{mp3Y!K6Lhou6DM_K~COb2exNf7My4u;<{@1KE4yyZZk=jdL{! zS@yG`j=)x&GoK6MGB@M{GfHj-2<>ueL#KbH_QUbyE_9Mx2thc!V70t#z6-EW?}B7a z4LwhXr+bsYOC7I#B+!6mV&>&ksEU)Y9`9Brv5!E=B;JL0 z+jmzH6T)UUmN5i9pH;{=gu+_P1ElTKYCEQvHv${JZ0 zyl+2ja6XjK#}m4!=c!@^1vnddVaTZNA7aaD^o|K_-K&!ZfKH-=i=j;)X zT_+^0!YyT@gh9FfrQaJVctgklHyul5>7}tzhAwMbl=Eiv$~+z>S6i&CK@H7X*D4DS zbvlBbD~zV2#o(NC|A{yukv6_j>U&2WF)^-9d?2*SpnR^(hY8xRr7mo7u6;Z#`hlMy zs5m%EHF}tj16@La1d2YsNk&-Rj_lqw#y(wE+l-2x1a;)oR(2Kwi6Z%r_HeSlxqP8{ zZ#)jIXAr09C!Zjuk6f}_O4d+l3We!Rz6HN9@E)rq7SXdTexTvdLNVpT#FTsrSejHP zvtH^k<3}Z5DqxNxO=OzQ$(*@3_4#OVivzwfb$qXZKg)#yh@wR@Cy~EDwHnSrv9?K6 z)EEef)E@Z2QON2x^MY!lpJ;&CSV@T>kkG@CkhMXnGR>QT8$mCxfuSA&GoL?2GR4*u zaC+I`=d=F{d!&K0iA+xaeDj)WOLD?z)-o4r8Xc<~>g#Je+Tq!4E7zVAeE&>3V-q_)CanJJ8hO0sGg>OMjg$+@y_6rPrJ3hZRNm4Ly^T5ej zILwqlPuPrrAiBFQ!upVDrEl?%7T~t^^Afbrtkox|<@h98Wltp0`Py+L+z4j*DK^)A zVTo9+=>AW7vQG0K*87n*M-k>y-5u{B44u_!;A8R-Y^|#Z!QO;mgJFI&y)*9No=4c^ z=@d_Kf`_n5(xDylhc#kE4qI%kH1?62*Pktyq#d9@v)BA{ywqj%%GB-(Uj+g}cMQ{l z0A0800rb#~9|0El3mM?f=9(v|5?RZ^ePayUwI56lq3fM4m*sy=`1hKSIS^vN{!(_- zRY9~tb^#>W(DeBGKslCl6XGt=^Q}+=rIWvuTyG7993oq%NAo0p@!aN<;~W$c{iP`y z)A;GLq6~g%waq>$6xO18Kqa1y8DOY}e7w{xM_VGT6j9~SJz_N#GXz8hME(ovCfZbR zZC~Y&sgDL=70V!hZsC{-<{B+5B8?U=KS?XxE#A_=&W>-nQ~_*yB;W+c$6cH^+4%fek#9vxe_{~BBG3?!;wm6nr1`4^9N z@rQAH>J#yWrWVMNcbA}2qQqA)h6(syQa!4Ww}C?1pAt|8eeg3~P^wwwZN0x#7W?P%M<8+;IhZ&w0qGyg&(gs^DPBc`QFYK-P*Ev+8IgJd!LB%K#c3t_uCzV z@ArMA6`Q7wwIgF0xWN7;Ujq3Bc0sgWZt?~IaV(TH-&j55(Fj{ z8S(Y+gL0d_ebL^!HYF{SOlWvmy>xDdZ;qxcS4ti;WXZ-Zv93>sx(pS<-Oxv*gZ5i- z;Y;cC_^2BzT8T2))wQ8JO$%rac}zXus$giC79rXxuuI(je&s&@8HCJVqd|=HjNsPh z&ro_si2kWjf=MLH3VrQ}Y5+R~sxdl#fsr%Czmm8wA#A=$fzLW!vuwq2QT22X>4PSO zs@g#(ZqLY(86+WEpysXiHy1BC*BVBqzAHS(*zLEB=1Ltd|%{Rvp&P{%y z!28M}ZxYs817b;{CCY~S1|3pg5=p;OEJ;kq=ui#5rHVneg!U|LRpw9DDn+$N zj9p_FIrc&c{{Gd8KJ-r3-yY zTzXB1VnOaaVsy)c+_%iUP!UaYYk(ny@wj$f>?|87TgPQgl!ta5U;qoNPeb%N(RBiB z)-JZXJBtT=7@r`d=EL+xZt>z>BBUrF_4W^M2Q|<9OVCH__>TC?z*;asWg%)|r~Ekf zBxg<1NH?=CheGhef z;U%kE8>XnRD&|2v0vw!`1`oyu?MPSqgS}Z&0)1?2eC&k+XCBjv3k3I81|Y82;6(W0 zdooX-PjkXd3psmDr}I%Up5B^Ex$RzP?}usvcPT{VS5Cc8%`RTgcOadPzMx3Yn1<{6 za`?KYf|ou=w=*RR;EXT^H?mxV%JB@k(5Ggg#~HC;$u>LX-jlHa(~6FPSlP_s#f~L` zF@i@LvNU1JU4W9?Tg=)>Ql?hZo)b9mf{g47_>J23+_(vk3Hn;Dx-2=D(}B^QI=6Dz zj7$u&30_Y&guR9tpjSwRbA8Ydz&iwoGSl3LCzdI5Iy(aHWEN$(_UxUO6u)8#LVI#c zETLKQ!(GD5+JA`_&BGT}AW(fTorSjq?ltZ|4-a=V=Er7(jv78;nZpfA|Kl-m|7~hH z|76{TDt*g?R76mTQ0ctuON(6RTe!0wMn+>pz+#gc^8K2n??Y$oAOnQdt@LjaFsGSBXsp#zyt#5 z6$ws2#r16At#PF&^$NDUI`J89w1ph+;H5O5Afm*BF{C>iUf{BiMpi4b4Y#>&AAmsx zP=(iGO-m(w4LN$tRAZAYb)=&CTnAgFQ`B0q&TTBG!!aqht&2l<>k_EypV(3juG zA^E&qc%ytj3`-z-TrWEVZ}>oV8Bll|zBntrM!QP}YEp-V$@X59JT8s>Fvh8g{Arq! z$VtW*r$Wwp@B9fr!Jj8hNH2t9nK=20+?TU3OIjb4*r}XZjM$2+W?6Wur4k)#J+8i! ztd^S30M~2i^#bvJcJ*94$sk5-J00XPIimz#AA)o+uTiQ#Ibjg49B(ql$)>lOx(mO< zL3PrYno*}7#VZmC(KvGgh?lzg<$<$XO~>l-Wo^FFZtsg%RkNrNjIfq6SCK#ji9(7@ zKfM@%kqaiwKZIU&uAq6Qm{avqn|ugik&NL z=2IId;kR%7vFX`fg6%SVqK#ds#u<%Xn`yOl`k_a1&f`4BZ@4B_ZR}AVtl0JfsKhHS z)js=t_<;-8BQjbA&Kf=BJ%5J$=J?2oQKz@N5yXc+cff);kq96Y)m+S7g^!X19naSG zwp@~iq|sgk9qKwq-?6LYnIM_-*aV-QPD~@x6nga~N021*d)gtT-<3y5W{+5#vil7O zee*NG#R`WA+$HXoZt37hn#;zJbY$m89$vQ^SujPc6fZD5Cb?AXD@ZySQn?|zPmsZ^ zp_cXiajBR$(NX8_oYK3Ss6<|1wz+Ztxan9Mxg%g$zdEAkXXEXbs4=H@CIbX4hpDiY zstV;b-BHTCQJ%*naQewQ3uU|iqLN7!uZnH}3T#%M#FBhj>>$UUwe{Qtyl3ps^??tp)DyHX6{S`} zah7vkQ9mpJi_N8`gw|^f8iIJsu|w9n>*e+9L&Jh|etsB43N)e-Q>GKr1L)xNVwGfw z6@rlwHjK@2Uuws_gfoGVch$&=6hINwG>U13Q()l268D1G>uOGftd-i6Xg5?S<$XiR zLl3Oe*Lv20V1RoFrU7-R0VpOw;htokFND#2N``Do-+vBBu;gTl!)>2uvx}i+pv>oT zEL9s_4&!MX4G5hV!F~twdK!XL>Qal`f!1z|WE-}7+qxEv+}o_{c5bf9VhZ2EAi6q0 z^Y*FZ0$Y)_NAiknUxg-?9!Dmarb) zwAC1Kfr{lgEl+aW$Xwv7l{nSoFZ&gPuQdybJhr7-nFMD%Z9-yx^)(r}#nI~T8nmt8 zqeU!+Z3(Ks#I29m!egx!(Ie=iF;jMN~5i@#|0e?U4Ft z>q?xr&shTQ%6z_KuOOq#8V_l7Fr1Ph%SiCeI&A0IjtA)-VAyZ|6v@Qku&LVohnS$P z3&nvKK;M`g?jsLD=FvI8O*zNRt0v?;5V&0z0j}Kx`liy2X0v1IpYyfQjXW`?UsRnxma zGxv!R*=PveluXAQeUW1oafT{Tc7uv@nE?ZI%$1Z)hn#nG5k--z?bTuN%h)211BUu! z4*V*5;5`nZ^CU1)72Z`&5}e6G!0T0;;93I2s4@+K0H4ed`4|ci{Mm|@+1%V{R}*u_ zr6Jkjs8Pqw&%2;zx=M1v-TNu;LsyZs1Vvl(`PW;A^G#g{MqEF^oMaoI0kxPN8+WXz zcJ8qT5bb!<0mTGEkuWR@ikzrbe8L2rIEc}u)?|buFaX1*CzR@F5Vh$^EP}jcM|Czm zrcNXgAB3<~YIy+2;CPCZEMM~aSq0@mi=fl+Q1cPaj1W&FNYEzDRK*TC&%_YK${MzR z{ou5{?Z2IoTZw6FR-B{ieKE%h3W?I{AgLI-Fu)(8v9nUxF&1Dq(*jhRR}Mx`ld_(a z2}0FF${#&9K4lC*m7jUMgU>+y{Lu-w`M>|>T2(AfA0=L z`&Vj+6=KhD)6R$4fArnn`k3I?!^~b2CF-0J)%nBliP6n-UZcAOqeqk3VbShw-p7Kr zAAk7h-=vwlJSo98nQScS(A3Nu^NHI9^wL6Xlx9_}SwxVrd!;IY~X} zA+vwSKsvD@^tjWF`ZKr_TjEvdFA!KK191acptkG&4V2B0Aa+$(O7ynHU(oqHKo+vb zoyV>Z_#a~4O%6p5B*prNa}z`UGIM;aB|W*6GkN&x@qK#HAeI86T0OYhwNK{%-sp35 zdw#HaNUXA4T#^s7<))$?V#|YNGm529@R+?s3L#6L{W+kMz`3T|N~lNDldhfiBknRk zSlLg?qe~Nkm(nL;T89T7h}XDAq$OMQHAAU;!A)JBryg4S*S)$gLVVs5f2DanJ$pTkz1?{7 zxhhu}8r1+55OzM3W32h3fpl;N7Q>vUjEo{h95!~KKaLD`SgGg980JJx6Xwgx-|-Gz z;WyvQ*e#0X#4)3WOK>76OuuO^o1Fv~h159RziijzZ*vj~~KD1k6DPuj4q`jzsCo29M^wEFi_X@hp# zFCet8w`v7`nRPc;E<$Go;o$ti)4*5uU56_bzt3)df#Tr~xfV-vdyBBp=eygU+4TN* zD_&+!DpxK4iqEVPK?hy&2Zg$HKJ4b*9FBTMj*AfFAra6FxJJ|m$aF^6I&i9Sqw-5- z-y2%=%@n?6{+J4_LI9E7>U}hxe6IL}M)&L2hBOVfrX>W<1SCd}XD4dWu3pyH zB|bATPhK3Bkx(l}5uUhGQR_{9m$)Vxly9#JkwJC3=NQ!e{qq(lYQe-bq4s7T`Z`ws zUqW7o94)<~3Zb|WFxij7M`WKxb9H>u`96^ah0Bu#dTnZilyr0Kkm`LAQz1Xy!cnb{ zrU^5Q$vo(n`cxB+s3q;$eE!Kl8~||}$p*#4T&O=@!J|I*oP^{M@qf7_q%^Kj`DzG7 z`D5HV-1b)2&w%K5)(@0UmoJGmRkj-s!>F4`|>_&deo{dqv^1RP+q0s(+-0I4FWto zt|HtMPofj)11MOTL}ASMooa><;KOMT&G!lfqlWy6ImUE207-Fq{Q|{2U3a!`Ae~8$k zigD0#2?#v$izl=v(-%wO={eB2%Qc(fX8=xs1cNgEC(gJ^4n+-gu#&dEcH<~chu%4+ z8KqtrC_zfsEi`^~`AgDEe{E0&fuy87)R8gn>bnf3n>z}W#qy3QCL~Op-q1uH4j6d# z{xS+9bR7#Z*imrU^Acu=+P1#8FB+*4f+-5dkG>K93F@?LkJNIUa0;@p@>8(^Fv6)( zUyEPizu>l(@FhpdLc!(j<-9|bMV=s4xA}|JYWdosasRwtXhdEV#1wWzLA8ll8gJy1 z|II3l@*VeFmDmd{=hbfkyUktp*dBHH;Z%e6h6bty5iVS<%>EeHvR@w}rCv!S3F5Ae zpOW)c10-@R?cUUqkd#2`BD1Wd*sQF^sbeB#PvT)WB})w4qPey8S?x`MM)|bOq;_Ha znd)Y~tJ#P35$;9NBXPnGrB0U(c5Lg0Agbx+5M#VgTmDWRE+o|qOUo5gAQEmTQVL$p zLYb9_!cV}AVk-BkqO6B{ReCIA4z)*>(WLx_8^I+V;$7fY!|y?C?N$sXw>)`!6giL# zKdA`UcC;$6lBnn*SU(U#*%UZUzEUlNta$YWKFt6fZ$|-qK@VD~(dpov61DuLWL?u{ z4G$({sH`ace7Khgi3Z^Pbbx(gtgV6ntd?AQ9D@~~wjYZ2VCESI_$s)r4-yT`CmX=y z+rIt4N)Bwxn|t~*u{Sccnx~s{IWX}i`qLKO0t??zgvL|DWEU0Hk z_712$czX9yNXK5b>)3@JG_D~gsZ$;W!#hcbnHV#% zBTkn};n|e3Je1G+6|!goIP?tCj=&*gN|3^ENBXHxnYNhuH|D79uh=TkMdUYTC+boW zkQ{i*A&R5fkF5==2j5;U*2XjwH4Y8e!wHcDsUWhWEiAd^XnJU~eC;C^iw+;6xf6pN z+%*JV-U0s-@X`ra0)kc^iNmlADpPUruVfG3%Qt4J6CQD1b6`WhS@6KQvz4BEHSVxt z`Sozk2dXlr0VYFE7yM-Fc8RLf!PuZ z#+F1@G1)ShV;%UsdSk0{o}B#X>YI#(Kk{CprN(7O!8ip|wVm{rw|mLX7c-uBnOp(Z zd$4YY8ne8ufuAJlWiKo>8|(B|y55rA+~tBjed3DDhA!Qhx>X@D=#Fn>$wr!__TcD{ zs%ENBSO!NDEoj?>tjyP^HYZ5GKIk@puA>pPD|HT>hs*f{sd>j3EF0hRR@+?FZKKQ7@Kw6>I#tQCWAed-lO>4y|0%hGH|`b8C#`6l=Y5{6mhO~x+{ zaogV$iZHrNC@BXTj|fXw`bcQa+Gq9$`*2}8!aX3lDTm#B$d70nI&q-cmjPT|k5wGT zfjuMjQ3!WfhEAB&ej-cpWC5KB=fb(KeVjqHRA&xWKfU2VCJKpkv}1>iWsK6+QhIv! ztxydI)KQjs8UBs8OtpK3k~+YBd1fy}EO|Q!Vc6xWcS$l@G6zB0MQRv357o3b;U_Mw zyi%`_wbfAqF*pgyTyzBox98?^YBvTBo?NHd^C5UKivabM@o)PQa(h5nEd^k;N4Oag z!EdM9rQz0u`O$!~>xmA86FN|UHh>r8EV{j!k@~fVhe>~~=u2GsOjDY6o8ri2@nvIx zf#FP}(@Tdspz?HbJSmR^&f_6qgq6{xx!WG9Ayw)g2YITeQGTq`*iqC>6_EM;=GD~3 zj}pkR>ZJe^eG#8(kKk&%JZYqvn)tt{?lWHUitA7IP<> z{!VYcGn<16Cqt3zER25Q+W%~<57N7&3DjN*hP-t}uH{{ubO0Fhn-}hG4IcM#3juJT zmG>Xx<&PfVK^_wY9beLppcM>2FW@sofAPQk6o?79E=~1;A9RCY669D)B;_}tPQcjO z#FuJ%77JSs)1G$dK0yVMubMz%33GVzLYir4^r}?U&c68^H3LaZ-o`8rpSiJ5)wT~U zsZ4h2s7t(v0@vQu&p#?xb=x54Ny*$4l=aNa{*i>Ph{yRbS#_AYE1S@ka!lVOjS7}s zQy1K!r)V)BCxB0n>|81*gijX;F#F>uRWa48wpHjg1bq`tThTshe)$du%hYcL$MlXD zcy+OM3X#jQSL<;sfm*%-hMB`C-7Aa$!{v`dO@LaK`lj8vANr-OWkSWp?h(jbE*2Wb zHSE0|Z6)Q^o!y#x{<`kMxC$sR!*i_qB&9s^Mq1u9XJf12 z;Ck4^N%f^kki3tVnd}Ubg_X_y(LY7&*0A-Pgrkh`KUx5p{&)}%ghP;vO&EWK+-xe~ z>6GEjM75j?ZJsy-MYYEK+lV&HKfX@F5CJbxBDb`4xA`YGPvaEkBLCdgq9I7vz3r7v zr6%mXaTFv;`RRGLaU0`i*8dAT2z z@lUe!66-_jqrc#tmRTV!DI?ijOgdU-v+SMTHiC~vYotn zZ)+!G?dt5Di~4!FD+O#g%Bf;gvo&?=zi{=2#u#xPjF)6n_q(7TiY+c20HO;*c58Dl zTeS+v0gKV7y|W^2YGQ5QzkIe4WgiI_yd_M7+Jj_fO97=!c?fNB680pf8g{lb{M7&r zOZOr)iCjX18T%Tw_!5>CaaCJF*9b-J@ zt`xsYA?e}d4`ChIi+dsemLR%>zaiq^AzUGJaNDJS)CI14iZMY$yuh!*d6s@zfe(s) z-SI&be`QW*?IiNb(E}-*I$MDMu#x`~59&cy8@2cg`ux}b|3o)7AkmGdjq)Uf|0{p| zv#sq6tN+k1*3x`Qgl3wA(>GPH1ekk3$keK2W4#~EL zNEoH!5BdHGyd<2vC9}#9_XE}cD~Fn1^p937{j~TXLLp_kg+Eoq3D0%f0*zum0j^xhzNnL_>YbhfLIgUWz`@oOfw19)S%?l zg*x8x^xjPiBqKbSpZ|Xx=^kzsmwrpd?}#X;Lbfn6?aC_6A+B#;M}H6W{eJ1X%SwZq zzDyb+9M5yg&`b_`T3$PR_a~tk(q4|g>mmO2cfrfCosd}c_Ai@dzH-L3XqKQjEbqEM z!>W$E_vK##R0wO1dXwB(@|}u{+7{V_NgHG;6-@39Xhq8OD$#!wg&oqu7*MBX#j{NA=9S|C&CqJL z&>(9-f0x)K-bV>>B)g+Y!^u{$MC2;Te+Z7mg+lF#=NO9ep0F>K*i(w?7CB{z+P*=I zfmM?yfnAl-)%laMyaR3f`3sMhbu!E>-Vig61No2DB%_ZHDMh(h`F|x-Z!7Iiz--W2 zsoU41<}px0lT@)&T|sqvZup4yaBhg|m=-gA!9!o5r~(?Pdr1it8m~(ylUoUKqxKrkav??SMk+{V z53{LgsgvCJX_B!T=M=Vep^>m!&^V|1;!l-iYOmBw$kV8a77ULV`4889a#%oxe7jt- zHb~a7aVRQ(XVV%Y1t^%1u|)OAD8c%}e9gA&=E?NaG;`nYG_q^a`01J;ZU(tW%NpszU$=uhX5u_uQsMBX)XToZ0CqKg2PRnBU>-|c>8)%lA{YaRO#vh0IG`OpH2(mhle1Y%7 z0+oIy`%B{i`(9jzbR{*ToenrU5J^Y-yb%WmSbnF0Rz7^KEO`p)UaAnM0?~O{D@m@F z(H3lm53`DE6NMzkL&5-wqCsH4e_PwoA<8?=5{B8;Zq+tQGIl$;1Z#qNfVSt~E@%E$ zrGuxBuPeuZIof5D)UdB1m~ZVvr-SMl>0qk%ahJFn)8b&Yb+syW*Jfg7<|LWj)+noW z=(>8LV~i`6SY?vTcwwl`8RO}-Wz-yWBKg+K)Jc*16g>&fZ27$#CID4y-}e&iPRGEY zJJ6lasmR29oo9ityZ`v!v;V-_)sy==>j{;(Ho$-e1-n~()!ok&yaee$sVpq>rTrU? z%chM8P}+iCy1kV7?Y+9k-YQHb0u-sD6uHmmJ3j#0&c^EE1ez<#bE&sDr+MHh9(Y-pPJH zFlC}N@2UM9^#ylUb#dtU4mhFj1T)21S*$k9(M1L;&e?U;ouz=acHW&uC&kS7pcKvj z%y+&`JET&F?esA5o!CQ3RrXjctit+7B;&T0m(Vnx_%y=}`-&Se_hl@j8+>pjJ4kJ9 zHeSQO-OWZL1By3KclVRgzw~i+vLJ$IWT1MkOlVfYV(=(H@Ra})SLZIPOTCv3maU36 z*Q7$npNtgq^hl6(?oD>os8&X*gF_YX1Hac`#ik8M68E$?MN^(j&-Aqu2llbjCJGLz zAE^~2#_5~_5Y5s6c~mjPW3+YNv;d>enYIbt1>O71B^Jl0OxONnS}XJxtv2ITz^4x961PBiWrX3^%bFj05#HO6`Ca-dCImex0J&-(aZ=fb%}~c&@(d*eV(PF`oJcDd`Q8h(GM($e<=GNXo!# zvc|P(8(ub>nuYlzhq;=`eUU=*)kk1|O_fY84N(={(ei9c9W8)CBArH!pn|wNSH+?k zuh9q5PI$&A&E2WOaTqBjT>F~WT-C%oVOCGM){R!$k7F)9X;61k4<ILm;^8P#`A>l|C9^4)xl3~jiUao23;<>zSMsr{^Z4*?{kGy&iRPq`vA%``|<8M z6~^YCB1*#zlU4#6vS1L0m{1L1ED^e#d6y{YMJ)+ZJO-4Cu|U1uSdWydshhdH2~fPW zAIu>s&Tt83?Yv!bN;;mvHqne`N%`TW(qO z!={A7_SQ!N!%JRF)@N?TX)4Nk@-g$6rOT(o>Z?h4G)~oY06;A!uBn(ib_D^qZxQs| z-ml#@125h$8#(D86CxSszv*Y~KiR2?Sc7Z}vb4;?G6K#ft}7b63A$%6)+_oZBe#XH zT%VH0;3E~GF?gY3jAE7}AEDOQ{~m*oVfgn#jtY$n#06qR(7EIOedhoE#NZ7TjrU2vOfp>pNsbgL@>YWlNQCRi0R|>{Q0=uYSdZ5{`~?&SJc{L zmmW?FxN+?F`u_=nU2oZ-(M}`@l@!e+C(Q|Fv{?FnnO? zJK0Kv5F_#D|zkhgZuo4DoI_2NJ$B@2EXs}qUcGd&jV{JrnD7W3|TeDr>7uhX9% z(6y6CylT}!Bz{#rQ8^J6c~|gzo%?>#%n#iq;E>aQ&&(g2;z2y!T5Z)57WtF&YCJ=E6HavS~_MZTXh3lQQuTFczDuunyM0R~p1HZ0& z1@H!T`M^PL-Kh#Va2eQ+WcIt6e{(OWShV;dvFpqk@~oKkQTV@(-gjdV-5;3Ui>%(Q zxb#m1waZ>TSK|WjR!_a<|LdXqGiY?7fl}bkr##}AH zpPxY?06&0iRcTmr!6m+izOib%%1Tkn~w=Kd3qBjgN z@IwGZKk951M4sdjJnK%~?J<9H$>pZopS+2_3s&8ZA78a0sY{BWqi?{Eos;C4W?$~! z5nr=LB)}(i4sj#y;bdRy`{BYP_^BnFK(OVstPWIuR|l$?*%AYLQ|f-rStTm*?f|Qkw&%OtTHt&Eby8VtRDVyMc(6;@1}PQ>y!$+4Wul>5 z z--Gzsb)V$?d|YFj<%&OZ)V^kT1?;!MdsO#=r#_=i`P2lkVfrbnY`!USM)v`Pw!_SUv!$97joMHVqy}+KG zjjAZv-)Qi~L&U~?-d6)WA{{1ztUiHrjO<^5je)`<#nD0X?qyCC-L-4(=FqTaM~=j= zqOay^#pQ>Vi!V&zY4OBXDLdM17RGvMONYj?)z;q=KELc2rS(#wo*@lWNss7mRzrO) z<>-*oNFzLR*DwEabnP%9Bf zep&Cs*L6N8YGYtV?|CB58;$Itp}5Zvz8~#39*L!TZ|@Obc4Dp~dY=G?hNmemYvowM zV$QY^*c$e`bi|4EMAd{Mp;WTTLJl#KL;c}5I0kEBj{v9OuG#nd0sVtZ)=?1$n);z1@8QJ8;N^9&5RAO2lY}YFt^}P zen^)&Wf@7n>%s2p0IeAcVo1LroEdQwt9);zd)>-JnJY};!{15wDEb<~m2>8w`$ois zeg>a-pIdM4z=<$h5TaFiev&w@H>Fms4t_x7Q-P#MrBIr>qyZ8`Q zzd50qYomYJV)#;oPJzJEop`JgWdcj>#eI2^J{wY<$MKs9#j1=v*2B=rtoxD$(F0>S z=ks^!wcgU7G_W_%Z@+eqL(`sP3flgDrqGu*RF*g5x8DYNJa$%;HjF3Jzi^iu;0V-K z&o!vFJwT}(nRVA6IIp?&pkW!6LT%dA6xn~AJ}8!%AsnyMYRk_Qn`x86@5{OSh779# zw|uIOizl=)0KQANd#!gmR(&B|Ir4m`1YUI#fV^9>1)Q7V-g0@jIC4833LeKUQYGE=* zl*hu68~~2Q0F@*9FnbzUY-!FdLm!6Rgj$^kO6n<(G-P>|Gv1^<% zWg$VuEBlVo7PZd$(K{pxAB{2bdn#_=(hoFR+!i@0D_NLKiJK9$FxWrZ%)h>>`}0ORY>&D(S0Ut_XKBep6L z9A7z8E0AaRP_s#Tnszv&vDlo$rcfW zMwF#lr}}j7!7e!n-if0M4C?+TBTXY zjXHuPmTbbz87-Ur*nlE+G!r%0K=fc4disw*1sRlI+Q5Osl|4(ZDRl8UIMy+bosO`Ymn2@S{Bm z+7+V^13PFXbcNVH&D^L?PBjU$(O?$fXdi03M zIH_e}rwARB%eUZ!_NB~Qu;*rayWhkKz;(IML5MJ7RFNtzNC_YbE%d6OfQsPV9`QZrzV~0a z`5{m8JjtH2XJ)PSneU!8)09uP+Pt?I{~GSB1SFRF?CBZHhorO`*&uIh{3}BwX3h4Y z`gGLE0%F}LEx-NNH@`9%yov)g8%>lIQ6K>C1?NYjx zKHb;4-i8LP(d^jvYXa&&trU>0Qx!v@;HNZ5tq)`2KbKnaseW<8+JaNH#YXAHL@mzd ze2)1tbGwjT!LNhG)ywyOb_e0gI6gIOUO(~^w!<2bODZB_$< z80*WogO%0;`xi`6^=;+fO*RhjYaoq&-oEI)|T)M5UV%lyL%wu&~z(w@d71>;P zJJ-6on<}YniV+K1&fh=RsX5&byKcOPZe|Y))`CT^)yg9A;nrUqS}XAq#h?9ylB_X- zRNd63QB_x;ni^~Yrzfal);;&|Za3A`SBX?aVqgcE56+HOap0qBAie0jmJf>#h_p)} zGFt@*tsO3`VemFd1pHW|k#_0vyq}<>5CAz92Xv=Q-{$|qoR9j)N@B;*XW+Po)He-s zK#oyHk^r8e zQD6|d$zeRIu=Rl{*;=h{W*GGKS-or@Kh%CbsqXmfXsUEPL|*y_jq%;8(HqJIew7Db za#V0S`HzJ{<}=FPC_=0;&3xD70GygS=snl)R zaQgRFP{!UKpCvF~j&7 zR=x}75bpHoyGwHSe2XIuAp*(iLMQG6bd_pR9Wv9R4g<9LSSqvo+vrHIZC3#&EwmRz zfD$Ae<;HY~$q=SRW~eHfVxTa$Qp*0%H6^4amybB|0nDj#b!h=^RY&f9RRi@scVHyi zIOp_PI6-6cSc?nP^`uBCe2GV+X%JLKuDvikaM{^fB?t2G%V**3%A~#nOC~9{h<=&- z#f~l!T6Rka0UVq&jCmm4qJrg3?4a(dT?Hz|Xye=W5A8DsTM8<5{9~PwCGZQ!(05N+ zvBA(Z%2FmbBS8j}>vfnJ60@6MdPn;L8}IX_nlU|GsP2G&g#o5iy8DatXT=lvqAtID zfi}h|oQK_Bgv_j2zFN*~F0ANoxjyyVsyV%DGC^NV-@K@6C=x%J#eJLU*C1OV)vb6A z?w(=|cw^SMa_XU-qzUKCqG!p@rhw)^6*00IFsr%_+Z!PsCa=tL5=!b$cE#I9+5hq{ zw>q%%5v2Myrnuv4Lh_MZSE(-*+)F(@7la4U?p#pfjsGAy++zYYrY#QP-hkr>d7*DD z*f+IDTY6^qGHLcltU9FwwWRb2s4rZ!yr(*e{w8uzNd^AN;u~DQ0zY0p-UE)>#h!HM z1@ZmHCsEIMuJnK5-POxWX_0eup`{F|lva2-0YXaF@k2JL`h|KyVzPARW;S>YfJ{Y(che2${aH%VFexJn}v;V`MBq`-GA1y0ut>r&kRSHqhk<+dckzmKntg1 z`Y_4gsQi;ncYgP0XQraJ-elBRz9{1ONcd65g;KToF9DQK@JB7$^Lv5(VDIA}H9_ZB z`xlSZqA5fv63{BE{x3xP>vs=ZNdfFO(>Xl~P1gpK{I0Xl_=Wlv(c|3VfTKLobKb1l-)-Ex^wCg&aP<7Zi3PbFUo& z>A1e`r{$W(+? zVkWIC;Lu+N&K!}GX;ev{7IAszQE48kOZ@l|vW39N(2*gehMJQt^idA6lI4SoVsm%b zALO^8ZqR*gjuXj-*W33`o7oL|>*1RC>Ug{B+~2#&>4jbD&wJYf`__@ccs^hK!XvngC`Ka=;U3M}1nf?T2?@U0+ z#%wJ<(8FtstmiQm56NP#;c8Rm%xRx#BmIF+x4)x6-(5HFwmvqShNiuM8{{}jNmZXP zHTw@j=ZH9$A&SMakm`P1F?yc=H*R^x8d|G-LunjKr9Y=Coif6Bl*6KJT|B18G+XK_ zdwbMtx)2WGC+C|1@7B^1+ABc)KiCiL1C9NwL-wisZuE|CaH&xj9LC)?Iwim z5HBhd#!}?XNZW?hFQX^p$Dp&U2vK6K+JNfHl&|xT0Bk@1aLxA54dS46HRL6&tJ5SW zeAYZ&pf7l4AzLXWXcWYv7oazC`BIM)EqA5WErsg~ha9eN&P2~Chr*S_i^9=7BhOED z&6q~)3)Mg;{8vSPkQ%O%j%&m*{2lJs!qilqO(aU=TL{)15?*x3nQQO4bsw(U%zRD= zJVr*O4_T3q7w+|u^pUkyruC9$ig9#z;SNPvZM=Q;?qtxH%d<~S#(lJS|B_9`v`d7= zbeD)Td;06X<8Htf*}m!sO)Ig1`MU5 z!;bgWyZkS*TGFHjW z_Fhqj2ZSTJovMj}Qo~Djf$VSv&ARgUZVaCF8WpnI3NI25OLhq8HUgsH7<)>mjRP7L zNqM>oeTFOU8ipw=X>FZlEo1}A;u;{72Dyj(^^0Se*1oxghyui5sW!A%J2QNtjG?AS z?=V~qXLn!P_gRXIH&ala%25gK-Nm}bkqb8lg;ZLeua8A4RuPI7Bxb#RzHWF)07QC5 zYton=z_xyd=A%`HY;L^sqf_DER*7K00bcEP!Ikm%1AhDS5eEITDU$wU5sg%So2u#3 z<2yz_{iS}qiK>`+*%)!NX_~S+8FX3BbFzG!Ax}0=9AmViJE>1T61Z3?>HBDR-DSwi zXd}B&6VtHq6ng3rKXM#uJenfW7v2B2e}nt5^>rYqx{1kLQMx*t9MrV4bX; zPFjcCtKK$=)!vY0Kp!s7hAMy5r8K__6AM?L!n<~4Fg$y8`m$&!bW64d*{zWJmA>SE zErw<4g@Sk`B2zo59KIqE2)kx>0CO~|SAmA>I)O}=>Svk?B4l^IHOQXaL^U&judfJ5 zN%>dy7Pk$f$m*MIqQ~%Va&|ro+K}ZV+ThGbS2Y#)LHxCou!Kh|&Od*SFNh$e>{98y zL);c77Q}qg8>mzxN)>WgtMd~frd=UsSkf*%2&cS=Ss3M7)YC*eE^@jCFvXow5RtBh z%7y81NEQIhEV`a*;pc9G5Iu-k)l1||cI9@Zl|rVci3;SaaZVm-f9)^wW&NZ|bBfni zH{%JPT+$M6;DPztSop@e)0X=HP^eFGjoU(fIN_Z?VQ^Et*k*TL?}T(mtQ#3;x{KO2 zu&Mn7ow)SjaP3Ec7?N2+I@W_wpM%F{*1f)Xh=6yzsr;%W19K^%n)-(;U}#O1i-7tE zS2A9s=&9Q@dNm6-;Y5!KWMGiQ*y=tmh8Cn6?kEuSRXp1<2KMNByo(ShzDsSbL4)J3 zau?FRS&O`CtI^TMkiLyhR0?5}0N!ivcz)vLCdW9XPbKG}o*5(wzXmE%C_?G_zRP_`lcOP(R%6Pi_bhOUAQ zixfVx9^K~EW-s5^F(6V;3@D}-EsbsV)H|$Tc9Yj&l2gm}PYCUByXH>lKDri}VC~K2 zQ3y#?dKEg;ez&zFdQO^nO1ANO%~Eqfx`g=GD8fo>ZZh9&m^yyvo_D{;FibMw#E!Lz zFDJku&TVJe^GB3tpqos7^^db2*F0^-XLb2~yWd7CogPE3gP+2BM$cW4PEwE|-rTO{ z5pfg$o7ZY(ZsjJN6=Pup7(R}nn)phE1jbmtp=3`7o&yS@=)deGU8`t1nqxPJ@5K5~ z7sXM9)8zaj6QHNL3B?qeH3ye(K((J|!RD8&xc7Z6_MPYR6E?z6>4kwO!$UkFvc?PB zY+o}VR+BObE{(Kfb}NNhVFzMNg}MEog!T#mGxcd;YqmhpA4tj+LLz(k|jMz?6pe%n57RbdT4~h$YN;5blVF*Y$NTr!sGmY zhvHWen$*eIp(HH5a;7NVM50V$82K#%l`23D7yOgDeV z2mbiL5W+-p)Dh9nt_5ME@R+9=uj67Y3V5*W+i3Tl--Q$>6|KzjbN`V%##xaPNU#J! zsZG1Rf1O6F<`bqWEVv_Z?i#RZ75*k1k3Ie-99IX$Yh~L+%Ti+>2ku4j#z+dU7BAUko&nv8kHc-9?3OiX1q)*KY zo{dNC-m~a;;%wJW&Qh}Iq)_m`9XrMDiyG}gM0vK2iK6Kx7=crcEj&9`I0eDePBpc( zE$Qv*H4PvydimMSX7;!ptKKso>^FCwbw}&D>)?_*S_*XiVPNcIv?*Dq`?SmIrg-`; z2#FoV%tsq1=~BtF%rbuT7ht`5dTFerD`T|ER~Kh4JKjs%Xj-skC6Uejoa704&+db% zl;WN&Pt}xzMBxKmYVVwtP*4u5`rU?;l*6C)=v|t;D|Rm7KSMokPCM-MCtHJ#L1eDm zvxMySp`t%-cC?{v_q}xPC;vru`)*C?Lz{R$&f~DrJHHW8S|_2wEp@=T3`c>juaJtn z+C91LGvCS!dx;j!w}xir6iP)-9f0&$dEe5uSQfMQWFN%ppem$|z9oh!BCTc1PD-7! zh@CMDYWdhEw?gLdv9i}Xpg#88nV2?|i*q@g+S>;IcK#C%j`UM^pv1xr6CqYM!#qA= zXuo6zeM+VlQ&YS3i9JclMoUB0Y1R!e2{C?u$6o5jPR+6FLLSOC3=BoFhrKxV-^yJ# z@8WL-=n!{2vH|kL(#_M)>6H$lkc2hc!9(-;PfiL*eD8cW7|W1KkMtNt@H*;b`x9fU z46#ej`GFrxO=OYW@6k;{ec#Gg&wM@};9h$k@{jJmQb2=`R&Y9BVpnj1F0dVW1*8C{ zsqa7lU&C;{c2WQu0d#ti{E<&)#BqkPmm@-$u)gN_-9kHRx4rl~oS<&lmMB;&;n!5W z%YY2~T9JzdcoFr?>3+0dYvH6ejwMI-Y?&DgP$ovo`b|ELaRIdN_Vf)fc4+^Rq=Y_k zx8aN3@_G^@MRtT}|4-Kmzc2;C#~J4#jX`wepqu9!$q!NrsQh*B4pHEFDF^n0w@JJT zX?}vRuH=9*W+(ycm0?L4Ji;iwlmkB=Egj2PnOo;fgUrxn)=j+^ZQrumtnm|XHmgSR z^0K^TLrosS31Z&tC;z?mXNfm-ouMnsV2Zh@{H@-1i?UA>RH-X$+w3nLql^ojN`*(0 zv0PQsi0Nh4WNYhH+?q(6{k!Irv<7jgpI}HBW2!z=tWq6VD)+h&$0$kMUg4-FB0koQZhbj%0e=)aVs}evGt~5W-Nu>1LAC>lBUGGg5s^`yE!*(7W&L>Yvc8 zN=2Jx*UNE~MoMi9euaD7MI)5s=5MB}UzdaWVRw>-sWeqfhRB?(Vg-f3od2raFS zCo1KNdFuzuS4JMkVKU?8V;hoFcoP7~wuc3TcDBn(_LcpsIQM=)Ljj)rL`6^=BrdODNjVcA(xSW6Ul|I8 z!v=Vu!W_FYayvzw4oTQ#lKqc=0&riVmEaVn7i82@Q)W@p0xblkyA+s=2a&)&Bs*L} zy^W~U__?~o;Nk}|a~MS6Y8i3fHcLs4``(TxvORae69?9)IIzp@1!7w(MF%-=I_ypK?qx4o-7 zKLW1Df9lyXTzs^D?E&NLfajh8%{;9qga`d;rR#aBVImI-D(jxV`|JH*7STZb#%TTSUQzp*1}r4J7d){rJf+g^ijdAYspiG zdn;Z}UgJ$T2s<+7)O01fio^$924a@##ao2E3n`_GJxjr4iU>jWn}&WweT^WeCjNLy>}6yI!>nwxe0!fVw7AYloaz02`r(9BQvvW(x! zO6)&sM1)Y&$~XWQxTX-YBll0aHjGikRyZ94Pt#ik#3W7RNw$og?Yj-Ye*cF$W3Hwt zny%Y!VatmlF_5$iy}N2Mn`~nL{Q0-Dso)A|0680&SA+ipbpC|wT)ek@PG}6|AbS7J zgQ!^m%0=v@Da&!LjzUG=s(%7<;j@yr?=`m~|GSV*LbXWzQPc|^sy}2X^~!ttgi|(6 znoef)o&w%F=o?7(jhN=L`ooa#1GaT5)4u`i5vUMV;`PTD?p7}x0mW11&h)lVzOXeZ z7T4RLrt?UJ{=GCW=F=9U+RIQO;(sb4P4tpmB?DS6Pg$x6S=V=H0Mrig@ZG}AOQnyA zT-27N_%nzsEsNu#27ZYl%pf`38ovlYxc~Fpag%q(vRSssvfo5R<6|UqXSU<~p7A_U z*B&*5fLN}X=VMQmVvs3>?#qFUum&keRc{J>3SIMQp~ z%Fu4W+I`P}hM10jWER7()0yvTLBwSx9aV}yRnCtN+dnknqHtyaJ}i=t@S*2YiBf!-QS}X%GvEtv=w%9 zQD*r|fQakmyDr*#ZPn+~_`rE}68N`Nff0IIVbzNW#`gA(xXqM4AGNlb-x#ekMg`Y$#;?~jjXQ~(jjT4Sb<~$e8MbTZU8EmFh)ceAqLCXP{ zO(0RyKe}*~D4@*vW#B(;of~*(j64vs-ykNbrwY6h(%j|oJh{>r?8js$vVzmpo@M6t zjyB)BHwq+IXR?YBvnKE(mSQn&yfozKQ$bgw6qx?2QG2Yv$a8$Y^q)H9@YKA{q{wcX zxT|X}_u0_GH|7B?k07hRh^AS3l=F)#mfD98Djq68wnyyUj1Aqz*UeMP2WVow3YM`y zyES1I_3i_^D8!lH-=gq<2*0haELAfz0%dWKHmov7tuOQUY&FOQMpxpUE3J3_dzIk5 zV2B$h`|o8XlE?tiA@2^{;UfMQdj12WeRNs>E%N>SStjJy(x)89_FuZ$UtfD~c>NJ2 z|5MW93V^Hge_Zyx@aMY!)a^2|;WwK9_j}Z{Uvu`ro{~1xha1voa{^Vqqgi>m-dBt4-Ubi(2)L*OGNB<8Wz9MS? diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py deleted file mode 100644 index 0e1e0dc3967..00000000000 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss.py +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os - -try: - import kfp.deprecated as kfp - from kfp.deprecated import onprem -except ImportError: - import kfp - from kfp import onprem - -import tfx.orchestration.kubeflow.kubeflow_dag_runner as kubeflow_dag_runner -import tensorflow_model_analysis as tfma -from tfx import v1 as tfx - -# Define pipeline params used for pipeline execution. -# Path to the module file, should be a GCS path, -# or a module file baked in the docker image used by the pipeline. -_taxi_module_file_param = tfx.dsl.experimental.RuntimeParameter( - name='module-file', - default='/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/taxi_utils_native_keras.py', - ptype=str, -) - -# Path to the CSV data file, under which their should be a data.csv file. -_data_root = '/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/data/simple' - -# Path of pipeline root, should be a GCS path. -_pipeline_root = os.path.join('gs://{{kfp-default-bucket}}', 'tfx_taxi_simple', - kfp.dsl.RUN_ID_PLACEHOLDER) - -# Path that ML models are pushed, should be a GCS path. -_serving_model_dir = os.path.join('gs://your-bucket', 'serving_model', - 'tfx_taxi_simple') -_push_destination = tfx.dsl.experimental.RuntimeParameter( - name='push_destination', - default=json.dumps({'filesystem': { - 'base_directory': _serving_model_dir - }}), - ptype=str, -) - - -def _create_pipeline( - pipeline_root: str, - csv_input_location: str, - taxi_module_file: tfx.dsl.experimental.RuntimeParameter, - push_destination: tfx.dsl.experimental.RuntimeParameter, - enable_cache: bool, -): - """Creates a simple Kubeflow-based Chicago Taxi TFX pipeline. - - Args: - pipeline_root: The root of the pipeline output. - csv_input_location: The location of the input data directory. - taxi_module_file: The location of the module file for Transform/Trainer. - enable_cache: Whether to enable cache or not. - - Returns: - A logical TFX pipeline.Pipeline object. - """ - example_gen = tfx.components.CsvExampleGen(input_base=csv_input_location) - statistics_gen = tfx.components.StatisticsGen( - examples=example_gen.outputs['examples']) - schema_gen = tfx.components.SchemaGen( - statistics=statistics_gen.outputs['statistics'], - infer_feature_shape=False, - ) - example_validator = tfx.components.ExampleValidator( - statistics=statistics_gen.outputs['statistics'], - schema=schema_gen.outputs['schema'], - ) - transform = tfx.components.Transform( - examples=example_gen.outputs['examples'], - schema=schema_gen.outputs['schema'], - module_file=taxi_module_file, - ) - trainer = tfx.components.Trainer( - module_file=taxi_module_file, - examples=transform.outputs['transformed_examples'], - schema=schema_gen.outputs['schema'], - transform_graph=transform.outputs['transform_graph'], - train_args=tfx.proto.TrainArgs(num_steps=10), - eval_args=tfx.proto.EvalArgs(num_steps=5), - ) - # Set the TFMA config for Model Evaluation and Validation. - eval_config = tfma.EvalConfig( - model_specs=[ - tfma.ModelSpec( - signature_name='serving_default', - label_key='tips_xf', - preprocessing_function_names=['transform_features']) - ], - metrics_specs=[ - tfma.MetricsSpec( - # The metrics added here are in addition to those saved with the - # model (assuming either a keras model or EvalSavedModel is used). - # Any metrics added into the saved model (for example using - # model.compile(..., metrics=[...]), etc) will be computed - # automatically. - metrics=[tfma.MetricConfig(class_name='ExampleCount')], - # To add validation thresholds for metrics saved with the model, - # add them keyed by metric name to the thresholds map. - thresholds={ - 'binary_accuracy': - tfma.MetricThreshold( - value_threshold=tfma.GenericValueThreshold( - lower_bound={'value': 0.5}), - change_threshold=tfma.GenericChangeThreshold( - direction=tfma.MetricDirection.HIGHER_IS_BETTER, - absolute={'value': -1e-10})) - }) - ], - slicing_specs=[ - # An empty slice spec means the overall slice, i.e. the whole dataset. - tfma.SlicingSpec(), - # Data can be sliced along a feature column. In this case, data is - # sliced along feature column trip_start_hour. - tfma.SlicingSpec(feature_keys=['trip_start_hour']) - ]) - - evaluator = tfx.components.Evaluator( - examples=example_gen.outputs['examples'], - model=trainer.outputs['model'], - eval_config=eval_config, - ) - - pusher = tfx.components.Pusher( - model=trainer.outputs['model'], - model_blessing=evaluator.outputs['blessing'], - push_destination=push_destination, - ) - - return tfx.dsl.Pipeline( - pipeline_name='parameterized_tfx_oss', - pipeline_root=pipeline_root, - components=[ - example_gen, statistics_gen, schema_gen, example_validator, - transform, trainer, evaluator, pusher - ], - enable_cache=enable_cache, - ) - - -if __name__ == '__main__': - enable_cache = True - pipeline = _create_pipeline( - _pipeline_root, - _data_root, - _taxi_module_file_param, - _push_destination, - enable_cache=enable_cache, - ) - # Make sure the version of TFX image used is consistent with the version of - # TFX SDK. - config = tfx.orchestration.experimental.KubeflowDagRunnerConfig( - kubeflow_metadata_config=tfx.orchestration.experimental - .get_default_kubeflow_metadata_config(), - tfx_image='gcr.io/tfx-oss-public/tfx:%s' % tfx.__version__, - pipeline_operator_funcs=kubeflow_dag_runner - .get_default_pipeline_operator_funcs(use_gcp_sa=False) + [ - onprem.add_default_resource_spec( - memory_limit='2Gi', cpu_limit='2', cpu_request='1'), - ], - ) - kfp_runner = tfx.orchestration.experimental.KubeflowDagRunner( - output_filename=__file__ + '.yaml', - config=config, - ) - - kfp_runner.run(pipeline) diff --git a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py b/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py deleted file mode 100644 index 225e5a08494..00000000000 --- a/samples/core/parameterized_tfx_oss/parameterized_tfx_oss_test.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -try: - import kfp.deprecated as kfp -except ImportError: - import kfp - -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -bucket = 'kfp-ci' - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'parameterized_tfx_oss.py'), - pipeline_file_compile_path=relative_path( - __file__, 'parameterized_tfx_oss.py.yaml'), - arguments={ - 'pipeline-root': - f'gs://{bucket}/tfx_taxi_simple', - 'push_destination': - json.dumps({ - "filesystem": { - "base_directory": - f"gs://{bucket}/tfx_taxi_simple/serving_model" - } - }) - }, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - timeout_mins=40, - ), -]) diff --git a/samples/core/parameterized_tfx_oss/taxi_pipeline_notebook.ipynb b/samples/core/parameterized_tfx_oss/taxi_pipeline_notebook.ipynb deleted file mode 100644 index 244192cb9b3..00000000000 --- a/samples/core/parameterized_tfx_oss/taxi_pipeline_notebook.ipynb +++ /dev/null @@ -1,314 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# TFX pipeline example - Chicago Taxi tips prediction\n", - "\n", - "## Overview\n", - "[Tensorflow Extended (TFX)](https://github.com/tensorflow/tfx) is a Google-production-scale machine\n", - "learning platform based on TensorFlow. It provides a configuration framework to express ML pipelines\n", - "consisting of TFX components, which brings the user large-scale ML task orchestration, artifact lineage, as well as the power of various [TFX libraries](https://www.tensorflow.org/resources/libraries-extensions). Kubeflow Pipelines can be used as the orchestrator supporting the \n", - "execution of a TFX pipeline.\n", - "\n", - "This sample demonstrates how to author a ML pipeline in TFX and run it on a KFP deployment. \n", - "\n", - "## Permission\n", - "\n", - "This pipeline requires Google Cloud Storage permission to run. \n", - "If KFP was deployed through K8S marketplace, please make sure **\"Allow access to the following Cloud APIs\"** is checked when creating the cluster. \n", - "Otherwise, follow instructions in [the guideline](https://github.com/kubeflow/pipelines/blob/master/manifests/gcp_marketplace/guide.md#gcp-service-account-credentials) to guarantee at least, that the service account has `storage.admin` role." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!python3 -m pip install pip --upgrade --quiet --user\n", - "!python3 -m pip install kfp --upgrade --quiet --user\n", - "pip install tfx==1.4.0 tensorflow==2.5.1 --quiet --user" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "Note: if you're warned by \n", - "```\n", - "WARNING: The script {LIBRARY_NAME} is installed in '/home/jupyter/.local/bin' which is not on PATH.\n", - "```\n", - "You might need to fix by running the next cell and restart the kernel." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, - "outputs": [], - "source": [ - "# Set `PATH` to include user python binary directory and a directory containing `skaffold`.\n", - "PATH=%env PATH\n", - "%env PATH={PATH}:/home/jupyter/.local/bin" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In this example we'll need TFX SDK later than 0.21 to leverage the [`RuntimeParameter`](https://github.com/tensorflow/tfx/blob/93ea0b4eda5a6000a07a1e93d93a26441094b6f5/tfx/orchestration/data_types.py#L137) feature.\n", - "\n", - "## RuntimeParameter in TFX DSL\n", - "Currently, TFX DSL only supports parameterizing field in the `PARAMETERS` section of `ComponentSpec`, see [here](https://github.com/tensorflow/tfx/blob/93ea0b4eda5a6000a07a1e93d93a26441094b6f5/tfx/types/component_spec.py#L126). This prevents runtime-parameterizing the pipeline topology. Also, if the declared type of the field is a protobuf, the user needs to pass in a dictionary with exactly the same names for each field, and specify one or more value as `RuntimeParameter` objects. In other word, the dictionary should be able to be passed in to [`ParseDict()` method](https://github.com/protocolbuffers/protobuf/blob/04a11fc91668884d1793bff2a0f72ee6ce4f5edd/python/google/protobuf/json_format.py#L433) and produce the correct pb message." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "import os\n", - "\n", - "import kfp\n", - "import tensorflow_model_analysis as tfma\n", - "from tfx import v1 as tfx" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# In TFX MLMD schema, pipeline name is used as the unique id of each pipeline.\n", - "# Assigning workflow ID as part of pipeline name allows the user to bypass\n", - "# some schema checks which are redundant for experimental pipelines.\n", - "pipeline_name = 'taxi_pipeline_with_parameters'\n", - "\n", - "# Path of pipeline data root, should be a GCS path.\n", - "# Note that when running on KFP, the pipeline root is always a runtime parameter.\n", - "# The value specified here will be its default.\n", - "pipeline_root = os.path.join('gs://{{kfp-default-bucket}}', 'tfx_taxi_simple',\n", - " kfp.dsl.RUN_ID_PLACEHOLDER)\n", - "\n", - "# Location of input data, should be a GCS path under which there is a csv file.\n", - "data_root = '/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/data/simple'\n", - "\n", - "# Path to the module file, GCS path.\n", - "# Module file is one of the recommended way to provide customized logic for component\n", - "# includeing Trainer and Transformer.\n", - "# See https://github.com/tensorflow/tfx/blob/93ea0b4eda5a6000a07a1e93d93a26441094b6f5/tfx/components/trainer/component.py#L38\n", - "taxi_module_file_param = tfx.dsl.experimental.RuntimeParameter(\n", - " name='module-file',\n", - " default='/opt/conda/lib/python3.7/site-packages/tfx/examples/chicago_taxi_pipeline/taxi_utils_native_keras.py',\n", - " ptype=str,\n", - ")\n", - "# Path that ML models are pushed, should be a GCS path.\n", - "# TODO: CHANGE the GCS bucket name to yours.\n", - "serving_model_dir = os.path.join('gs://your-bucket', 'serving_model', 'tfx_taxi_simple')\n", - "push_destination = tfx.dsl.experimental.RuntimeParameter(\n", - " name='push_destination',\n", - " default=json.dumps({'filesystem': {'base_directory': serving_model_dir}}),\n", - " ptype=str,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## TFX Components\n", - "\n", - "Please refer to the [official guide](https://www.tensorflow.org/tfx/guide#tfx_pipeline_components) for the detailed explanation and purpose of each TFX component." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "example_gen = tfx.components.CsvExampleGen(input_base=data_root)\n", - "\n", - "statistics_gen = tfx.components.StatisticsGen(examples=example_gen.outputs['examples'])\n", - "\n", - "schema_gen = tfx.components.SchemaGen(\n", - " statistics=statistics_gen.outputs['statistics'], infer_feature_shape=False)\n", - "\n", - "example_validator = tfx.components.ExampleValidator(\n", - " statistics=statistics_gen.outputs['statistics'],\n", - " schema=schema_gen.outputs['schema'])\n", - "\n", - "# The module file used in Transform and Trainer component is paramterized by\n", - "# _taxi_module_file_param.\n", - "transform = tfx.components.Transform(\n", - " examples=example_gen.outputs['examples'],\n", - " schema=schema_gen.outputs['schema'],\n", - " module_file=taxi_module_file_param)\n", - "\n", - "# The numbers of steps in train_args are specified as RuntimeParameter with\n", - "# name 'train-steps' and 'eval-steps', respectively.\n", - "trainer = tfx.components.Trainer(\n", - " module_file=taxi_module_file_param,\n", - " examples=transform.outputs['transformed_examples'],\n", - " schema=schema_gen.outputs['schema'],\n", - " transform_graph=transform.outputs['transform_graph'],\n", - " train_args=tfx.proto.TrainArgs(num_steps=10),\n", - " eval_args=tfx.proto.EvalArgs(num_steps=5))\n", - "\n", - "# Set the TFMA config for Model Evaluation and Validation.\n", - "eval_config = tfma.EvalConfig(\n", - " model_specs=[\n", - " tfma.ModelSpec(\n", - " signature_name='serving_default', label_key='tips_xf',\n", - " preprocessing_function_names=['transform_features'])\n", - " ],\n", - " metrics_specs=[\n", - " tfma.MetricsSpec(\n", - " # The metrics added here are in addition to those saved with the\n", - " # model (assuming either a keras model or EvalSavedModel is used).\n", - " # Any metrics added into the saved model (for example using\n", - " # model.compile(..., metrics=[...]), etc) will be computed\n", - " # automatically.\n", - " metrics=[\n", - " tfma.MetricConfig(class_name='ExampleCount')\n", - " ],\n", - " # To add validation thresholds for metrics saved with the model,\n", - " # add them keyed by metric name to the thresholds map.\n", - " thresholds = {\n", - " 'binary_accuracy': tfma.MetricThreshold(\n", - " value_threshold=tfma.GenericValueThreshold(\n", - " lower_bound={'value': 0.5}),\n", - " change_threshold=tfma.GenericChangeThreshold(\n", - " direction=tfma.MetricDirection.HIGHER_IS_BETTER,\n", - " absolute={'value': -1e-10}))\n", - " }\n", - " )\n", - " ],\n", - " slicing_specs=[\n", - " # An empty slice spec means the overall slice, i.e. the whole dataset.\n", - " tfma.SlicingSpec(),\n", - " # Data can be sliced along a feature column. In this case, data is\n", - " # sliced along feature column trip_start_hour.\n", - " tfma.SlicingSpec(feature_keys=['trip_start_hour'])\n", - " ])\n", - "\n", - "# The name of slicing column is specified as a RuntimeParameter.\n", - "evaluator = tfx.components.Evaluator(\n", - " examples=example_gen.outputs['examples'],\n", - " model=trainer.outputs['model'],\n", - " eval_config=eval_config)\n", - "\n", - "pusher = tfx.components.Pusher(\n", - " model=trainer.outputs['model'],\n", - " model_blessing=evaluator.outputs['blessing'],\n", - " push_destination=push_destination)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Create the DSL pipeline object.\n", - "# This pipeline obj carries the business logic of the pipeline, but no runner-specific information\n", - "# was included.\n", - "dsl_pipeline = tfx.dsl.Pipeline(\n", - " pipeline_name=pipeline_name,\n", - " pipeline_root=pipeline_root,\n", - " components=[\n", - " example_gen, statistics_gen, schema_gen, example_validator, transform,\n", - " trainer, evaluator, pusher\n", - " ],\n", - " enable_cache=True,\n", - " beam_pipeline_args=['--direct_num_workers=%d' % 0],\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Specify a TFX docker image. For the full list of tags please see:\n", - "# https://hub.docker.com/r/tensorflow/tfx/tags\n", - "tfx_image = 'gcr.io/tfx-oss-public/tfx:1.4.0'\n", - "config = tfx.orchestration.experimental.KubeflowDagRunnerConfig(\n", - " kubeflow_metadata_config=tfx.orchestration.experimental\n", - " .get_default_kubeflow_metadata_config(),\n", - " tfx_image=tfx_image)\n", - "kfp_runner = tfx.orchestration.experimental.KubeflowDagRunner(config=config)\n", - "# KubeflowDagRunner compiles the DSL pipeline object into KFP pipeline package.\n", - "# By default it is named .tar.gz\n", - "kfp_runner.run(dsl_pipeline)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "run_result = kfp.Client(\n", - " host='1234567abcde-dot-us-central2.pipelines.googleusercontent.com' # Put your KFP endpoint here\n", - ").create_run_from_pipeline_package(\n", - " pipeline_name + '.tar.gz', \n", - " arguments={\n", - " # Uncomment following lines in order to use custom GCS bucket/module file/training data.\n", - " # 'pipeline-root': 'gs:///tfx_taxi_simple/' + kfp.dsl.RUN_ID_PLACEHOLDER,\n", - " # 'module-file': '', # delete this line to use default module file.\n", - " # 'data-root': '' # delete this line to use default data.\n", - "})" - ] - } - ], - "metadata": { - "environment": { - "name": "tf2-gpu.2-4.m69", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-4:m69" - }, - "kernelspec": { - "display_name": "Python [conda env:root] *", - "language": "python", - "name": "conda-root-py" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - }, - "pycharm": { - "stem_cell": { - "cell_type": "raw", - "metadata": { - "collapsed": false - }, - "source": [] - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/samples/core/pipeline_parallelism/pipeline_parallelism_limits.py b/samples/core/pipeline_parallelism/pipeline_parallelism_limits.py deleted file mode 100644 index e4e62f45685..00000000000 --- a/samples/core/pipeline_parallelism/pipeline_parallelism_limits.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -from kfp.deprecated import dsl, compiler, components - - -@components.create_component_from_func -def print_op(msg): - """Print a message.""" - print(msg) - - -@dsl.pipeline( - name='pipeline-service-account', - description='The pipeline shows how to set the max number of parallel pods in a pipeline.' -) -def pipeline_parallelism(): - op1 = print_op('hey, what are you up to?') - op2 = print_op('train my model.') - dsl.get_pipeline_conf().set_parallelism(1) - -if __name__ == '__main__': - compiler.Compiler().compile(pipeline_parallelism, __file__ + '.yaml') diff --git a/samples/core/pipeline_transformers/pipeline_transformers.py b/samples/core/pipeline_transformers/pipeline_transformers.py deleted file mode 100644 index ec70ea8df10..00000000000 --- a/samples/core/pipeline_transformers/pipeline_transformers.py +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import kfp.deprecated.components as comp -from kfp.deprecated import dsl, compiler - - -@comp.create_component_from_func -def print_op(msg: str): - """Print a message.""" - print(msg) - - -def add_annotation(op): - op.add_pod_annotation(name='hobby', value='football') - return op - - -@dsl.pipeline( - name='pipeline-transformer', - description='The pipeline shows how to apply functions to all ops in the pipeline by pipeline transformers' -) -def transform_pipeline(): - op1 = print_op('hey, what are you up to?') - op2 = print_op('train my model.') - dsl.get_pipeline_conf().add_op_transformer(add_annotation) - -if __name__ == '__main__': - compiler.Compiler().compile(transform_pipeline, __file__ + '.yaml') diff --git a/samples/core/pipeline_transformers/pipeline_transformers_test.py b/samples/core/pipeline_transformers/pipeline_transformers_test.py deleted file mode 100644 index 2fd19033844..00000000000 --- a/samples/core/pipeline_transformers/pipeline_transformers_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'pipeline_transformers.py'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), -]) diff --git a/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu.py b/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu.py deleted file mode 100644 index fd88e1e5e2c..00000000000 --- a/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This is a simple toy example demonstrating how to use preemptible computing resources.""" - -from kfp.deprecated import dsl, compiler, gcp - - -class FlipCoinOp(dsl.ContainerOp): - """Flip a coin and output heads or tails randomly.""" - - def __init__(self): - super(FlipCoinOp, self).__init__( - name='Flip', - image='python:alpine3.6', - command=['sh', '-c'], - arguments=[ - 'python -c "import random; result = \'heads\' if random.randint(0,1) == 0 ' - 'else \'tails\'; print(result)" | tee /tmp/output' - ], - file_outputs={'output': '/tmp/output'}) - - -@dsl.pipeline( - name='pipeline-flip-coin', description='shows how to use dsl.Condition.') -def flipcoin(): - flip = FlipCoinOp().apply(gcp.use_preemptible_nodepool()).set_gpu_limit( - 1, 'nvidia').set_retry(5) - - -if __name__ == '__main__': - compiler.Compiler().compile(flipcoin, __file__ + '.yaml') diff --git a/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu_test.py b/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu_test.py deleted file mode 100644 index 09460bd03ea..00000000000 --- a/samples/core/preemptible_tpu_gpu/preemptible_tpu_gpu_test.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'preemptible_tpu_gpu.py'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - run_pipeline=False, - ), -]) diff --git a/samples/core/resource_ops/resource_ops.py b/samples/core/resource_ops/resource_ops.py deleted file mode 100644 index 24932ca29f6..00000000000 --- a/samples/core/resource_ops/resource_ops.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -This example demonstrates how to use ResourceOp to specify the value of env var. -""" - -import json -from kfp.deprecated import dsl, compiler - - -_CONTAINER_MANIFEST = """ -{ - "apiVersion": "batch/v1", - "kind": "Job", - "metadata": { - "generateName": "resourceop-basic-job-" - }, - "spec": { - "template": { - "metadata": { - "name": "resource-basic" - }, - "spec": { - "containers": [{ - "name": "sample-container", - "image": "registry.k8s.io/busybox", - "command": ["/usr/bin/env"] - }], - "restartPolicy": "Never" - } - }, - "backoffLimit": 4 - } -} -""" - - -@dsl.pipeline( - name="resourceop-basic", - description="A Basic Example on ResourceOp Usage." -) -def resourceop_basic(): - - # Start a container. Print out env vars. - op = dsl.ResourceOp( - name='test-step', - k8s_resource=json.loads(_CONTAINER_MANIFEST), - action='create' - ) - - -if __name__ == '__main__': - compiler.Compiler().compile(resourceop_basic, __file__ + '.yaml') diff --git a/samples/core/resource_ops/resource_ops_test.py b/samples/core/resource_ops/resource_ops_test.py deleted file mode 100644 index 90fd9ea1568..00000000000 --- a/samples/core/resource_ops/resource_ops_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'resource_ops.py'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), -]) diff --git a/samples/core/resource_spec/resource_spec_test.py b/samples/core/resource_spec/resource_spec_test.py index b92dd121535..9255a2986a4 100644 --- a/samples/core/resource_spec/resource_spec_test.py +++ b/samples/core/resource_spec/resource_spec_test.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kfp import dsl +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase from resource_spec import my_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase def EXPECTED_OOM(run_id, run, **kwargs): @@ -23,13 +23,9 @@ def EXPECTED_OOM(run_id, run, **kwargs): run_pipeline_func([ + TestCase(pipeline_func=my_pipeline,), TestCase( pipeline_func=my_pipeline, - mode=dsl.PipelineExecutionMode.V2_ENGINE, - ), - TestCase( - pipeline_func=my_pipeline, - mode=dsl.PipelineExecutionMode.V2_ENGINE, arguments={'n': 21234567}, verify_func=EXPECTED_OOM, ), diff --git a/samples/core/resource_spec/runtime_resource_request_test.py b/samples/core/resource_spec/runtime_resource_request_test.py index 7f74331882f..55e21701ec3 100644 --- a/samples/core/resource_spec/runtime_resource_request_test.py +++ b/samples/core/resource_spec/runtime_resource_request_test.py @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase from runtime_resource_request import resource_request_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase def EXPECTED_OOM(run_id, run, **kwargs): @@ -22,13 +23,9 @@ def EXPECTED_OOM(run_id, run, **kwargs): run_pipeline_func([ + TestCase(pipeline_func=resource_request_pipeline,), TestCase( pipeline_func=resource_request_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - ), - TestCase( - pipeline_func=resource_request_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, arguments={'n': 21234567}, verify_func=EXPECTED_OOM, ), diff --git a/samples/core/retry/retry_test.py b/samples/core/retry/retry_test.py index c6ffa8cd274..fa9cbe0ce63 100644 --- a/samples/core/retry/retry_test.py +++ b/samples/core/retry/retry_test.py @@ -13,11 +13,10 @@ # limitations under the License. import kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func +from kfp.samples.test.utils import relative_path +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'retry.py'), - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - ), + TestCase(pipeline_file=relative_path(__file__, 'retry.py'),), ]) diff --git a/samples/core/secret/secret_test.py b/samples/core/secret/secret_test.py index 8758db4d4b9..ecf86df0d02 100644 --- a/samples/core/secret/secret_test.py +++ b/samples/core/secret/secret_test.py @@ -12,12 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func +from kfp.samples.test.utils import relative_path +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'secret.py'), - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - ), + TestCase(pipeline_file=relative_path(__file__, 'secret.py'),), ]) diff --git a/samples/core/sidecar/sidecar.py b/samples/core/sidecar/sidecar.py deleted file mode 100644 index 10d07ad2011..00000000000 --- a/samples/core/sidecar/sidecar.py +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019, 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated import dsl, compiler - - -@dsl.pipeline( - name="pipeline-with-sidecar", - description= - "A pipeline that demonstrates how to add a sidecar to an operation." -) -def pipeline_with_sidecar(): - # sidecar with sevice that reply "hello world" to any GET request - echo = dsl.Sidecar( - name="echo", - image="nginx:1.13", - command=["nginx", "-g", "daemon off;"], - ) - - # container op with sidecar - op1 = dsl.ContainerOp( - name="download", - image="busybox:latest", - command=["sh", "-c"], - arguments=[ - "until wget http://localhost:80 -O /tmp/results.txt; do sleep 5; done && cat /tmp/results.txt" - ], - sidecars=[echo], - file_outputs={"downloaded": "/tmp/results.txt"}, - ) - - -if __name__ == '__main__': - compiler.Compiler().compile(pipeline_with_sidecar, __file__ + '.yaml') diff --git a/samples/core/sidecar/sidecar_test.py b/samples/core/sidecar/sidecar_test.py deleted file mode 100644 index ad7177f454d..00000000000 --- a/samples/core/sidecar/sidecar_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'sidecar.py'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), -]) diff --git a/samples/core/use_run_info/use_run_id.py b/samples/core/use_run_info/use_run_id.py deleted file mode 100644 index 9dbce233117..00000000000 --- a/samples/core/use_run_info/use_run_id.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.deprecated import components, dsl, compiler - -def get_run_info(run_id: str): - """Example of getting run info for current pipeline run.""" - print(f'Current run ID is {run_id}.') - # KFP API server is usually available as ml-pipeline service in the same - # namespace, but for full Kubeflow deployment, you need to edit this to - # http://ml-pipeline.kubeflow:8888, because your pipelines are running in - # user namespaces, but the API is at kubeflow namespace. - import kfp - client = kfp.Client(host='http://ml-pipeline:8888') - run_info = client.get_run(run_id=run_id) - # Hide verbose info - print(run_info.run) - - -get_run_info_component = components.create_component_from_func( - func=get_run_info, - packages_to_install=['kfp'], -) - - -@dsl.pipeline( - name='use-run-id', - description='A pipeline that demonstrates how to use run information, including run ID etc.' -) -def pipeline_use_run_id(run_id: str = kfp.dsl.RUN_ID_PLACEHOLDER): - """kfp.dsl.RUN_ID_PLACEHOLDER inside a pipeline parameter will be populated - with KFP Run ID at runtime.""" - run_info_op = get_run_info_component(run_id=run_id) - - -if __name__ == '__main__': - compiler.Compiler().compile(pipeline_use_run_id, __file__ + '.yaml') diff --git a/samples/core/use_run_info/use_run_id_test.py b/samples/core/use_run_info/use_run_id_test.py deleted file mode 100644 index a26c3cb47a1..00000000000 --- a/samples/core/use_run_info/use_run_id_test.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from .use_run_id import pipeline_use_run_id -from kfp.samples.test.utils import run_pipeline_func, TestCase - -run_pipeline_func([ - TestCase( - pipeline_func=pipeline_use_run_id, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), -]) diff --git a/samples/core/visualization/confusion_matrix.csv b/samples/core/visualization/confusion_matrix.csv deleted file mode 100644 index 933f7100160..00000000000 --- a/samples/core/visualization/confusion_matrix.csv +++ /dev/null @@ -1,9 +0,0 @@ -rose,rose,100 -rose,lily,20 -rose,iris,5 -lily,rose,40 -lily,lily,200 -lily,iris,33 -iris,rose,0 -iris,lily,10 -iris,iris,200 diff --git a/samples/core/visualization/confusion_matrix.py b/samples/core/visualization/confusion_matrix.py deleted file mode 100644 index 1716d620f2f..00000000000 --- a/samples/core/visualization/confusion_matrix.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -from kfp.deprecated.components import create_component_from_func - -# Advanced function -# Demonstrates imports, helper functions and multiple outputs -from typing import NamedTuple - - -@create_component_from_func -def confusion_visualization(matrix_uri: str = 'https://raw.githubusercontent.com/kubeflow/pipelines/master/samples/core/visualization/confusion_matrix.csv') -> NamedTuple('VisualizationOutput', [('mlpipeline_ui_metadata', 'UI_metadata')]): - """Provide confusion matrix csv file to visualize as metrics.""" - import json - - metadata = { - 'outputs' : [{ - 'type': 'confusion_matrix', - 'format': 'csv', - 'schema': [ - {'name': 'target', 'type': 'CATEGORY'}, - {'name': 'predicted', 'type': 'CATEGORY'}, - {'name': 'count', 'type': 'NUMBER'}, - ], - 'source': matrix_uri, - 'labels': ['rose', 'lily', 'iris'], - }] - } - - from collections import namedtuple - visualization_output = namedtuple('VisualizationOutput', [ - 'mlpipeline_ui_metadata']) - return visualization_output(json.dumps(metadata)) - -@dsl.pipeline( - name='confusion-matrix-pipeline', - description='A sample pipeline to generate Confusion Matrix for UI visualization.' -) -def confusion_matrix_pipeline(): - confusion_visualization_task = confusion_visualization() - # You can also upload samples/core/visualization/confusion_matrix.csv to Google Cloud Storage. - # And call the component function with gcs path parameter like below: - # confusion_visualization_task2 = confusion_visualization('gs:////confusion_matrix.csv') diff --git a/samples/core/visualization/confusion_matrix_test.py b/samples/core/visualization/confusion_matrix_test.py deleted file mode 100644 index ef39829f564..00000000000 --- a/samples/core/visualization/confusion_matrix_test.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp_server_api -import unittest -from pprint import pprint -from .confusion_matrix import confusion_matrix_pipeline, confusion_visualization -from kfp.samples.test.utils import KfpMlmdClient, run_pipeline_func, TestCase - -import kfp.deprecated as kfp - - -def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): - t = unittest.TestCase() - t.maxDiff = None # we always want to see full diff - t.assertEqual(run.status, 'Succeeded') - client = KfpMlmdClient(mlmd_connection_config=mlmd_connection_config) - tasks = client.get_tasks(run_id=run.id) - pprint(tasks) - - confusion_visualization = tasks['confusion-visualization'] - output = [ - a for a in confusion_visualization.outputs.artifacts - if a.name == 'mlpipeline_ui_metadata' - ][0] - pprint(output) - - t.assertEqual( - confusion_visualization.get_dict()['outputs']['artifacts'][0]['name'], - 'mlpipeline_ui_metadata') - - -run_pipeline_func([ - TestCase( - pipeline_func=confusion_matrix_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY) -]) diff --git a/samples/core/visualization/hello-world.html b/samples/core/visualization/hello-world.html deleted file mode 100644 index 551b96b8c4e..00000000000 --- a/samples/core/visualization/hello-world.html +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - -
Hello World!
- - - - diff --git a/samples/core/visualization/html.py b/samples/core/visualization/html.py deleted file mode 100644 index 409c83cbf12..00000000000 --- a/samples/core/visualization/html.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated import dsl, components - -# Advanced function -# Demonstrates imports, helper functions and multiple outputs -from typing import NamedTuple - -@components.create_component_from_func -def html_visualization(gcsPath: str) -> NamedTuple('VisualizationOutput', [('mlpipeline_ui_metadata', 'UI_metadata')]): - import json - - metadata = { - 'outputs': [{ - 'type': 'web-app', - 'storage': 'inline', - 'source': '

Hello, World!

', - }] - } - - # Temporarily hack for empty string scenario: https://github.com/kubeflow/pipelines/issues/5830 - if gcsPath and gcsPath != 'BEGIN-KFP-PARAM[]END-KFP-PARAM': - metadata.get('outputs').append({ - 'type': 'web-app', - 'storage': 'gcs', - 'source': gcsPath, - }) - - from collections import namedtuple - visualization_output = namedtuple('VisualizationOutput', [ - 'mlpipeline_ui_metadata']) - return visualization_output(json.dumps(metadata)) - -@dsl.pipeline( - name='html-pipeline', - description='A sample pipeline to generate HTML for UI visualization.' -) -def html_pipeline(): - html_visualization_task = html_visualization("") - # html_visualization_task = html_visualization_op("gs://jamxl-kfp-bucket/v2-compatible/html/hello-world.html") - # Replace the parameter gcsPath with actual google cloud storage path with html file. - # For example: Upload hello-world.html in the same folder to gs://bucket-name/hello-world.html. - # Then uncomment the following line. - # html_visualization_task = html_visualization_op("gs://bucket-name/hello-world.html") diff --git a/samples/core/visualization/html_test.py b/samples/core/visualization/html_test.py deleted file mode 100644 index 2469400185d..00000000000 --- a/samples/core/visualization/html_test.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from .html import html_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase - -run_pipeline_func([ - TestCase( - pipeline_func=html_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY) -]) diff --git a/samples/core/visualization/markdown.py b/samples/core/visualization/markdown.py deleted file mode 100644 index 31564d7a9ba..00000000000 --- a/samples/core/visualization/markdown.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -import kfp.deprecated.components as comp - -# Advanced function -# Demonstrates imports, helper functions and multiple outputs -from typing import NamedTuple - -@comp.create_component_from_func -def markdown_visualization() -> NamedTuple('VisualizationOutput', [('mlpipeline_ui_metadata', 'UI_metadata')]): - import json - - # Exports a sample tensorboard: - metadata = { - 'outputs': [ - { - # Markdown that is hardcoded inline - 'storage': 'inline', - 'source': '''# Inline Markdown - -* [Kubeflow official doc](https://www.kubeflow.org/). -''', - 'type': 'markdown', - }, - { - # Markdown that is read from a file - 'source': 'https://raw.githubusercontent.com/kubeflow/pipelines/master/README.md', - # Alternatively, use Google Cloud Storage for sample. - # 'source': 'gs://jamxl-kfp-bucket/v2-compatible/markdown/markdown_example.md', - 'type': 'markdown', - }] - } - - from collections import namedtuple - divmod_output = namedtuple('VisualizationOutput', [ - 'mlpipeline_ui_metadata']) - return divmod_output(json.dumps(metadata)) - - -@dsl.pipeline( - name='markdown-pipeline', - description='A sample pipeline to generate markdown for UI visualization.' -) -def markdown_pipeline(): - # Passing a task output reference as operation arguments - # For an operation with a single return value, the output reference can be accessed using `task.output` or `task.outputs['output_name']` syntax - markdown_visualization_task = markdown_visualization() diff --git a/samples/core/visualization/markdown_test.py b/samples/core/visualization/markdown_test.py deleted file mode 100644 index 629df3191e9..00000000000 --- a/samples/core/visualization/markdown_test.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .markdown import markdown_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase - -import kfp.deprecated as kfp - -run_pipeline_func([ - TestCase( - pipeline_func=markdown_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY) -]) diff --git a/samples/core/visualization/roc.csv b/samples/core/visualization/roc.csv deleted file mode 100644 index 305937108bc..00000000000 --- a/samples/core/visualization/roc.csv +++ /dev/null @@ -1,293 +0,0 @@ -0.0,0.00265957446809,0.999972701073 -0.0,0.226063829787,0.996713876724 -0.000139742873113,0.226063829787,0.996708869934 -0.000139742873113,0.534574468085,0.818370699883 -0.00041922861934,0.534574468085,0.809346497059 -0.00041922861934,0.574468085106,0.745872795582 -0.000558971492454,0.574468085106,0.742943286896 -0.000558971492454,0.577127659574,0.740854740143 -0.000698714365567,0.577127659574,0.71646541357 -0.000698714365567,0.61170212766,0.62472563982 -0.000838457238681,0.61170212766,0.602016150951 -0.000838457238681,0.614361702128,0.571930527687 -0.00111794298491,0.614361702128,0.544027626514 -0.00111794298491,0.617021276596,0.511012971401 -0.00153717160425,0.617021276596,0.496669858694 -0.00153717160425,0.619680851064,0.49652710557 -0.00167691447736,0.619680851064,0.493446528912 -0.00167691447736,0.625,0.481318473816 -0.00181665735048,0.625,0.475450545549 -0.00181665735048,0.627659574468,0.463429301977 -0.00223588596982,0.627659574468,0.453910887241 -0.00223588596982,0.640957446809,0.423157393932 -0.00279485746227,0.640957446809,0.4135017097 -0.00279485746227,0.646276595745,0.378102302551 -0.00293460033538,0.646276595745,0.372579962015 -0.00293460033538,0.648936170213,0.369351655245 -0.00321408608161,0.648936170213,0.351242959499 -0.00321408608161,0.656914893617,0.339924275875 -0.00335382895472,0.656914893617,0.329248338938 -0.00335382895472,0.659574468085,0.32767277956 -0.00363331470095,0.659574468085,0.320836901665 -0.00363331470095,0.667553191489,0.314506083727 -0.00377305757406,0.667553191489,0.312844485044 -0.00377305757406,0.670212765957,0.305956125259 -0.00391280044718,0.670212765957,0.297088235617 -0.00391280044718,0.678191489362,0.291977673769 -0.0041922861934,0.678191489362,0.286320656538 -0.0041922861934,0.68085106383,0.27750852704 -0.00447177193963,0.68085106383,0.276229381561 -0.00447177193963,0.683510638298,0.26968190074 -0.0051704863052,0.683510638298,0.251649200916 -0.0051704863052,0.686170212766,0.241131410003 -0.00558971492454,0.686170212766,0.233562991023 -0.00558971492454,0.688829787234,0.228251799941 -0.00628842929011,0.688829787234,0.22050255537 -0.00628842929011,0.691489361702,0.219731196761 -0.00698714365567,0.691489361702,0.20610666275 -0.00698714365567,0.69414893617,0.205215752125 -0.00768585802124,0.69414893617,0.192584708333 -0.00768585802124,0.699468085106,0.19175067544 -0.00852431525992,0.699468085106,0.18044243753 -0.00852431525992,0.702127659574,0.18012419343 -0.00992174399106,0.702127659574,0.164085775614 -0.00992174399106,0.704787234043,0.163971275091 -0.0107602012297,0.704787234043,0.148682847619 -0.0107602012297,0.707446808511,0.147884652019 -0.0108999441029,0.707446808511,0.147360503674 -0.0108999441029,0.710106382979,0.146837860346 -0.011039686976,0.710106382979,0.145389840007 -0.0113191727222,0.710106382979,0.145035877824 -0.0115986584684,0.710106382979,0.143553003669 -0.0115986584684,0.715425531915,0.139511361718 -0.012297372834,0.715425531915,0.137044250965 -0.012297372834,0.718085106383,0.136469885707 -0.0125768585802,0.718085106383,0.135680988431 -0.0125768585802,0.720744680851,0.135541856289 -0.0128563443264,0.720744680851,0.132694289088 -0.0128563443264,0.723404255319,0.132042109966 -0.0131358300727,0.723404255319,0.130991369486 -0.0131358300727,0.726063829787,0.130810260773 -0.0143935159307,0.726063829787,0.127302840352 -0.0143935159307,0.728723404255,0.123035728931 -0.0155114589156,0.728723404255,0.119730189443 -0.0155114589156,0.731382978723,0.119614243507 -0.016070430408,0.731382978723,0.11746224016 -0.016070430408,0.734042553191,0.116296350956 -0.0162101732812,0.734042553191,0.115002587438 -0.0162101732812,0.73670212766,0.114347569644 -0.0167691447736,0.73670212766,0.112534038723 -0.0167691447736,0.739361702128,0.111200802028 -0.0173281162661,0.739361702128,0.10874915868 -0.0173281162661,0.742021276596,0.108626417816 -0.0174678591392,0.742021276596,0.108434587717 -0.0174678591392,0.747340425532,0.105106987059 -0.0178870877585,0.747340425532,0.103880673647 -0.0178870877585,0.75,0.101953282952 -0.018446059251,0.75,0.0989835187793 -0.018446059251,0.752659574468,0.0988245904446 -0.0185858021241,0.752659574468,0.0988096669316 -0.0185858021241,0.755319148936,0.0987068340182 -0.022219116825,0.755319148936,0.0884411558509 -0.022219116825,0.757978723404,0.0880292057991 -0.0227780883175,0.757978723404,0.0860563218594 -0.0227780883175,0.765957446809,0.0856974124908 -0.0244550027949,0.765957446809,0.0827387049794 -0.0244550027949,0.768617021277,0.0824092328548 -0.025852431526,0.768617021277,0.0802820026875 -0.025852431526,0.773936170213,0.0794423744082 -0.0272498602571,0.773936170213,0.0782480686903 -0.0272498602571,0.776595744681,0.0782372131944 -0.0273896031302,0.776595744681,0.0777427777648 -0.0273896031302,0.779255319149,0.0776203125715 -0.0278088317496,0.779255319149,0.0771161615849 -0.0278088317496,0.781914893617,0.0767232328653 -0.0296254891001,0.781914893617,0.0745082348585 -0.0296254891001,0.784574468085,0.0744256600738 -0.0329793180548,0.784574468085,0.0692687630653 -0.0329793180548,0.789893617021,0.0688586980104 -0.0339575181666,0.789893617021,0.0679297447205 -0.0339575181666,0.795212765957,0.0673902630806 -0.034516489659,0.795212765957,0.0668559446931 -0.034516489659,0.797872340426,0.0667828992009 -0.0385690329793,0.797872340426,0.0621695742011 -0.0385690329793,0.800531914894,0.0621492564678 -0.0398267188373,0.800531914894,0.0610552579165 -0.0398267188373,0.803191489362,0.0609075687826 -0.0422023476803,0.803191489362,0.0591055937111 -0.0422023476803,0.80585106383,0.0589610114694 -0.0424818334265,0.80585106383,0.0587852373719 -0.0427613191727,0.80585106383,0.0587054751813 -0.0429010620458,0.80585106383,0.0585668422282 -0.0429010620458,0.808510638298,0.0585095882416 -0.0434600335383,0.808510638298,0.0582418218255 -0.0434600335383,0.811170212766,0.0580357164145 -0.0447177193963,0.811170212766,0.0575000457466 -0.0447177193963,0.813829787234,0.057237111032 -0.045556176635,0.813829787234,0.0567515231669 -0.045556176635,0.816489361702,0.0566952005029 -0.0493292342091,0.816489361702,0.054908964783 -0.0493292342091,0.81914893617,0.054908555001 -0.0522638345444,0.81914893617,0.0540485493839 -0.0869200670766,0.837765957447,0.0540481060743 -0.0897149245388,0.837765957447,0.0534495189786 -0.0897149245388,0.840425531915,0.0534458272159 -0.0916713247624,0.840425531915,0.0530435182154 -0.0919508105087,0.840425531915,0.0530236177146 -0.0926495248742,0.840425531915,0.05276074633 -0.0926495248742,0.843085106383,0.0527284368873 -0.0968418110676,0.843085106383,0.0516891926527 -0.0968418110676,0.848404255319,0.0516689941287 -0.0971212968139,0.848404255319,0.0516517348588 -0.0978200111794,0.848404255319,0.0515133664012 -0.0978200111794,0.851063829787,0.0514727458358 -0.098518725545,0.851063829787,0.0513115003705 -0.0989379541643,0.851063829787,0.0512120835483 -0.0990776970375,0.851063829787,0.0512094497681 -0.0990776970375,0.853723404255,0.0510897599161 -0.103549468977,0.853723404255,0.0497896969318 -0.103549468977,0.856382978723,0.0497280210257 -0.10997764114,0.856382978723,0.0474901869893 -0.10997764114,0.859042553191,0.0474526695907 -0.112632755729,0.859042553191,0.0463959015906 -0.112632755729,0.86170212766,0.0463563986123 -0.113051984349,0.86170212766,0.0462292470038 -0.113051984349,0.864361702128,0.0461205877364 -0.116545556177,0.864361702128,0.0451705344021 -0.116545556177,0.867021276596,0.0451255217195 -0.122135271101,0.867021276596,0.0436667315662 -0.122135271101,0.869680851064,0.0436404012144 -0.129541643376,0.869680851064,0.0417694710195 -0.129541643376,0.872340425532,0.0417521372437 -0.132476243712,0.872340425532,0.0410900376737 -0.132476243712,0.875,0.0410351417959 -0.133454443823,0.875,0.0408792309463 -0.133454443823,0.877659574468,0.0408616252244 -0.136109558413,0.877659574468,0.0400370657444 -0.136109558413,0.880319148936,0.040026742965 -0.141140301845,0.880319148936,0.0389389395714 -0.141140301845,0.882978723404,0.0388600751758 -0.144214645053,0.882978723404,0.0380442813039 -0.144214645053,0.885638297872,0.0380435734987 -0.144354387926,0.885638297872,0.0380318425596 -0.145053102292,0.885638297872,0.0380309186876 -0.147847959754,0.885638297872,0.0375708192587 -0.147847959754,0.88829787234,0.0375643596053 -0.148406931247,0.88829787234,0.0374849624932 -0.148406931247,0.890957446809,0.037457678467 -0.152179988821,0.890957446809,0.0365050286055 -0.152179988821,0.893617021277,0.0365023724735 -0.155813303522,0.893617021277,0.0358927957714 -0.155813303522,0.896276595745,0.0358639582992 -0.160285075461,0.896276595745,0.0347978547215 -0.160285075461,0.898936170213,0.0347460657358 -0.161263275573,0.898936170213,0.0344333276153 -0.161263275573,0.901595744681,0.0344075076282 -0.165315818893,0.901595744681,0.0334138572216 -0.165315818893,0.904255319149,0.0333491191268 -0.166713247624,0.904255319149,0.0331395342946 -0.166713247624,0.906914893617,0.0331283695996 -0.167272219117,0.906914893617,0.0329953655601 -0.167272219117,0.909574468085,0.0329646356404 -0.170346562325,0.909574468085,0.032298386097 -0.170346562325,0.912234042553,0.0322876498103 -0.170765790945,0.912234042553,0.0321065820754 -0.170765790945,0.914893617021,0.0320993773639 -0.173141419788,0.914893617021,0.0316329412162 -0.173420905534,0.914893617021,0.0316303148866 -0.173420905534,0.917553191489,0.0315392017365 -0.173840134153,0.917553191489,0.0315163135529 -0.173840134153,0.920212765957,0.0314948558807 -0.178311906093,0.920212765957,0.0304054673761 -0.178311906093,0.922872340426,0.030398292467 -0.187255449972,0.922872340426,0.0286690667272 -0.187255449972,0.928191489362,0.0285397898406 -0.191168250419,0.928191489362,0.027714766562 -0.191447736165,0.928191489362,0.0276669133455 -0.194661822247,0.928191489362,0.0268968436867 -0.194661822247,0.93085106383,0.0268879886717 -0.207518166574,0.93085106383,0.0249800942838 -0.20779765232,0.93085106383,0.0249694548547 -0.214924538849,0.93085106383,0.0239670034498 -0.214924538849,0.933510638298,0.0239541381598 -0.232532140861,0.933510638298,0.0217730533332 -0.232532140861,0.936170212766,0.0217149294913 -0.233230855226,0.936170212766,0.0216377712786 -0.233230855226,0.938829787234,0.0215987041593 -0.235326998323,0.938829787234,0.0213730819523 -0.235606484069,0.938829787234,0.0213605053723 -0.240497484628,0.938829787234,0.0208390112966 -0.240497484628,0.941489361702,0.0208286009729 -0.270681945221,0.941489361702,0.0175404846668 -0.270961430967,0.941489361702,0.0175240058452 -0.293180547792,0.941489361702,0.0153321614489 -0.293180547792,0.94414893617,0.0153201026842 -0.295136948016,0.94414893617,0.0151711003855 -0.295136948016,0.949468085106,0.015157263726 -0.296534376747,0.949468085106,0.0149736832827 -0.296534376747,0.954787234043,0.0148836160079 -0.31330352152,0.954787234043,0.0135203432292 -0.313583007267,0.954787234043,0.0135047240183 -0.32629960872,0.954787234043,0.0125663643703 -0.32629960872,0.957446808511,0.0125655969605 -0.349077697037,0.957446808511,0.0108915641904 -0.349357182784,0.957446808511,0.0108738066629 -0.349496925657,0.957446808511,0.010870013386 -0.349496925657,0.960106382979,0.0108604822308 -0.351593068753,0.960106382979,0.0107438350096 -0.351593068753,0.962765957447,0.0107424715534 -0.363890441587,0.962765957447,0.00993007514626 -0.363890441587,0.965425531915,0.00992914754897 -0.366964784796,0.965425531915,0.00961788836867 -0.366964784796,0.968085106383,0.00961527973413 -0.412241475685,0.968085106383,0.00682261260226 -0.412241475685,0.970744680851,0.00681434897706 -0.474287311347,0.970744680851,0.00428174575791 -0.474287311347,0.973404255319,0.00427702162415 -0.47442705422,0.973404255319,0.00426952214912 -0.47442705422,0.976063829787,0.00426776753739 -0.490357741755,0.976063829787,0.00370605476201 -0.490357741755,0.978723404255,0.00370509829372 -0.498043599776,0.978723404255,0.00347696058452 -0.498043599776,0.981382978723,0.0034729426261 -0.504192286193,0.981382978723,0.00334339239635 -0.504192286193,0.984042553191,0.0033431253396 -0.507685858021,0.984042553191,0.00325413467363 -0.507685858021,0.98670212766,0.00325242429972 -0.542062604807,0.98670212766,0.002510008635 -0.542062604807,0.989361702128,0.00250863679685 -0.561626607043,0.989361702128,0.00214650481939 -0.561626607043,0.992021276596,0.00214606616646 -0.564002235886,0.992021276596,0.00208524963818 -0.564002235886,0.994680851064,0.00208017346449 -0.568334264952,0.994680851064,0.00201129238121 -0.568334264952,0.997340425532,0.00201003090478 -0.622275013974,0.997340425532,0.00129171903245 -0.622833985467,0.997340425532,0.00128900515847 -0.641978759083,0.997340425532,0.00111989711877 -0.641978759083,1.0,0.00111590418965 -0.685438792622,1.0,0.00077383039752 -0.685718278368,1.0,0.000771555351093 -0.696897708217,1.0,0.000697182375006 -0.697177193963,1.0,0.000693261914421 -0.753773057574,1.0,0.000390878791222 -0.754332029067,1.0,0.000390189263271 -0.780324203466,1.0,0.000298931176076 -0.780603689212,1.0,0.000298746454064 -0.791084404695,1.0,0.000269930344075 -0.791503633315,1.0,0.000269200187176 -0.849217439911,1.0,0.000134454210638 -0.849496925657,1.0,0.000134396002977 -0.857043040805,1.0,0.000122760713566 -0.857322526551,1.0,0.000121779616165 -0.875908328675,1.0,9.21194587136e-05 -0.876187814421,1.0,9.21100945561e-05 -0.929849077697,1.0,3.43635547324e-05 -0.930128563443,1.0,3.42938255926e-05 -0.934320849637,1.0,3.1993848097e-05 -0.934600335383,1.0,3.19227256114e-05 -0.996646171045,1.0,7.59769136494e-07 -0.996925656792,1.0,6.99779377555e-07 -0.997903856903,1.0,4.9072565389e-07 -0.99818334265,1.0,4.48664820851e-07 -1.0,1.0,4.33395328514e-08 diff --git a/samples/core/visualization/roc.py b/samples/core/visualization/roc.py deleted file mode 100644 index b5da899e7f4..00000000000 --- a/samples/core/visualization/roc.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -from kfp.deprecated.components import create_component_from_func - -# Advanced function -# Demonstrates imports, helper functions and multiple outputs -from typing import NamedTuple - - -@create_component_from_func -def roc_visualization(roc_csv_uri: str='https://raw.githubusercontent.com/kubeflow/pipelines/master/samples/core/visualization/roc.csv') -> NamedTuple('VisualizationOutput', [('mlpipeline_ui_metadata', 'UI_metadata')]): - """Provide roc curve csv file to visualize as metrics.""" - import json - - metadata = { - 'outputs': [{ - 'type': 'roc', - 'format': 'csv', - 'schema': [ - {'name': 'fpr', 'type': 'NUMBER'}, - {'name': 'tpr', 'type': 'NUMBER'}, - {'name': 'thresholds', 'type': 'NUMBER'}, - ], - 'source': roc_csv_uri - }] - } - - from collections import namedtuple - visualization_output = namedtuple('VisualizationOutput', [ - 'mlpipeline_ui_metadata']) - return visualization_output(json.dumps(metadata)) - - -@dsl.pipeline( - name='roc-curve-pipeline', - description='A sample pipeline to generate ROC Curve for UI visualization.' -) -def roc_curve_pipeline(): - roc_visualization_task = roc_visualization() - # You can also upload samples/core/visualization/roc.csv to Google Cloud Storage. - # And call the component function with gcs path parameter like below: - # roc_visualization_task2 = roc_visualization('gs:////roc.csv') diff --git a/samples/core/visualization/roc_test.py b/samples/core/visualization/roc_test.py deleted file mode 100644 index 27b4330387e..00000000000 --- a/samples/core/visualization/roc_test.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp_server_api -import unittest -from pprint import pprint -from .roc import roc_curve_pipeline -from kfp.samples.test.utils import KfpMlmdClient, run_pipeline_func, TestCase - -import kfp.deprecated as kfp - - -def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): - t = unittest.TestCase() - t.maxDiff = None # we always want to see full diff - t.assertEqual(run.status, 'Succeeded') - client = KfpMlmdClient(mlmd_connection_config=mlmd_connection_config) - tasks = client.get_tasks(run_id=run.id) - pprint(tasks) - - roc_visualization = tasks['roc-visualization'] - output = [ - a for a in roc_visualization.outputs.artifacts - if a.name == 'mlpipeline_ui_metadata' - ][0] - pprint(output) - - t.assertEqual( - roc_visualization.get_dict()['outputs']['artifacts'][0]['name'], - 'mlpipeline_ui_metadata') - - -run_pipeline_func([ - TestCase( - pipeline_func=roc_curve_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY) -]) diff --git a/samples/core/visualization/table.csv b/samples/core/visualization/table.csv deleted file mode 100644 index f50eea5dd58..00000000000 --- a/samples/core/visualization/table.csv +++ /dev/null @@ -1,7 +0,0 @@ --122.403125209,37.7338352076,15 --122.423622537,37.7647573853,21 --122.501117615,37.760692028,3 --122.420529801,37.7110023301,18 --122.448578485,37.7697977169,7 --122.458220812,37.7633123961,14 -122.440383636,37.7713219756,1 diff --git a/samples/core/visualization/table.py b/samples/core/visualization/table.py deleted file mode 100644 index 72d06ce7388..00000000000 --- a/samples/core/visualization/table.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -from kfp.deprecated.components import create_component_from_func - -# Advanced function -# Demonstrates imports, helper functions and multiple outputs -from typing import NamedTuple - - -@create_component_from_func -def table_visualization(train_file_path: str = 'https://raw.githubusercontent.com/zijianjoy/pipelines/5651f41071816594b2ed27c88367f5efb4c60b50/samples/core/visualization/table.csv') -> NamedTuple('VisualizationOutput', [('mlpipeline_ui_metadata', 'UI_metadata')]): - """Provide number to visualize as table metrics.""" - import json - - header = ['Average precision ', 'Precision', 'Recall'] - metadata = { - 'outputs' : [{ - 'type': 'table', - 'storage': 'gcs', - 'format': 'csv', - 'header': header, - 'source': train_file_path - }] - } - - from collections import namedtuple - visualization_output = namedtuple('VisualizationOutput', [ - 'mlpipeline_ui_metadata']) - return visualization_output(json.dumps(metadata)) - - -@dsl.pipeline( - name='table-pipeline', - description='A sample pipeline to generate scalar metrics for UI visualization.' -) -def table_pipeline(): - table_visualization_task = table_visualization() - # You can also upload samples/core/visualization/table.csv to Google Cloud Storage. - # And call the component function with gcs path parameter like below: - # table_visualization_task = table_visualization('gs:////table.csv') diff --git a/samples/core/visualization/table_test.py b/samples/core/visualization/table_test.py deleted file mode 100644 index 5962cf06a3c..00000000000 --- a/samples/core/visualization/table_test.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp_server_api -import unittest -from pprint import pprint -from .table import table_pipeline -from kfp.samples.test.utils import KfpMlmdClient, run_pipeline_func, TestCase - -import kfp.deprecated as kfp - - -def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, - argo_workflow_name: str, **kwargs): - t = unittest.TestCase() - t.maxDiff = None # we always want to see full diff - t.assertEqual(run.status, 'Succeeded') - client = KfpMlmdClient(mlmd_connection_config=mlmd_connection_config) - tasks = client.get_tasks(argo_workflow_name=argo_workflow_name) - pprint(tasks) - - table_visualization = tasks['table-visualization'] - output = [ - a for a in table_visualization.outputs.artifacts - if a.name == 'mlpipeline_ui_metadata' - ][0] - pprint(output) - - t.assertEqual( - table_visualization.get_dict()['outputs']['artifacts'][0]['name'], - 'mlpipeline_ui_metadata') - - -run_pipeline_func([ - TestCase( - pipeline_func=table_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY) -]) diff --git a/samples/core/visualization/tensorboard_gcs.py b/samples/core/visualization/tensorboard_gcs.py deleted file mode 100644 index a347828ce4b..00000000000 --- a/samples/core/visualization/tensorboard_gcs.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated import dsl, components -from kfp.deprecated.components import create_component_from_func - -prepare_tensorboard = components.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1.5.0/components/tensorflow/tensorboard/prepare_tensorboard/component.yaml' -) - - -def train(log_dir: 'URI'): - # Reference: https://www.tensorflow.org/tensorboard/get_started - import tensorflow as tf - - mnist = tf.keras.datasets.mnist - - (x_train, y_train), (x_test, y_test) = mnist.load_data() - x_train, x_test = x_train / 255.0, x_test / 255.0 - - def create_model(): - return tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(512, activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10, activation='softmax') - ]) - - model = create_model() - model.compile( - optimizer='adam', - loss='sparse_categorical_crossentropy', - metrics=['accuracy'] - ) - - tensorboard_callback = tf.keras.callbacks.TensorBoard( - log_dir=log_dir, histogram_freq=1 - ) - - model.fit( - x=x_train, - y=y_train, - epochs=5, - validation_data=(x_test, y_test), - callbacks=[tensorboard_callback] - ) - - -# Be careful when choosing a tensorboard image: -# * tensorflow/tensorflow may fail with image pull backoff, because of dockerhub rate limiting. -# * tensorboard in tensorflow 2.3+ does not work with KFP, refer to https://github.com/kubeflow/pipelines/issues/5521. -train_op = create_component_from_func( - train, base_image='gcr.io/deeplearning-platform-release/tf2-cpu.2-4' -) - - -@dsl.pipeline(name='pipeline-tensorboard-gcs') -def my_pipeline( - log_dir=f'gs://{{kfp-default-bucket}}/tensorboard/logs/{dsl.RUN_ID_PLACEHOLDER}' -): - prepare_tb_task = prepare_tensorboard(log_dir_uri=log_dir) - tensorboard_task = train_op(log_dir=log_dir).after(prepare_tb_task) diff --git a/samples/core/visualization/tensorboard_minio.py b/samples/core/visualization/tensorboard_minio.py deleted file mode 100644 index db989a8bff6..00000000000 --- a/samples/core/visualization/tensorboard_minio.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import json -from kfp.deprecated.onprem import use_k8s_secret -from kfp.deprecated import dsl, components -from kfp.deprecated.components import OutputPath, create_component_from_func - - -prepare_tensorboard = components.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1b107eb4bb2510ecb99fd5f4fb438cbf7c96a87a/components/contrib/tensorflow/tensorboard/prepare_tensorboard/component.yaml' -) - - -def train(minio_endpoint: str, log_bucket: str, log_dir: str): - # Reference: https://www.tensorflow.org/tensorboard/get_started - import tensorflow as tf - - mnist = tf.keras.datasets.mnist - - (x_train, y_train), (x_test, y_test) = mnist.load_data() - x_train, x_test = x_train / 255.0, x_test / 255.0 - - def create_model(): - return tf.keras.models.Sequential([ - tf.keras.layers.Flatten(input_shape=(28, 28)), - tf.keras.layers.Dense(512, activation='relu'), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(10, activation='softmax') - ]) - - model = create_model() - model.compile( - optimizer='adam', - loss='sparse_categorical_crossentropy', - metrics=['accuracy'] - ) - - log_dir_local = "logs/fit" - tensorboard_callback = tf.keras.callbacks.TensorBoard( - log_dir=log_dir_local, histogram_freq=1 - ) - - model.fit( - x=x_train, - y=y_train, - epochs=5, - validation_data=(x_test, y_test), - callbacks=[tensorboard_callback] - ) - - # Copy the local logs folder to minio. - # - # TODO: we may write a filesystem watch process that continuously copy logs - # dir to minio, so that we can watch live training logs via tensorboard. - # - # Note, although tensorflow supports minio via s3:// protocol. We want to - # demo how minio can be used instead, e.g. the same approach can be used with - # frameworks only support local path. - from minio import Minio - import os - minio_access_key = os.getenv('MINIO_ACCESS_KEY') - minio_secret_key = os.getenv('MINIO_SECRET_KEY') - if not minio_access_key or not minio_secret_key: - raise Exception('MINIO_ACCESS_KEY or MINIO_SECRET_KEY env is not set') - client = Minio( - minio_endpoint, - access_key=minio_access_key, - secret_key=minio_secret_key, - secure=False - ) - count = 0 - from pathlib import Path - for path in Path("logs").rglob("*"): - if not path.is_dir(): - object_name = os.path.join( - log_dir, os.path.relpath(start=log_dir_local, path=path) - ) - client.fput_object( - bucket_name=log_bucket, - object_name=object_name, - file_path=path, - ) - count = count + 1 - print(f'{path} uploaded to minio://{log_bucket}/{object_name}') - print(f'{count} log files uploaded to minio://{log_bucket}/{log_dir}') - - -# tensorflow/tensorflow:2.4 may fail with image pull backoff, because of dockerhub rate limiting. -train_op = create_component_from_func( - train, - base_image='gcr.io/deeplearning-platform-release/tf2-cpu.2-3:latest', - packages_to_install=['minio'], # TODO: pin minio version -) - - -@dsl.pipeline(name='pipeline-tensorboard-minio') -def my_pipeline( - minio_endpoint: str = 'minio-service:9000', - log_bucket: str = 'mlpipeline', - log_dir: str = f'tensorboard/logs/{dsl.RUN_ID_PLACEHOLDER}', - # Pin to tensorflow 2.3, because in 2.4+ tensorboard cannot load in KFP: - # refer to https://github.com/kubeflow/pipelines/issues/5521. - tf_image: str = 'gcr.io/deeplearning-platform-release/tf2-cpu.2-3:latest' -): - # tensorboard uses s3 protocol to access minio - prepare_tb_task = prepare_tensorboard( - log_dir_uri=f's3://{log_bucket}/{log_dir}', - image=tf_image, - pod_template_spec=json.dumps({ - 'spec': { - 'containers': [{ - # These env vars make tensorboard access KFP in-cluster minio - # using s3 protocol. - # Reference: https://blog.min.io/hyper-scale-machine-learning-with-minio-and-tensorflow/ - 'env': [{ - 'name': 'AWS_ACCESS_KEY_ID', - 'valueFrom': { - 'secretKeyRef': { - 'name': 'mlpipeline-minio-artifact', - 'key': 'accesskey' - } - } - }, { - 'name': 'AWS_SECRET_ACCESS_KEY', - 'valueFrom': { - 'secretKeyRef': { - 'name': 'mlpipeline-minio-artifact', - 'key': 'secretkey' - } - } - }, { - 'name': 'AWS_REGION', - 'value': 'minio' - }, { - 'name': 'S3_ENDPOINT', - 'value': f'{minio_endpoint}', - }, { - 'name': 'S3_USE_HTTPS', - 'value': '0', - }, { - 'name': 'S3_VERIFY_SSL', - 'value': '0', - }] - }], - }, - }) - ) - train_task = train_op( - minio_endpoint=minio_endpoint, - log_bucket=log_bucket, - log_dir=log_dir, - ) - train_task.set_memory_request('2Gi').set_memory_limit('2Gi') - train_task.apply( - use_k8s_secret( - secret_name='mlpipeline-minio-artifact', - k8s_secret_key_to_env={ - 'secretkey': 'MINIO_SECRET_KEY', - 'accesskey': 'MINIO_ACCESS_KEY' - }, - ) - ) - # optional, let training task use the same tensorflow image as specified tensorboard - # this does not work in v2 compatible mode - # train_task.container.image = tf_image - train_task.after(prepare_tb_task) diff --git a/samples/core/visualization/tensorboard_minio_test.py b/samples/core/visualization/tensorboard_minio_test.py deleted file mode 100644 index 0a2a3d78f5f..00000000000 --- a/samples/core/visualization/tensorboard_minio_test.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -import json -from pprint import pprint -import kfp.deprecated as kfp -import kfp_server_api -from google.cloud import storage - -from .tensorboard_minio import my_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient - - -def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): - t = unittest.TestCase() - t.maxDiff = None # we always want to see full diff - t.assertEqual(run.status, 'Succeeded') - - client = KfpMlmdClient(mlmd_connection_config=mlmd_connection_config) - tasks = client.get_tasks(run_id=run.id) - # uncomment to debug - # pprint(tasks) - vis = tasks['create-tensorboard-visualization'] - pprint(vis) - mlpipeline_ui_metadata = vis.outputs.artifacts[0] - t.assertEqual(mlpipeline_ui_metadata.name, 'mlpipeline-ui-metadata') - - # download artifact content - storage_client = storage.Client() - blob = storage.Blob.from_string(mlpipeline_ui_metadata.uri, storage_client) - data = blob.download_as_text(storage_client) - data = json.loads(data) - t.assertTrue(data["outputs"][0]["source"]) - # source is a URI that is generated differently each run - data["outputs"][0]["source"] = "" - t.assertEqual( - { - "outputs": [{ - "type": - "tensorboard", - "source": - "", - "image": - "gcr.io/deeplearning-platform-release/tf2-cpu.2-3:latest", - "pod_template_spec": { - "spec": { - "containers": [{ - "env": [{ - "name": "AWS_ACCESS_KEY_ID", - "valueFrom": { - "secretKeyRef": { - "name": "mlpipeline-minio-artifact", - "key": "accesskey" - } - } - }, { - "name": "AWS_SECRET_ACCESS_KEY", - "valueFrom": { - "secretKeyRef": { - "name": "mlpipeline-minio-artifact", - "key": "secretkey" - } - } - }, { - "name": "AWS_REGION", - "value": "minio" - }, { - "name": "S3_ENDPOINT", - "value": "minio-service:9000" - }, { - "name": "S3_USE_HTTPS", - "value": "0" - }, { - "name": "S3_VERIFY_SSL", - "value": "0" - }] - }] - } - } - }] - }, data) - - -run_pipeline_func([ - TestCase( - pipeline_func=my_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), -]) diff --git a/samples/core/volume_snapshot_ops/README.md b/samples/core/volume_snapshot_ops/README.md deleted file mode 100644 index 68fcdf192bc..00000000000 --- a/samples/core/volume_snapshot_ops/README.md +++ /dev/null @@ -1,44 +0,0 @@ -## Simplify the creation of `VolumeSnapshot` instances - -**`VolumeSnapshotOp`:** A specified `ResourceOp` for `VolumeSnapshot` creation. - ---- - -**NOTE:** `VolumeSnapshot`s is an Alpha feature. -You should check if your Kubernetes cluster admin has them enabled. - ---- - -### Arguments: -The following arguments are an extension to the `ResourceOp` arguments. -If a `k8s_resource` is passed, then none of the following may be provided. - -* `resource_name`: The name of the resource which will be created. - This string will be prepended with the workflow name. - This may contain `PipelineParam`s. - (_required_) -* `pvc`: The name of the PVC to be snapshotted. - This may contain `PipelineParam`s. - (_optional_) -* `snapshot_class`: The snapshot storage class to be used. - This may contain `PipelineParam`s. - (_optional_) -* `volume`: An instance of a `V1Volume`, or its inherited type (e.g. `PipelineVolume`). - This may contain `PipelineParam`s. - (_optional_) -* `annotations`: Annotations to be patched in the `VolumeSnapshot`. - These may contain `PipelineParam`s. - (_optional_) - -**NOTE:** One of the `pvc` or `volume` needs to be provided. - -### Outputs -Additionally to the whole specification of the resource and its name (`ResourceOp` defaults), a -`VolumeSnapshotOp` also outputs the `restoreSize` of the bounded `VolumeSnapshot` (as -`step.outputs["size"]`). -This is the minimum size for a PVC created by that snapshot. - -### Useful attribute -The `VolumeSnapshotOp` step has a `.snapshot` attribute which is a `V1TypedLocalObjectReference`. -This can be passed as a `data_source` to create a PVC out of that `VolumeSnapshot`. -The user may otherwise use the `step.outputs["name"]` as `data_source`. diff --git a/samples/core/volume_snapshot_ops/volume_snapshot_ops.py b/samples/core/volume_snapshot_ops/volume_snapshot_ops.py deleted file mode 100644 index 8a6609e54b0..00000000000 --- a/samples/core/volume_snapshot_ops/volume_snapshot_ops.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated import dsl, compiler - - -@dsl.pipeline( - name="volume-snapshotop-sequential", - description="The fourth example of the design doc." -) -def volume_snapshotop_sequential(url): - vop = dsl.VolumeOp( - name="create_volume", - resource_name="vol1", - size="1Gi", - modes=dsl.VOLUME_MODE_RWM - ) - - step1 = dsl.ContainerOp( - name="step1_ingest", - image="google/cloud-sdk:279.0.0", - command=["sh", "-c"], - arguments=["mkdir /data/step1 && " - "gsutil cat %s | gzip -c >/data/step1/file1.gz" % url], - pvolumes={"/data": vop.volume} - ) - - step1_snap = dsl.VolumeSnapshotOp( - name="step1_snap", - resource_name="step1_snap", - volume=step1.pvolume - ) - - step2 = dsl.ContainerOp( - name="step2_gunzip", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["mkdir /data/step2 && " - "gunzip /data/step1/file1.gz -c >/data/step2/file1"], - pvolumes={"/data": step1.pvolume} - ) - - step2_snap = dsl.VolumeSnapshotOp( - name="step2_snap", - resource_name="step2_snap", - volume=step2.pvolume - ) - - step3 = dsl.ContainerOp( - name="step3_copy", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["mkdir /data/step3 && " - "cp -av /data/step2/file1 /data/step3/file3"], - pvolumes={"/data": step2.pvolume} - ) - - step3_snap = dsl.VolumeSnapshotOp( - name="step3_snap", - resource_name="step3_snap", - volume=step3.pvolume - ) - - step4 = dsl.ContainerOp( - name="step4_output", - image="library/bash:4.4.23", - command=["cat", "/data/step2/file1", "/data/step3/file3"], - pvolumes={"/data": step3.pvolume} - ) - -if __name__ == '__main__': - compiler.Compiler().compile(volume_snapshotop_sequential, __file__ + '.yaml') diff --git a/samples/core/volume_snapshot_ops/volume_snapshot_ops_test.py b/samples/core/volume_snapshot_ops/volume_snapshot_ops_test.py deleted file mode 100644 index 148f38c34ac..00000000000 --- a/samples/core/volume_snapshot_ops/volume_snapshot_ops_test.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.samples.test.utils import TestCase, relative_path, run_pipeline_func - -run_pipeline_func([ - TestCase( - pipeline_file=relative_path(__file__, 'volume_snapshot_ops.py'), - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - run_pipeline=False, - ), -]) diff --git a/samples/core/xgboost_training_cm/README.md b/samples/core/xgboost_training_cm/README.md deleted file mode 100644 index 0515f91dbf3..00000000000 --- a/samples/core/xgboost_training_cm/README.md +++ /dev/null @@ -1,79 +0,0 @@ -## Overview - -The `xgboost_training_cm.py` pipeline creates XGBoost models on structured data in CSV format. Both classification and regression are supported. - -The pipeline starts by creating a Google DataProc cluster, and then running analysis, transformation, distributed training and -prediction in the created cluster. -Then a single node confusion-matrix and ROC aggregator is used (for classification case) to -provide the confusion matrix data, and ROC data to the front end, respectively. -Finally, a delete cluster operation runs to destroy the cluster it creates -in the beginning. The delete cluster operation is used as an exit handler, meaning it will run regardless of whether the pipeline fails -or not. - -## Requirements - -> :warning: If you are using **full-scope** or **workload identity enabled** cluster in hosted pipeline beta version, **DO NOT** follow this section. However you'll still need to enable corresponding GCP API. - -Preprocessing uses Google Cloud DataProc. Therefore, you must enable the -[Cloud Dataproc API](https://pantheon.corp.google.com/apis/library/dataproc.googleapis.com?q=dataproc) for the given GCP project. This is the -general [guideline](https://cloud.google.com/endpoints/docs/openapi/enable-api) for enabling GCP APIs. -If KFP was deployed through K8S marketplace, please follow instructions in [the guideline](https://github.com/kubeflow/pipelines/blob/master/manifests/gcp_marketplace/guide.md#gcp-service-account-credentials) -to make sure the service account used has the role `storage.admin` and `dataproc.admin`. - -### Quota - -By default, Dataproc `create_cluster` creates a master instance of machine type 'n1-standard-4', -together with two worker instances of machine type 'n1-standard-4'. This sums up -to a request consuming 12.0 vCPU quota. The user GCP project needs to guarantee -this quota is available to make this sample work. - -> :warning: Free-tier GCP account might not be able to fulfill this quota requirement. For upgrading your account please follow [this link](). - -## Compile - -Follow the guide to [building a pipeline](https://www.kubeflow.org/docs/guides/pipelines/build-pipeline/) to install the Kubeflow Pipelines SDK and compile the sample Python into a workflow specification. The specification takes the form of a YAML file compressed into a `.zip` file. - -## Deploy - -Open the Kubeflow pipelines UI. Create a new pipeline, and then upload the compiled specification (`.zip` file) as a new pipeline template. - -## Run - -All arguments come with default values. This pipeline is preloaded as a Demo pipeline in Pipeline UI. You can run the pipeline without any changes. - -## Modifying the pipeline -To do additional exploration you may change some of the parameters, or pipeline input that is currently specified in the pipeline definition. - -* `output` is a Google Storage path which holds pipeline run results. -Note that each pipeline run will create a unique directory under `output` so it will not override previous results. -* `workers` is number of worker nodes used for this training. -* `rounds` is the number of XGBoost training iterations. Set the value to 200 to get a reasonable trained model. -* `train_data` points to a CSV file that contains the training data. For a sample see 'gs://ml-pipeline-playground/sfpd/train.csv'. -* `eval_data` points to a CSV file that contains the training data. For a sample see 'gs://ml-pipeline-playground/sfpd/eval.csv'. -* `schema` points to a schema file for train and eval datasets. For a sample see 'gs://ml-pipeline-playground/sfpd/schema.json'. - -## Components source - -Create Cluster: - [source code](https://github.com/kubeflow/pipelines/blob/release-1.7/components/gcp/container/component_sdk/python/kfp_component/google/dataproc/_create_cluster.py) - -Analyze (step one for preprocessing), Transform (step two for preprocessing) are using pyspark job -submission component, with - [source code](https://github.com/kubeflow/pipelines/blob/release-1.7/components/gcp/container/component_sdk/python/kfp_component/google/dataproc/_submit_pyspark_job.py) - -Distributed Training and predictions are using spark job submission component, with - [source code](https://github.com/kubeflow/pipelines/blob/release-1.7/components/gcp/container/component_sdk/python/kfp_component/google/dataproc/_submit_spark_job.py) - -Delete Cluster: - [source code](https://github.com/kubeflow/pipelines/blob/release-1.7/components/gcp/container/component_sdk/python/kfp_component/google/dataproc/_delete_cluster.py) - -The container file is located [here](https://github.com/kubeflow/pipelines/tree/release-1.7/components/gcp/container) - -For visualization, we use confusion matrix and ROC. -Confusion Matrix: - [source code](https://github.com/kubeflow/pipelines/tree/release-1.7/components/local/confusion_matrix/src), - [container](https://github.com/kubeflow/pipelines/tree/release-1.7/components/local/confusion_matrix) -and ROC: - [source code](https://github.com/kubeflow/pipelines/tree/release-1.7/components/local/roc/src), - [container](https://github.com/kubeflow/pipelines/tree/release-1.7/components/local/roc) - diff --git a/samples/core/xgboost_training_cm/xgboost_training_cm.py b/samples/core/xgboost_training_cm/xgboost_training_cm.py deleted file mode 100644 index 2e8de67ee07..00000000000 --- a/samples/core/xgboost_training_cm/xgboost_training_cm.py +++ /dev/null @@ -1,312 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import json -from kfp.deprecated import dsl, compiler, components - -import os -import subprocess - -diagnose_me_op = components.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/566dddfdfc0a6a725b6e50ea85e73d8d5578bbb9/components/diagnostics/diagnose_me/component.yaml') - -confusion_matrix_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1.8.0-alpha.0/components/local/confusion_matrix/component.yaml') - -roc_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/1.8.0-alpha.0/components/local/roc/component.yaml') - -dataproc_create_cluster_op = components.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1.7.0-rc.3/components/gcp/dataproc/create_cluster/component.yaml') - -dataproc_delete_cluster_op = components.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1.7.0-rc.3/components/gcp/dataproc/delete_cluster/component.yaml') - -dataproc_submit_pyspark_op = components.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1.7.0-rc.3/components/gcp/dataproc/submit_pyspark_job/component.yaml' -) - -dataproc_submit_spark_op = components.load_component_from_url( - 'https://raw.githubusercontent.com/kubeflow/pipelines/1.7.0-rc.3/components/gcp/dataproc/submit_spark_job/component.yaml' -) - -_PYSRC_PREFIX = 'gs://ml-pipeline/sample-pipeline/xgboost' # Common path to python src. - -_XGBOOST_PKG = 'gs://ml-pipeline/sample-pipeline/xgboost/xgboost4j-example-0.8-SNAPSHOT-jar-with-dependencies.jar' - -_TRAINER_MAIN_CLS = 'ml.dmlc.xgboost4j.scala.example.spark.XGBoostTrainer' - -_PREDICTOR_MAIN_CLS = 'ml.dmlc.xgboost4j.scala.example.spark.XGBoostPredictor' - - -def delete_directory_from_gcs(dir_path): - """Delete a GCS dir recursively. Ignore errors.""" - try: - subprocess.call(['gsutil', '-m', 'rm', '-r', dir_path]) - except: - pass - - -# ! Please do not forget to enable the Dataproc API in your cluster https://console.developers.google.com/apis/api/dataproc.googleapis.com/overview - -# ================================================================ -# The following classes should be provided by components provider. - - -def dataproc_analyze_op( - project, - region, - cluster_name, - schema, - train_data, - output): - """Submit dataproc analyze as a pyspark job. - - :param project: GCP project ID. - :param region: Which zone to run this analyze. - :param cluster_name: Name of the cluster. - :param schema: GCS path to the schema. - :param train_data: GCS path to the training data. - :param output: GCS path to store the output. - """ - return dataproc_submit_pyspark_op( - project_id=project, - region=region, - cluster_name=cluster_name, - main_python_file_uri=os.path.join(_PYSRC_PREFIX, 'analyze_run.py'), - args=['--output', str(output), '--train', str(train_data), '--schema', str(schema)] - ) - - -def dataproc_transform_op( - project, - region, - cluster_name, - train_data, - eval_data, - target, - analysis, - output -): - """Submit dataproc transform as a pyspark job. - - :param project: GCP project ID. - :param region: Which zone to run this analyze. - :param cluster_name: Name of the cluster. - :param train_data: GCS path to the training data. - :param eval_data: GCS path of the eval csv file. - :param target: Target column name. - :param analysis: GCS path of the analysis results - :param output: GCS path to use for output. - """ - - # Remove existing [output]/train and [output]/eval if they exist. - delete_directory_from_gcs(os.path.join(output, 'train')) - delete_directory_from_gcs(os.path.join(output, 'eval')) - - return dataproc_submit_pyspark_op( - project_id=project, - region=region, - cluster_name=cluster_name, - main_python_file_uri=os.path.join(_PYSRC_PREFIX, - 'transform_run.py'), - args=[ - '--output', - str(output), - '--analysis', - str(analysis), - '--target', - str(target), - '--train', - str(train_data), - '--eval', - str(eval_data) - ]) - - -def dataproc_train_op( - project, - region, - cluster_name, - train_data, - eval_data, - target, - analysis, - workers, - rounds, - output, - is_classification=True -): - - if is_classification: - config='gs://ml-pipeline/sample-data/xgboost-config/trainconfcla.json' - else: - config='gs://ml-pipeline/sample-data/xgboost-config/trainconfreg.json' - - return dataproc_submit_spark_op( - project_id=project, - region=region, - cluster_name=cluster_name, - main_class=_TRAINER_MAIN_CLS, - spark_job=json.dumps({'jarFileUris': [_XGBOOST_PKG]}), - args=json.dumps([ - str(config), - str(rounds), - str(workers), - str(analysis), - str(target), - str(train_data), - str(eval_data), - str(output) - ])) - - -def dataproc_predict_op( - project, - region, - cluster_name, - data, - model, - target, - analysis, - output -): - - return dataproc_submit_spark_op( - project_id=project, - region=region, - cluster_name=cluster_name, - main_class=_PREDICTOR_MAIN_CLS, - spark_job=json.dumps({'jarFileUris': [_XGBOOST_PKG]}), - args=json.dumps([ - str(model), - str(data), - str(analysis), - str(target), - str(output) - ])) - -# ======================================================================= - -@dsl.pipeline( - name='xgboost-trainer', - description='A trainer that does end-to-end distributed training for XGBoost models.' -) -def xgb_train_pipeline( - output='gs://{{kfp-default-bucket}}', - project='{{kfp-project-id}}', - diagnostic_mode='HALT_ON_ERROR', - rounds=5, -): - output_template = str(output) + '/' + dsl.RUN_ID_PLACEHOLDER + '/data' - region='us-central1' - workers=2 - quota_check=[{'region':region,'metric':'CPUS','quota_needed':12.0}] - train_data='gs://ml-pipeline/sample-data/sfpd/train.csv' - eval_data='gs://ml-pipeline/sample-data/sfpd/eval.csv' - schema='gs://ml-pipeline/sample-data/sfpd/schema.json' - true_label='ACTION' - target='resolution' - required_apis='dataproc.googleapis.com' - cluster_name='xgb-%s' % dsl.RUN_ID_PLACEHOLDER - - # Current GCP pyspark/spark op do not provide outputs as return values, instead, - # we need to use strings to pass the uri around. - analyze_output = output_template - transform_output_train = os.path.join(output_template, 'train', 'part-*') - transform_output_eval = os.path.join(output_template, 'eval', 'part-*') - train_output = os.path.join(output_template, 'train_output') - predict_output = os.path.join(output_template, 'predict_output') - - _diagnose_me_op = diagnose_me_op( - bucket=output, - execution_mode=diagnostic_mode, - project_id=project, - target_apis=required_apis, - quota_check=quota_check) - - with dsl.ExitHandler(exit_op=dataproc_delete_cluster_op( - project_id=project, - region=region, - name=cluster_name - )): - _create_cluster_op = dataproc_create_cluster_op( - project_id=project, - region=region, - name=cluster_name, - initialization_actions=[ - os.path.join(_PYSRC_PREFIX, - 'initialization_actions.sh'), - ], - image_version='1.5' - ).after(_diagnose_me_op) - - _analyze_op = dataproc_analyze_op( - project=project, - region=region, - cluster_name=cluster_name, - schema=schema, - train_data=train_data, - output=output_template - ).after(_create_cluster_op).set_display_name('Analyzer') - - _transform_op = dataproc_transform_op( - project=project, - region=region, - cluster_name=cluster_name, - train_data=train_data, - eval_data=eval_data, - target=target, - analysis=analyze_output, - output=output_template - ).after(_analyze_op).set_display_name('Transformer') - - _train_op = dataproc_train_op( - project=project, - region=region, - cluster_name=cluster_name, - train_data=transform_output_train, - eval_data=transform_output_eval, - target=target, - analysis=analyze_output, - workers=workers, - rounds=rounds, - output=train_output - ).after(_transform_op).set_display_name('Trainer') - - _predict_op = dataproc_predict_op( - project=project, - region=region, - cluster_name=cluster_name, - data=transform_output_eval, - model=train_output, - target=target, - analysis=analyze_output, - output=predict_output - ).after(_train_op).set_display_name('Predictor') - - _cm_op = confusion_matrix_op( - predictions=os.path.join(predict_output, 'part-*.csv'), - output_dir=output_template - ).after(_predict_op) - - _roc_op = roc_op( - predictions_dir=os.path.join(predict_output, 'part-*.csv'), - true_class=true_label, - true_score_column=true_label, - output_dir=output_template - ).after(_predict_op) - -if __name__ == '__main__': - compiler.Compiler().compile(xgb_train_pipeline, __file__ + '.yaml') diff --git a/samples/test/after_test.py b/samples/test/after_test.py index 04fdde001f3..a53410c64c3 100644 --- a/samples/test/after_test.py +++ b/samples/test/after_test.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase + from .after import my_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase run_pipeline_func([ - TestCase( - pipeline_func=my_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE), + TestCase(pipeline_func=my_pipeline), ]) diff --git a/samples/test/cache_v2_compatible_test.py b/samples/test/cache_v2_compatible_test.py deleted file mode 100644 index aac32d627a8..00000000000 --- a/samples/test/cache_v2_compatible_test.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Two step v2-compatible pipeline.""" - -# %% - -from __future__ import annotations - -import random -import string -import unittest -import functools - -import kfp_server_api - -from .two_step import two_step_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient, KfpTask -from ml_metadata.proto import Execution - - -def verify_tasks(t: unittest.TestCase, tasks: dict[str, KfpTask], task_state, - uri: str, some_int: int): - task_names = [*tasks.keys()] - t.assertCountEqual(task_names, ['train-op', 'preprocess'], 'task names') - - preprocess = tasks['preprocess'] - train = tasks['train-op'] - - t.assertEqual( - { - 'name': 'preprocess', - 'inputs': { - 'artifacts': [], - 'parameters': { - 'some_int': some_int, - 'uri': uri - } - }, - 'outputs': { - 'artifacts': [{ - 'metadata': { - 'display_name': 'output_dataset_one', - }, - 'name': 'output_dataset_one', - 'type': 'system.Dataset' - }], - 'parameters': { - 'output_parameter_one': some_int - } - }, - 'type': 'system.ContainerExecution', - 'state': task_state, - }, preprocess.get_dict()) - t.assertEqual( - { - 'name': 'train-op', - 'inputs': { - 'artifacts': [{ - 'metadata': { - 'display_name': 'output_dataset_one', - }, - 'name': 'dataset', - 'type': 'system.Dataset', - }], - 'parameters': { - 'num_steps': some_int - } - }, - 'outputs': { - 'artifacts': [{ - 'metadata': { - 'display_name': 'model', - }, - 'name': 'model', - 'type': 'system.Model', - }], - 'parameters': {} - }, - 'type': 'system.ContainerExecution', - 'state': task_state, - }, train.get_dict()) - - -def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, uri: str, - some_int, state: int, **kwargs): - t = unittest.TestCase() - t.maxDiff = None # we always want to see full diff - t.assertEqual(run.status, 'Succeeded') - client = KfpMlmdClient(mlmd_connection_config=mlmd_connection_config) - tasks = client.get_tasks(run_id=run.id) - verify_tasks(t, tasks, state, uri, some_int) - - -if __name__ == '__main__': - letters = string.ascii_lowercase - random_uri = 'http://' + ''.join(random.choice(letters) for i in range(5)) - random_int = random.randint(0, 10000) - # TODO: update to v2 engine test - # run_pipeline_func([ - # TestCase( - # pipeline_func=two_step_pipeline, - # arguments={ - # 'uri': f'{random_uri}', - # 'some_int': f'{random_int}' - # }, - # verify_func=functools.partial( - # verify, - # uri=random_uri, - # some_int=random_int, - # state=Execution.State.COMPLETE, - # ), - # mode=kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE, - # enable_caching=True - # ), - # ]), - # run_pipeline_func([ - # TestCase( - # pipeline_func=two_step_pipeline, - # arguments={ - # 'uri': f'{random_uri}', - # 'some_int': f'{random_int}' - # }, - # verify_func=functools.partial( - # verify, - # uri=random_uri, - # some_int=random_int, - # state=Execution.State.CACHED - # ), - # mode=kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE, - # enable_caching=True - # ), - # ]) - -# %% diff --git a/samples/test/fail.py b/samples/test/fail.py deleted file mode 100644 index 6533b2d8289..00000000000 --- a/samples/test/fail.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Fail pipeline.""" - -from kfp.deprecated import components, dsl - - -def fail(): - '''Fails''' - import sys - sys.exit(1) - - -fail_op = components.create_component_from_func( - fail, base_image='alpine:latest') - - -@dsl.pipeline(name='fail-pipeline') -def fail_pipeline(): - fail_task = fail_op() diff --git a/samples/test/fail_parameter_value_missing.py b/samples/test/fail_parameter_value_missing.py deleted file mode 100644 index 6e905837e45..00000000000 --- a/samples/test/fail_parameter_value_missing.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from kfp.deprecated import dsl, components - -echo = components.load_component_from_text( - """ -name: Echo -inputs: -- {name: text, type: String} -implementation: - container: - image: alpine - command: - - echo - - {inputValue: text} -""" -) - - -@dsl.pipeline(name='parameter_value_missing') -def pipeline( - parameter: - str # parameter should be specified when submitting, but we are missing it in the test -): - echo_op = echo(text=parameter) diff --git a/samples/test/fail_parameter_value_missing_test.py b/samples/test/fail_parameter_value_missing_test.py deleted file mode 100644 index 99c427e399a..00000000000 --- a/samples/test/fail_parameter_value_missing_test.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from .fail_parameter_value_missing import pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase - - -def verify(run, run_id: str, **kwargs): - assert run.status == 'Succeeded' - # TODO(Bobgy): should a pipeline fail when it is missing a required input? - # assert run.status == 'Failed' - - -run_pipeline_func([ - TestCase( - pipeline_func=pipeline, - verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY) -]) diff --git a/samples/test/fail_test.py b/samples/test/fail_test.py index ee7a489cf30..27f4ce11c35 100644 --- a/samples/test/fail_test.py +++ b/samples/test/fail_test.py @@ -1,4 +1,4 @@ -# Copyright 2021 The Kubeflow Authors +o # Copyright 2021 The Kubeflow Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,13 +14,19 @@ """Fail pipeline.""" from __future__ import annotations + import unittest -import kfp.deprecated as kfp + +import kfp +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TaskInputs +from kfp.samples.test.utils import TaskOutputs +from kfp.samples.test.utils import TestCase import kfp_server_api from ml_metadata.proto import Execution -from .fail import fail_pipeline + from .fail_v2 import fail_pipeline as fail_v2_pipeline -from kfp.samples.test.utils import TaskInputs, TaskOutputs, run_pipeline_func, TestCase, KfpTask def verify(run, **kwargs): @@ -50,9 +56,5 @@ def verify_v2(t: unittest.TestCase, run: kfp_server_api.ApiRun, TestCase( pipeline_func=fail_v2_pipeline, verify_func=verify_v2, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE), - TestCase( - pipeline_func=fail_pipeline, - verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY), + ), ]) diff --git a/samples/test/legacy_data_passing.py b/samples/test/legacy_data_passing.py deleted file mode 100644 index e462f9b3b24..00000000000 --- a/samples/test/legacy_data_passing.py +++ /dev/null @@ -1,151 +0,0 @@ -# This pipeline demonstrates and verifies all ways of data passing supported by KFP. - -# %% [markdown] -# KFP has simple data passing model. There are three different kinds of arguments and tw different ways to consume an input argument. All combinations are supported. - -# Any input can be consumed: -# * As value (`inputValue` placeholder) -# * As file (`inputPath` placeholder). - -# Input argument can come from: -# * Constant value -# * Pipeline parameter -# * Upstream component output - -# Combining these options there are 6 end-to-end data passing cases, each of which works regardless of type: - -# 1. Pass constant value which gets consumed as value. (Common) -# 2. Pass constant value which gets consumed as file. (Often used in test pipelines) -# 3. Pass pipeline parameter that gets consumed as value. (Common) -# 4. Pass pipeline parameter that gets consumed as file. (Rare. Sometimes used with JSON parameters) -# 5. Pass task output that gets consumed as value. (Common) -# 6. Pass task output that gets consumed as file. (Common) - -# The only restriction on types is that when both upstream output and downstream input have types, the types must match. - -# %% -import kfp.deprecated as kfp -from kfp.deprecated.components import create_component_from_func, InputPath, OutputPath - - -# Components -# Produce -@create_component_from_func -def produce_anything(data_path: OutputPath()): - with open(data_path, "w") as f: - f.write("produce_anything") - - -@create_component_from_func -def produce_something(data_path: OutputPath("Something")): - with open(data_path, "w") as f: - f.write("produce_something") - - -@create_component_from_func -def produce_something2() -> 'Something': - return "produce_something2" - - -@create_component_from_func -def produce_string() -> str: - return "produce_string" - - -# Consume as value -@create_component_from_func -def consume_anything_as_value(data): - print("consume_anything_as_value: " + data) - - -@create_component_from_func -def consume_something_as_value(data: "Something"): - print("consume_something_as_value: " + data) - - -@create_component_from_func -def consume_string_as_value(data: str): - print("consume_string_as_value: " + data) - - -# Consume as file -@create_component_from_func -def consume_anything_as_file(data_path: InputPath()): - with open(data_path) as f: - print("consume_anything_as_file: " + f.read()) - - -@create_component_from_func -def consume_something_as_file(data_path: InputPath('Something')): - with open(data_path) as f: - print("consume_something_as_file: " + f.read()) - - -@create_component_from_func -def consume_string_as_file(data_path: InputPath(str)): - with open(data_path) as f: - print("consume_string_as_file: " + f.read()) - - -# Pipeline -@kfp.dsl.pipeline(name='data_passing_pipeline') -def data_passing_pipeline( - anything_param="anything_param", - something_param: "Something" = "something_param", - string_param: str = "string_param", -): - produced_anything = produce_anything().output - produced_something = produce_something().output - produced_string = produce_string().output - - # Pass constant value; consume as value - consume_anything_as_value("constant") - consume_something_as_value("constant") - consume_string_as_value("constant") - - # Pass constant value; consume as file - consume_anything_as_file("constant") - consume_something_as_file("constant") - consume_string_as_file("constant") - - # Pass pipeline parameter; consume as value - consume_anything_as_value(anything_param) - consume_anything_as_value(something_param) - consume_anything_as_value(string_param) - consume_something_as_value(anything_param) - consume_something_as_value(something_param) - consume_string_as_value(anything_param) - consume_string_as_value(string_param) - - # Pass pipeline parameter; consume as file - consume_anything_as_file(anything_param) - consume_anything_as_file(something_param) - consume_anything_as_file(string_param) - consume_something_as_file(anything_param) - consume_something_as_file(something_param) - consume_string_as_file(anything_param) - consume_string_as_file(string_param) - - # Pass task output; consume as value - consume_anything_as_value(produced_anything) - consume_anything_as_value(produced_something) - consume_anything_as_value(produced_string) - consume_something_as_value(produced_anything) - consume_something_as_value(produced_something) - consume_string_as_value(produced_anything) - consume_string_as_value(produced_string) - - # Pass task output; consume as file - consume_anything_as_file(produced_anything) - consume_anything_as_file(produced_something) - consume_anything_as_file(produced_string) - consume_something_as_file(produced_anything) - consume_something_as_file(produced_something) - consume_string_as_file(produced_anything) - consume_string_as_file(produced_string) - - -if __name__ == "__main__": - kfp_endpoint = None - kfp.Client(host=kfp_endpoint).create_run_from_pipeline_func( - data_passing_pipeline, arguments={}) diff --git a/samples/test/legacy_data_passing_test.py b/samples/test/legacy_data_passing_test.py deleted file mode 100644 index 3952d024ebf..00000000000 --- a/samples/test/legacy_data_passing_test.py +++ /dev/null @@ -1,10 +0,0 @@ -from .legacy_data_passing import data_passing_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase -from kfp.deprecated.dsl import PipelineExecutionMode - -run_pipeline_func([ - TestCase( - pipeline_func=data_passing_pipeline, - mode=PipelineExecutionMode.V1_LEGACY, - ) -]) diff --git a/samples/test/legacy_exit_handler.py b/samples/test/legacy_exit_handler.py deleted file mode 100755 index df54f004855..00000000000 --- a/samples/test/legacy_exit_handler.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.deprecated import dsl, compiler - - -def gcs_download_op(url): - return dsl.ContainerOp( - name='GCS - Download', - image='google/cloud-sdk:279.0.0', - command=['sh', '-ce'], - arguments=['gsutil cp $0 $1 && cat $1', url, '/tmp/results.txt'], - file_outputs={ - 'data': '/tmp/results.txt', - } - ) - - -def echo_op(text): - return dsl.ContainerOp( - name='echo', - image='library/bash:4.4.23', - command=['sh', '-cex'], - arguments=['echo "$0"', text], - ) - - -@dsl.pipeline( - name='Exit Handler', - description= - 'Downloads a message and prints it. The exit handler will run after the pipeline finishes (successfully or not).' -) -def download_and_print(url='gs://ml-pipeline/shakespeare1.txt'): - """A sample pipeline showing exit handler.""" - - exit_task = echo_op('exit!') - - with dsl.ExitHandler(exit_task): - download_task = gcs_download_op(url) - echo_task = echo_op(download_task.output) - - -if __name__ == '__main__': - compiler.Compiler().compile(download_and_print, __file__ + '.yaml') diff --git a/samples/test/legacy_exit_handler_test.py b/samples/test/legacy_exit_handler_test.py deleted file mode 100755 index 51b22cf4c9e..00000000000 --- a/samples/test/legacy_exit_handler_test.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from .legacy_exit_handler import download_and_print -from kfp.samples.test.utils import run_pipeline_func, TestCase - -run_pipeline_func([ - # This sample is expected to not work on v2 compatible mode. - TestCase( - pipeline_func=download_and_print, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY) -]) diff --git a/samples/test/lightweight_python_functions_v2_pipeline_test.py b/samples/test/lightweight_python_functions_v2_pipeline_test.py index 60b5dd3ca71..1869ddfa543 100644 --- a/samples/test/lightweight_python_functions_v2_pipeline_test.py +++ b/samples/test/lightweight_python_functions_v2_pipeline_test.py @@ -12,14 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest from pprint import pprint +import unittest + +import kfp +from kfp.samples.test.utils import KfpMlmdClient +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase import kfp_server_api -import kfp.deprecated.dsl as dsl +from ml_metadata.proto import Execution from .lightweight_python_functions_v2_pipeline import pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient -from ml_metadata.proto import Execution def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): @@ -121,5 +124,5 @@ def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): TestCase( pipeline_func=pipeline, verify_func=verify, - mode=dsl.PipelineExecutionMode.V2_ENGINE), + ), ]) diff --git a/samples/test/lightweight_python_functions_v2_with_outputs_test.py b/samples/test/lightweight_python_functions_v2_with_outputs_test.py index 7d60c92b78c..616b1a148d7 100644 --- a/samples/test/lightweight_python_functions_v2_with_outputs_test.py +++ b/samples/test/lightweight_python_functions_v2_with_outputs_test.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os from pprint import pprint import unittest -import kfp.deprecated as kfp + +from kfp.samples.test.utils import KfpMlmdClient +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase import kfp_server_api -import os from minio import Minio from .lightweight_python_functions_v2_with_outputs import pipeline -from kfp.samples.test.utils import KfpMlmdClient, run_pipeline_func, TestCase def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): @@ -52,8 +54,5 @@ def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): run_pipeline_func([ - TestCase( - pipeline_func=pipeline, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - ), + TestCase(pipeline_func=pipeline,), ]) diff --git a/samples/test/metrics_visualization_v1.py b/samples/test/metrics_visualization_v1.py deleted file mode 100644 index 5170194edd5..00000000000 --- a/samples/test/metrics_visualization_v1.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -from ..core.visualization.confusion_matrix import confusion_visualization -from ..core.visualization.html import html_visualization -from ..core.visualization.markdown import markdown_visualization -from ..core.visualization.roc import roc_visualization -from ..core.visualization.table import table_visualization - -# Note: This test is to verify that visualization metrics on V1 is runnable by KFP. -# However, this pipeline is only runnable on V1 mode, but not V2 compatible mode. - -@dsl.pipeline( - name='metrics-visualization-v1-pipeline') -def metrics_visualization_v1_pipeline(): - confusion_visualization_task = confusion_visualization() - html_visualization_task = html_visualization("") - markdown_visualization_task = markdown_visualization() - roc_visualization_task = roc_visualization() - table_visualization_task = table_visualization() diff --git a/samples/test/metrics_visualization_v1_test.py b/samples/test/metrics_visualization_v1_test.py deleted file mode 100644 index 1fa1809f0c2..00000000000 --- a/samples/test/metrics_visualization_v1_test.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pprint import pprint -from .metrics_visualization_v1 import metrics_visualization_v1_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase - -import kfp.deprecated as kfp - -run_pipeline_func([ - TestCase( - pipeline_func=metrics_visualization_v1_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ) -]) diff --git a/samples/test/metrics_visualization_v2_test.py b/samples/test/metrics_visualization_v2_test.py index e30fc85b5a8..4f37869b114 100644 --- a/samples/test/metrics_visualization_v2_test.py +++ b/samples/test/metrics_visualization_v2_test.py @@ -13,14 +13,18 @@ # limitations under the License. from __future__ import annotations + import unittest import unittest.mock as mock -import kfp.deprecated as kfp + +import kfp +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase import kfp_server_api +from ml_metadata.proto import Execution from .metrics_visualization_v2 import metrics_visualization_pipeline -from kfp.samples.test.utils import KfpTask, run_pipeline_func, TestCase -from ml_metadata.proto import Execution def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, @@ -209,5 +213,5 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, TestCase( pipeline_func=metrics_visualization_pipeline, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE), + ), ]) diff --git a/samples/test/parameter_with_format.py b/samples/test/parameter_with_format.py deleted file mode 100644 index 6598bdcabe8..00000000000 --- a/samples/test/parameter_with_format.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated import components -from kfp.deprecated import dsl - - -@components.create_component_from_func -def print_op(name: str) -> str: - print(name) - return name - - -@dsl.pipeline(name='pipeline-with-pipelineparam-containing-format') -def my_pipeline(name: str = 'KFP'): - print_task = print_op('Hello {}'.format(name)) - print_op('{}, again.'.format(print_task.output)) diff --git a/samples/test/parameter_with_format_test.py b/samples/test/parameter_with_format_test.py deleted file mode 100644 index c42c737fa89..00000000000 --- a/samples/test/parameter_with_format_test.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from .parameter_with_format import my_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase - - -def verify(run, run_id: str, **kwargs): - assert run.status == 'Succeeded' - # TODO(Bobgy): verify output - - -run_pipeline_func([ - # Cannot test V2_ENGINE and V1_LEGACY using the same code. - # V2_ENGINE requires importing everything from v2 namespace. - # TestCase( - # pipeline_func=my_pipeline, - # verify_func=verify, - # mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - # ), - TestCase( - pipeline_func=my_pipeline, - verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), -]) diff --git a/samples/test/placeholder_concat_test.py b/samples/test/placeholder_concat_test.py index fe741e76c14..924217f4c12 100644 --- a/samples/test/placeholder_concat_test.py +++ b/samples/test/placeholder_concat_test.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase + from .placeholder_concat import pipeline_with_concat_placeholder -from kfp.samples.test.utils import run_pipeline_func, TestCase def verify(run, run_id: str, **kwargs): @@ -27,6 +29,5 @@ def verify(run, run_id: str, **kwargs): TestCase( pipeline_func=pipeline_with_concat_placeholder, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/test/placeholder_if.py b/samples/test/placeholder_if.py deleted file mode 100644 index b5bceec2fc3..00000000000 --- a/samples/test/placeholder_if.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2020,2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated import components -from kfp.deprecated import dsl - -component_op = components.load_component_from_text(''' -name: Component with optional inputs -inputs: -- {name: required_input, type: String, optional: false} -- {name: optional_input_1, type: String, optional: true} -- {name: optional_input_2, type: String, optional: true} -implementation: - container: - image: registry.k8s.io/busybox - command: - - echo - args: - - --arg0 - - {inputValue: required_input} - - if: - cond: - isPresent: optional_input_1 - then: - - --arg1 - - {inputValue: optional_input_1} - - if: - cond: - isPresent: optional_input_2 - then: - - --arg2 - - {inputValue: optional_input_2} - else: - - --arg2 - - 'default value' -''') - - -@dsl.pipeline(name='one-step-pipeline-with-if-placeholder-supply-both') -def pipeline_both(input0: str = 'input0', - input1: str = 'input1', - input2: str = 'input2'): - # supply both optional_input_1 and optional_input_2 - component = component_op( - required_input=input0, optional_input_1=input1, optional_input_2=input2) - - -@dsl.pipeline(name='one-step-pipeline-with-if-placeholder-supply-none') -def pipeline_none(input0: str = 'input0'): - # supply neither optional_input_1 nor optional_input_2 - # Note, KFP only supports compile-time optional arguments, e.g. it's not - # supported to write a pipeline that supplies both inputs and pass None - # at runtime -- in that case, the input arguments will be interpreted as - # the raw text "None". - component = component_op(required_input=input0) diff --git a/samples/test/placeholder_if_test.py b/samples/test/placeholder_if_test.py index 9669c7c868c..bed2b829e42 100644 --- a/samples/test/placeholder_if_test.py +++ b/samples/test/placeholder_if_test.py @@ -12,23 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import kfp.deprecated as kfp -from .placeholder_if import pipeline_both, pipeline_none -# from .placeholder_if_v2 import pipeline_both as pipeline_both_v2, pipeline_none as pipeline_none_v2 -from kfp.samples.test.utils import run_pipeline_func, TestCase +import kfp +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase + +from .placeholder_if_v2 import pipeline_both as pipeline_both_v2 +from .placeholder_if_v2 import pipeline_none as pipeline_none_v2 run_pipeline_func([ - # TODO(chesu): fix compile failure, https://github.com/kubeflow/pipelines/issues/6966 - # TestCase( - # pipeline_func=pipeline_none_v2, - # mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE), - # TestCase( - # pipeline_func=pipeline_both_v2, - # mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE), - TestCase( - pipeline_func=pipeline_none, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY), - TestCase( - pipeline_func=pipeline_both, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY), + TestCase(pipeline_func=pipeline_none_v2), + TestCase(pipeline_func=pipeline_both_v2), ]) diff --git a/samples/test/reused_component.py b/samples/test/reused_component.py deleted file mode 100644 index d2e710bc2ba..00000000000 --- a/samples/test/reused_component.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2020-2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated import components -from kfp.deprecated import dsl - -add_op = components.load_component_from_text( - ''' -name: Add -description: | - Component to add two numbers -inputs: -- name: op1 - type: Integer -- name: op2 - type: Integer -outputs: -- name: sum - type: Integer -implementation: - container: - image: google/cloud-sdk:latest - command: - - sh - - -c - - | - set -e -x - echo "$(($0+$1))" | gsutil cp - "$2" - - {inputValue: op1} - - {inputValue: op2} - - {outputPath: sum} -''' -) - - -@dsl.pipeline(name='add-pipeline') -def my_pipeline( - a: int = 2, - b: int = 5, -): - first_add_task = add_op(a, 3) - second_add_task = add_op(first_add_task.outputs['sum'], b) - third_add_task = add_op(second_add_task.outputs['sum'], 7) diff --git a/samples/test/reused_component_test.py b/samples/test/reused_component_test.py deleted file mode 100644 index 6864be8879e..00000000000 --- a/samples/test/reused_component_test.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -import kfp.deprecated as kfp -from .reused_component import my_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient - - -def verify(run, mlmd_connection_config, **kwargs): - t = unittest.TestCase() - t.maxDiff = None # we always want to see full diff - t.assertEqual(run.status, 'Succeeded') - - client = KfpMlmdClient(mlmd_connection_config=mlmd_connection_config) - - tasks = client.get_tasks(run_id=run.id) - t.assertEqual(17, tasks['add-3'].outputs.parameters['sum'], - 'add result should be 17') - - -run_pipeline_func([ - # Cannot test V2_ENGINE and V1_LEGACY using the same code. - # V2_ENGINE requires importing everything from v2 namespace. - # TestCase( - # pipeline_func=my_pipeline, - # verify_func=verify, - # mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE - # ), - TestCase( - pipeline_func=my_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY, - ), -]) diff --git a/samples/test/two_step.py b/samples/test/two_step.py deleted file mode 100644 index 00435fed51f..00000000000 --- a/samples/test/two_step.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Two step v2-compatible pipeline.""" - -from kfp.deprecated import components, dsl -from kfp.deprecated.components import InputPath, OutputPath - - -def preprocess( - uri: str, some_int: int, output_parameter_one: OutputPath(int), - output_dataset_one: OutputPath('Dataset') -): - with open(output_dataset_one, 'w') as f: - f.write(uri) - with open(output_parameter_one, 'w') as f: - f.write("{}".format(some_int)) - - -preprocess_op = components.create_component_from_func( - preprocess, base_image='python:3.9' -) - - -@components.create_component_from_func -def train_op( - dataset: InputPath('Dataset'), - model: OutputPath('Model'), - num_steps: int = 100 -): - '''Dummy Training Step.''' - - with open(dataset, 'r') as input_file: - input_string = input_file.read() - with open(model, 'w') as output_file: - for i in range(num_steps): - output_file.write( - "Step {}\n{}\n=====\n".format(i, input_string) - ) - - -@dsl.pipeline(name='two-step-pipeline') -def two_step_pipeline(uri: str = 'uri-to-import', some_int: int = 1234): - preprocess_task = preprocess_op(uri=uri, some_int=some_int) - train_task = train_op( - num_steps=preprocess_task.outputs['output_parameter_one'], - dataset=preprocess_task.outputs['output_dataset_one'] - ) diff --git a/samples/test/two_step_test.py b/samples/test/two_step_test.py deleted file mode 100644 index d8187e89aa2..00000000000 --- a/samples/test/two_step_test.py +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Two step v2-compatible pipeline.""" - -# %% - -from __future__ import annotations -import unittest -from pprint import pprint - -import kfp.deprecated as kfp -import kfp_server_api - -from .two_step import two_step_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient, KfpTask -from ml_metadata.proto import Execution - - -def verify_tasks(t: unittest.TestCase, tasks: dict[str, KfpTask]): - task_names = [*tasks.keys()] - t.assertCountEqual(task_names, ['train-op', 'preprocess'], 'task names') - - preprocess = tasks['preprocess'] - train = tasks['train-op'] - - pprint('======= preprocess task =======') - pprint(preprocess.get_dict()) - pprint('======= train task =======') - pprint(train.get_dict()) - pprint('==============') - - t.assertEqual( - { - 'name': 'preprocess', - 'inputs': { - 'artifacts': [], - 'parameters': { - 'some_int': 1234, - 'uri': 'uri-to-import' - } - }, - 'outputs': { - 'artifacts': [{ - 'metadata': { - 'display_name': 'output_dataset_one', - }, - 'name': 'output_dataset_one', - 'type': 'system.Dataset' - }], - 'parameters': { - 'output_parameter_one': 1234 - } - }, - 'type': 'system.ContainerExecution', - 'state': Execution.State.COMPLETE, - }, preprocess.get_dict()) - t.assertEqual( - { - 'name': 'train-op', - 'inputs': { - 'artifacts': [{ - 'metadata': { - 'display_name': 'output_dataset_one', - }, - 'name': 'dataset', - 'type': 'system.Dataset', - }], - 'parameters': { - 'num_steps': 1234 - } - }, - 'outputs': { - 'artifacts': [{ - 'metadata': { - 'display_name': 'model', - }, - 'name': 'model', - 'type': 'system.Model', - }], - 'parameters': {} - }, - 'type': 'system.ContainerExecution', - 'state': Execution.State.COMPLETE, - }, train.get_dict()) - - -def verify_artifacts(t: unittest.TestCase, tasks: dict, artifact_uri_prefix): - for task in tasks.values(): - for artifact in task.outputs.artifacts: - t.assertTrue(artifact.uri.startswith(artifact_uri_prefix)) - - -def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): - t = unittest.TestCase() - t.maxDiff = None # we always want to see full diff - t.assertEqual(run.status, 'Succeeded') - client = KfpMlmdClient(mlmd_connection_config=mlmd_connection_config) - tasks = client.get_tasks(run_id=run.id) - verify_tasks(t, tasks) - - -def verify_with_default_pipeline_root(run: kfp_server_api.ApiRun, - mlmd_connection_config, **kwargs): - t = unittest.TestCase() - t.maxDiff = None # we always want to see full diff - t.assertEqual(run.status, 'Succeeded') - client = KfpMlmdClient(mlmd_connection_config=mlmd_connection_config) - tasks = client.get_tasks(run_id=run.id) - verify_tasks(t, tasks) - verify_artifacts(t, tasks, 'minio://mlpipeline/v2/artifacts') - - -def verify_with_specific_pipeline_root(run: kfp_server_api.ApiRun, - mlmd_connection_config, **kwargs): - t = unittest.TestCase() - t.maxDiff = None # we always want to see full diff - t.assertEqual(run.status, 'Succeeded') - client = KfpMlmdClient(mlmd_connection_config=mlmd_connection_config) - tasks = client.get_tasks(run_id=run.id) - verify_tasks(t, tasks) - verify_artifacts(t, tasks, 'minio://mlpipeline/override/artifacts') - - -if __name__ == '__main__': - run_pipeline_func([ - TestCase( - pipeline_func=two_step_pipeline, - mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY), - # Cannot test V2_ENGINE and V1_LEGACY using the same code. - # V2_ENGINE requires importing everything from v2 namespace. - # TestCase( - # pipeline_func=two_step_pipeline, - # verify_func=verify, - # mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, - # ), - ]) - -# %% diff --git a/samples/test/two_step_with_uri_placeholder_test.py b/samples/test/two_step_with_uri_placeholder_test.py index a56b270d18d..215b778732f 100644 --- a/samples/test/two_step_with_uri_placeholder_test.py +++ b/samples/test/two_step_with_uri_placeholder_test.py @@ -13,16 +13,19 @@ # limitations under the License. """Two step v2-compatible pipeline with URI placeholders.""" -import unittest from pprint import pprint from typing import Dict +import unittest -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import KfpMlmdClient +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase import kfp_server_api +from ml_metadata.proto import Execution from .two_step_with_uri_placeholder import two_step_with_uri_placeholder -from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient, KfpTask -from ml_metadata.proto import Execution def verify_tasks(t: unittest.TestCase, tasks: Dict[str, KfpTask]): @@ -84,6 +87,5 @@ def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, **kwargs): TestCase( pipeline_func=two_step_with_uri_placeholder, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/test/utils/kfp/samples/test/utils.py b/samples/test/utils/kfp/samples/test/utils.py index 01ecf6749ed..0005f678d9b 100644 --- a/samples/test/utils/kfp/samples/test/utils.py +++ b/samples/test/utils/kfp/samples/test/utils.py @@ -14,32 +14,31 @@ from __future__ import annotations +from dataclasses import asdict +from dataclasses import dataclass import json import logging import os +from pprint import pprint import random import subprocess import sys import tempfile import time -import unittest -from dataclasses import asdict -from dataclasses import dataclass -from pprint import pprint from typing import Callable, Optional +import unittest +from google.protobuf.json_format import MessageToDict import kfp import kfp.compiler import kfp_server_api -import nbformat -from google.protobuf.json_format import MessageToDict -from kfp.deprecated.onprem import add_default_resource_spec from ml_metadata import metadata_store from ml_metadata.metadata_store.metadata_store import ListOptions from ml_metadata.proto import Event from ml_metadata.proto import Execution from ml_metadata.proto import metadata_store_pb2 from nbconvert import PythonExporter +import nbformat MINUTE = 60 @@ -80,7 +79,6 @@ class TestCase: pipeline_func: Optional[Callable] = None pipeline_file: Optional[str] = None pipeline_file_compile_path: Optional[str] = None - mode: kfp.deprecated.dsl.PipelineExecutionMode = kfp.deprecated.dsl.PipelineExecutionMode.V1_LEGACY enable_caching: bool = False arguments: Optional[dict[str, str]] = None verify_func: Verifier = _default_verify_func @@ -99,10 +97,8 @@ def run_pipeline_func(test_cases: list[TestCase]): raise ValueError('No test cases!') def test_wrapper( - run_pipeline: Callable[[ - Callable, str, str, kfp.deprecated.dsl - .PipelineExecutionMode, bool, dict, bool - ], kfp_server_api.ApiRunDetail], + run_pipeline: Callable[[Callable, str, str, bool, dict, bool], + kfp_server_api.ApiRunDetail], mlmd_connection_config: metadata_store_pb2.MetadataStoreClientConfig, ): for case in test_cases: @@ -129,21 +125,10 @@ def test_wrapper( 'TestCase.run_pipeline = False can only be specified when used together with pipeline_file.' ) - if case.mode == kfp.deprecated.dsl.PipelineExecutionMode.V2_COMPATIBLE: - print( - f'Unexpected v2 compatible mode test for: {pipeline_name}') - raise RuntimeError - - if case.mode == kfp.deprecated.dsl.PipelineExecutionMode.V2_ENGINE: - print(f'Running v2 engine mode test for: {pipeline_name}') - if case.mode == kfp.deprecated.dsl.PipelineExecutionMode.V1_LEGACY: - print(f'Running v1 legacy test for: {pipeline_name}') - run_detail = run_pipeline( pipeline_func=case.pipeline_func, pipeline_file=case.pipeline_file, pipeline_file_compile_path=case.pipeline_file_compile_path, - mode=case.mode, enable_caching=case.enable_caching, arguments=case.arguments or {}, dry_run=not case.run_pipeline, @@ -160,12 +145,6 @@ def test_wrapper( t.maxDiff = None # we always want to see full diff tasks = {} client = None - # we cannot stably use MLMD to query status in v1, because it may be async. - if case.mode == kfp.deprecated.dsl.PipelineExecutionMode.V2_ENGINE: - client = KfpMlmdClient( - mlmd_connection_config=mlmd_connection_config) - tasks = client.get_tasks(run_id=run_detail.run.id) - pprint(tasks) case.verify_func( run=run_detail.run, run_detail=run_detail, @@ -220,8 +199,6 @@ def _run_test(callback): def main( pipeline_root: Optional[str] = None, # example - launcher_v2_image: Optional[str] = None, - driver_image: Optional[str] = None, experiment: str = 'v2_sample_test_samples', metadata_service_host: Optional[str] = None, metadata_service_port: int = 8080, @@ -235,10 +212,6 @@ def main( :param pipeline_root: pipeline root that holds intermediate artifacts, example gs://your-bucket/path/to/workdir. :type pipeline_root: str, optional - :param launcher_v2_image: override launcher v2 image, only used in V2_ENGINE mode - :type launcher_v2_image: URI, optional - :param driver_image: override driver image, only used in V2_ENGINE mode - :type driver_image: URI, optional :param experiment: experiment the run is added to, defaults to 'v2_sample_test_samples' :type experiment: str, optional :param metadata_service_host: host for metadata grpc service, defaults to METADATA_GRPC_SERVICE_HOST or 'metadata-grpc-service' @@ -260,17 +233,7 @@ def main( metadata_service_host = os.getenv('METADATA_GRPC_SERVICE_HOST', 'metadata-grpc-service') logger.info(f'METADATA_GRPC_SERVICE_HOST={metadata_service_host}') - if launcher_v2_image is None: - launcher_v2_image = os.getenv('KFP_LAUNCHER_V2_IMAGE') - if not launcher_v2_image: - raise Exception("launcher_v2_image is empty") - logger.info(f'KFP_LAUNCHER_V2_IMAGE={launcher_v2_image}') - if driver_image is None: - driver_image = os.getenv('KFP_DRIVER_IMAGE') - if not driver_image: - raise Exception("driver_image is empty") - logger.info(f'KFP_DRIVER_IMAGE={driver_image}') - client = kfp.deprecated.Client() + client = kfp.Client() # TODO(Bobgy): avoid using private fields when getting loaded config kfp_endpoint = client._existing_config.host kfp_ui_endpoint = client._uihost @@ -282,8 +245,6 @@ def run_pipeline( pipeline_func: Optional[Callable], pipeline_file: Optional[str], pipeline_file_compile_path: Optional[str], - mode: kfp.deprecated.dsl.PipelineExecutionMode = kfp.deprecated.dsl - .PipelineExecutionMode.V2_ENGINE, enable_caching: bool = False, arguments: Optional[dict] = None, dry_run: bool = False, # just compile the pipeline without running it @@ -292,67 +253,16 @@ def run_pipeline( arguments = arguments or {} def _create_run(): - if mode == kfp.deprecated.dsl.PipelineExecutionMode.V2_ENGINE: - return run_v2_pipeline( - client=client, - fn=pipeline_func, - file=pipeline_file, - driver_image=driver_image, - launcher_v2_image=launcher_v2_image, - pipeline_root=pipeline_root, - enable_caching=enable_caching, - arguments={ - **arguments, - }, - ) - else: - conf = kfp.deprecated.dsl.PipelineConf() - conf.add_op_transformer( - # add a default resource request & limit to all container tasks - add_default_resource_spec( - cpu_request='0.5', - cpu_limit='1', - memory_limit='512Mi', - )) - if mode == kfp.deprecated.dsl.PipelineExecutionMode.V1_LEGACY: - conf.add_op_transformer(_disable_cache) - if pipeline_func: - return client.create_run_from_pipeline_func( - pipeline_func, - pipeline_conf=conf, - mode=mode, - arguments=arguments, - experiment_name=experiment, - ) - else: - pyfile = pipeline_file - if pipeline_file.endswith(".ipynb"): - pyfile = tempfile.mktemp( - suffix='.py', prefix="pipeline_py_code") - _nb_sample_to_py(pipeline_file, pyfile) - if dry_run: - subprocess.check_call([sys.executable, pyfile]) - return - package_path = None - if pipeline_file_compile_path: - subprocess.check_call([sys.executable, pyfile]) - package_path = pipeline_file_compile_path - else: - package_path = tempfile.mktemp( - suffix='.yaml', prefix="kfp_package") - from kfp.deprecated.compiler.main import \ - compile_pyfile - compile_pyfile( - pyfile=pyfile, - output_path=package_path, - mode=mode, - pipeline_conf=conf, - ) - return client.create_run_from_pipeline_package( - pipeline_file=package_path, - arguments=arguments, - experiment_name=experiment, - ) + return run_v2_pipeline( + client=client, + fn=pipeline_func, + file=pipeline_file, + pipeline_root=pipeline_root, + enable_caching=enable_caching, + arguments={ + **arguments, + }, + ) run_result = _retry_with_backoff(fn=_create_run) if dry_run: @@ -389,69 +299,30 @@ def _create_run(): def run_v2_pipeline( - client: kfp.deprecated.Client, + client: kfp.Client, fn: Optional[Callable], file: Optional[str], - driver_image: Optional[str], - launcher_v2_image: Optional[str], pipeline_root: Optional[str], enable_caching: bool, arguments: dict[str, str], ): - original_pipeline_spec = tempfile.mktemp( - suffix='.json', prefix="original_pipeline_spec") + pipeline_spec_file = tempfile.mktemp( + suffix='.yaml', prefix="original_pipeline_spec") if fn: kfp.compiler.Compiler().compile( - pipeline_func=fn, package_path=original_pipeline_spec) + pipeline_func=fn, package_path=pipeline_spec_file) else: pyfile = file if file.endswith(".ipynb"): pyfile = tempfile.mktemp(suffix='.py', prefix="pipeline_py_code") _nb_sample_to_py(file, pyfile) from kfp.cli.compile import dsl_compile - dsl_compile(py=pyfile, output=original_pipeline_spec) - - # remove following overriding logic once we use create_run_from_job_spec to trigger kfp pipeline run - with open(original_pipeline_spec) as f: - pipeline_job_dict = { - 'pipelineSpec': json.load(f), - 'runtimeConfig': {}, - } - - for component in [pipeline_job_dict['pipelineSpec']['root']] + list( - pipeline_job_dict['pipelineSpec']['components'].values()): - if 'dag' in component: - for task in component['dag']['tasks'].values(): - task['cachingOptions'] = {'enableCache': enable_caching} + dsl_compile(py=pyfile, output=pipeline_spec_file) - if arguments: - pipeline_job_dict['runtimeConfig']['parameterValues'] = {} - - for k, v in arguments.items(): - pipeline_job_dict['runtimeConfig']['parameterValues'][k] = v - - pipeline_job = tempfile.mktemp(suffix='.json', prefix="pipeline_job") - with open(pipeline_job, 'w') as f: - json.dump(pipeline_job_dict, f) - - argo_workflow_spec = tempfile.mktemp(suffix='.yaml') - with open(argo_workflow_spec, 'w') as f: - args = [ - 'kfp-v2-compiler', - '--job', - pipeline_job, - ] - if driver_image: - args += ['--driver', driver_image] - if launcher_v2_image: - args += ['--launcher', launcher_v2_image] - if pipeline_root: - args += ['--pipeline_root', pipeline_root] - # call v2 backend compiler CLI to compile pipeline spec to argo workflow - subprocess.check_call(args, stdout=f) return client.create_run_from_pipeline_package( - pipeline_file=argo_workflow_spec, + pipeline_file=pipeline_spec_file, arguments={}, + pipeline_root=pipeline_root, enable_caching=enable_caching) @@ -724,14 +595,6 @@ def _parse_parameters(execution: metadata_store_pb2.Execution) -> dict: return parameters -def _disable_cache(task): - # Skip tasks which are not container ops. - if not isinstance(task, kfp.deprecated.dsl.ContainerOp): - return task - task.execution_options.caching_strategy.max_cache_staleness = "P0D" - return task - - def _nb_sample_to_py(notebook_path: str, output_path: str): """nb_sample_to_py converts notebook kfp sample to a python file. diff --git a/samples/v2/cache_test.py b/samples/v2/cache_test.py index a1fb533b121..27f9c3ec679 100644 --- a/samples/v2/cache_test.py +++ b/samples/v2/cache_test.py @@ -17,17 +17,20 @@ from __future__ import annotations +import functools import random import string import unittest -import functools -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import KfpMlmdClient +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TestCase import kfp_server_api +from ml_metadata.proto import Execution from ..test.two_step import two_step_pipeline -from kfp.samples.test.utils import run_pipeline_func, TestCase, KfpMlmdClient, KfpTask -from ml_metadata.proto import Execution def verify_tasks(t: unittest.TestCase, tasks: dict[str, KfpTask], task_state, @@ -120,7 +123,6 @@ def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, uri: str, some_int=random_int, state=Execution.State.COMPLETE, ), - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, enable_caching=True), ]), run_pipeline_func([ @@ -135,7 +137,6 @@ def verify(run: kfp_server_api.ApiRun, mlmd_connection_config, uri: str, uri=random_uri, some_int=random_int, state=Execution.State.CACHED), - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, enable_caching=True), ]) diff --git a/samples/v2/component_with_optional_inputs_test.py b/samples/v2/component_with_optional_inputs_test.py index 5649bf62024..24fb854ede0 100644 --- a/samples/v2/component_with_optional_inputs_test.py +++ b/samples/v2/component_with_optional_inputs_test.py @@ -17,7 +17,7 @@ import unittest -import kfp.deprecated as kfp +import kfp from kfp.samples.test.utils import KfpTask from kfp.samples.test.utils import run_pipeline_func from kfp.samples.test.utils import TaskInputs @@ -48,11 +48,11 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, 'state': Execution.State.COMPLETE, }, component_op_dict) + if __name__ == '__main__': run_pipeline_func([ TestCase( pipeline_func=pipeline, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/v2/hello_world_test.py b/samples/v2/hello_world_test.py index dd6a2b548ee..4aad6efe4a3 100644 --- a/samples/v2/hello_world_test.py +++ b/samples/v2/hello_world_test.py @@ -17,11 +17,15 @@ import unittest -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TaskInputs +from kfp.samples.test.utils import TaskOutputs +from kfp.samples.test.utils import TestCase import kfp_server_api from ml_metadata.proto import Execution -from kfp.samples.test.utils import KfpTask, TaskInputs, TaskOutputs, TestCase, run_pipeline_func from .hello_world import pipeline_hello_world @@ -49,6 +53,5 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, TestCase( pipeline_func=pipeline_hello_world, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/v2/pipeline_container_no_input_test.py b/samples/v2/pipeline_container_no_input_test.py index f77572e53fe..2c750c18245 100644 --- a/samples/v2/pipeline_container_no_input_test.py +++ b/samples/v2/pipeline_container_no_input_test.py @@ -17,11 +17,15 @@ import unittest -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TaskInputs +from kfp.samples.test.utils import TaskOutputs +from kfp.samples.test.utils import TestCase import kfp_server_api from ml_metadata.proto import Execution -from kfp.samples.test.utils import KfpTask, TaskInputs, TaskOutputs, TestCase, run_pipeline_func from .pipeline_container_no_input import pipeline_container_no_input @@ -47,6 +51,5 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, TestCase( pipeline_func=pipeline_container_no_input, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/v2/pipeline_with_env_test.py b/samples/v2/pipeline_with_env_test.py index f203a98cf9d..ee0dc977ef7 100644 --- a/samples/v2/pipeline_with_env_test.py +++ b/samples/v2/pipeline_with_env_test.py @@ -17,11 +17,15 @@ import unittest -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TaskInputs +from kfp.samples.test.utils import TaskOutputs +from kfp.samples.test.utils import TestCase import kfp_server_api from ml_metadata.proto import Execution -from kfp.samples.test.utils import KfpTask, TaskInputs, TaskOutputs, TestCase, run_pipeline_func from .pipeline_with_env import pipeline_with_env @@ -35,20 +39,15 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, name='print-env-op', type='system.ContainerExecution', state=Execution.State.COMPLETE, - inputs=TaskInputs( - parameters={}, artifacts=[]), - outputs=TaskOutputs( - parameters={}, artifacts=[])), + inputs=TaskInputs(parameters={}, artifacts=[]), + outputs=TaskOutputs(parameters={}, artifacts=[])), 'check-env': KfpTask( name='check-env', type='system.ContainerExecution', state=Execution.State.COMPLETE, - inputs=TaskInputs( - parameters={}, artifacts=[]), - outputs=TaskOutputs( - parameters={}, artifacts=[])), - + inputs=TaskInputs(parameters={}, artifacts=[]), + outputs=TaskOutputs(parameters={}, artifacts=[])), }, tasks, ) @@ -59,6 +58,5 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, TestCase( pipeline_func=pipeline_with_env, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), - ]) \ No newline at end of file + ]) diff --git a/samples/v2/pipeline_with_importer_test.py b/samples/v2/pipeline_with_importer_test.py index 9725c9a78b5..8593afd507d 100644 --- a/samples/v2/pipeline_with_importer_test.py +++ b/samples/v2/pipeline_with_importer_test.py @@ -17,7 +17,7 @@ import unittest -import kfp.deprecated as kfp +import kfp from kfp.samples.test.utils import KfpTask from kfp.samples.test.utils import run_pipeline_func from kfp.samples.test.utils import TestCase @@ -99,6 +99,5 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, TestCase( pipeline_func=pipeline_with_importer, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/v2/pipeline_with_secret_as_env_test.py b/samples/v2/pipeline_with_secret_as_env_test.py index 8968e2116e4..5108522ef81 100644 --- a/samples/v2/pipeline_with_secret_as_env_test.py +++ b/samples/v2/pipeline_with_secret_as_env_test.py @@ -17,11 +17,15 @@ import unittest -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TaskInputs +from kfp.samples.test.utils import TaskOutputs +from kfp.samples.test.utils import TestCase import kfp_server_api from ml_metadata.proto import Execution -from kfp.samples.test.utils import KfpTask, TaskInputs, TaskOutputs, TestCase, run_pipeline_func from .pipeline_secret_env import pipeline_secret_env @@ -35,6 +39,5 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, TestCase( pipeline_func=pipeline_secret_env, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/v2/pipeline_with_secret_as_volume_test.py b/samples/v2/pipeline_with_secret_as_volume_test.py index ad4c24ea711..cb9a9e985c4 100644 --- a/samples/v2/pipeline_with_secret_as_volume_test.py +++ b/samples/v2/pipeline_with_secret_as_volume_test.py @@ -17,11 +17,15 @@ import unittest -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TaskInputs +from kfp.samples.test.utils import TaskOutputs +from kfp.samples.test.utils import TestCase import kfp_server_api from ml_metadata.proto import Execution -from kfp.samples.test.utils import KfpTask, TaskInputs, TaskOutputs, TestCase, run_pipeline_func from .pipeline_secret_volume import pipeline_secret_volume @@ -35,6 +39,5 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, TestCase( pipeline_func=pipeline_secret_volume, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/v2/pipeline_with_volume_test.py b/samples/v2/pipeline_with_volume_test.py index a9630c68f80..4602731f47e 100644 --- a/samples/v2/pipeline_with_volume_test.py +++ b/samples/v2/pipeline_with_volume_test.py @@ -15,11 +15,15 @@ import unittest -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TaskInputs +from kfp.samples.test.utils import TaskOutputs +from kfp.samples.test.utils import TestCase import kfp_server_api from ml_metadata.proto import Execution -from kfp.samples.test.utils import KfpTask, TaskInputs, TaskOutputs, TestCase, run_pipeline_func from .pipeline_with_volume import pipeline_with_volume @@ -27,11 +31,11 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, tasks: dict[str, KfpTask], **kwargs): t.assertEqual(run.status, 'Succeeded') + if __name__ == '__main__': run_pipeline_func([ TestCase( pipeline_func=pipeline_with_volume, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/v2/producer_consumer_param_test.py b/samples/v2/producer_consumer_param_test.py index 086e92dd22d..d569d6012aa 100644 --- a/samples/v2/producer_consumer_param_test.py +++ b/samples/v2/producer_consumer_param_test.py @@ -14,15 +14,21 @@ """Hello world v2 engine pipeline.""" from __future__ import annotations -import unittest + from pprint import pprint +import unittest -import kfp.deprecated as kfp +import kfp +from kfp.samples.test.utils import KfpMlmdClient +from kfp.samples.test.utils import KfpTask +from kfp.samples.test.utils import run_pipeline_func +from kfp.samples.test.utils import TaskInputs +from kfp.samples.test.utils import TaskOutputs +from kfp.samples.test.utils import TestCase import kfp_server_api +from ml_metadata.proto import Execution from .producer_consumer_param import producer_consumer_param_pipeline -from kfp.samples.test.utils import KfpTask, TaskInputs, TaskOutputs, run_pipeline_func, TestCase, KfpMlmdClient -from ml_metadata.proto import Execution def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, @@ -63,6 +69,5 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, TestCase( pipeline_func=producer_consumer_param_pipeline, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/samples/v2/two_step_pipeline_containerized_test.py b/samples/v2/two_step_pipeline_containerized_test.py index 9aa76556070..17923a696fb 100644 --- a/samples/v2/two_step_pipeline_containerized_test.py +++ b/samples/v2/two_step_pipeline_containerized_test.py @@ -17,7 +17,7 @@ import unittest -import kfp.deprecated as kfp +import kfp from kfp.samples.test.utils import KfpTask from kfp.samples.test.utils import run_pipeline_func from kfp.samples.test.utils import TaskInputs @@ -81,6 +81,5 @@ def verify(t: unittest.TestCase, run: kfp_server_api.ApiRun, TestCase( pipeline_func=two_step_pipeline_containerized, verify_func=verify, - mode=kfp.dsl.PipelineExecutionMode.V2_ENGINE, ), ]) diff --git a/sdk/CONTRIBUTING.md b/sdk/CONTRIBUTING.md index 01a523e7c8b..ad995901238 100644 --- a/sdk/CONTRIBUTING.md +++ b/sdk/CONTRIBUTING.md @@ -105,7 +105,7 @@ Please organize your imports using [isort](https://pycqa.github.io/isort/index.h From the project root, run the following code to format your code: ```sh -isort sdk/python --sg sdk/python/kfp/deprecated +isort sdk/python ``` #### Pylint [Encouraged] diff --git a/sdk/python/kfp/deprecated/__init__.py b/sdk/python/kfp/deprecated/__init__.py deleted file mode 100644 index 83acc09ba2d..00000000000 --- a/sdk/python/kfp/deprecated/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__version__ = '1.8.11' - -from . import components -from . import containers -from . import dsl -from . import auth -from ._client import Client -from ._config import * -from ._local_client import LocalClient -from ._runners import * diff --git a/sdk/python/kfp/deprecated/__main__.py b/sdk/python/kfp/deprecated/__main__.py deleted file mode 100644 index 48c8a0012b1..00000000000 --- a/sdk/python/kfp/deprecated/__main__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .cli.cli import main - -# TODO(hongyes): add more commands: -# kfp compile (migrate from dsl-compile) -# kfp experiment (manage experiments) - -if __name__ == '__main__': - main() diff --git a/sdk/python/kfp/deprecated/_auth.py b/sdk/python/kfp/deprecated/_auth.py deleted file mode 100644 index 5a8953fa222..00000000000 --- a/sdk/python/kfp/deprecated/_auth.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import os -import google.auth -import google.auth.app_engine -import google.auth.compute_engine.credentials -import google.auth.iam -from google.auth.transport.requests import Request -import google.oauth2.credentials -import google.oauth2.service_account -import requests_toolbelt.adapters.appengine -from webbrowser import open_new_tab -import requests -import json - -IAM_SCOPE = 'https://www.googleapis.com/auth/iam' -OAUTH_TOKEN_URI = 'https://www.googleapis.com/oauth2/v4/token' -LOCAL_KFP_CREDENTIAL = os.path.expanduser('~/.config/kfp/credentials.json') - - -def get_gcp_access_token(): - """Get and return GCP access token for the current Application Default - Credentials. - - If not set, returns None. For more information, see - https://cloud.google.com/sdk/gcloud/reference/auth/application-default/print-access-token - """ - token = None - try: - creds, project = google.auth.default( - scopes=["https://www.googleapis.com/auth/cloud-platform"]) - if not creds.valid: - auth_req = Request() - creds.refresh(auth_req) - if creds.valid: - token = creds.token - except Exception as e: - logging.warning('Failed to get GCP access token: %s', e) - return token - - -def get_auth_token(client_id, other_client_id, other_client_secret): - """Gets auth token from default service account or user account.""" - if os.path.exists(LOCAL_KFP_CREDENTIAL): - # fetch IAP auth token using the locally stored credentials. - with open(LOCAL_KFP_CREDENTIAL, 'r') as f: - credentials = json.load(f) - if client_id in credentials: - return id_token_from_refresh_token( - credentials[client_id]['other_client_id'], - credentials[client_id]['other_client_secret'], - credentials[client_id]['refresh_token'], client_id) - if other_client_id is None or other_client_secret is None: - # fetch IAP auth token: service accounts - token = get_auth_token_from_sa(client_id) - else: - # fetch IAP auth token: user account - # Obtain the ID token for provided Client ID with user accounts. - # Flow: get authorization code -> exchange for refresh token -> obtain and return ID token - refresh_token = get_refresh_token_from_client_id( - other_client_id, other_client_secret) - credentials = {} - if os.path.exists(LOCAL_KFP_CREDENTIAL): - with open(LOCAL_KFP_CREDENTIAL, 'r') as f: - credentials = json.load(f) - credentials[client_id] = {} - credentials[client_id]['other_client_id'] = other_client_id - credentials[client_id]['other_client_secret'] = other_client_secret - credentials[client_id]['refresh_token'] = refresh_token - #TODO: handle the case when the refresh_token expires. - # which only happens if the refresh_token is not used once for six months. - if not os.path.exists(os.path.dirname(LOCAL_KFP_CREDENTIAL)): - os.makedirs(os.path.dirname(LOCAL_KFP_CREDENTIAL)) - with open(LOCAL_KFP_CREDENTIAL, 'w') as f: - json.dump(credentials, f) - token = id_token_from_refresh_token(other_client_id, - other_client_secret, refresh_token, - client_id) - return token - - -def get_auth_token_from_sa(client_id): - """Gets auth token from default service account. - - If no service account credential is found, returns None. - """ - service_account_credentials = get_service_account_credentials(client_id) - if service_account_credentials: - return get_google_open_id_connect_token(service_account_credentials) - return None - - -def get_service_account_credentials(client_id): - # Figure out what environment we're running in and get some preliminary - # information about the service account. - bootstrap_credentials, _ = google.auth.default(scopes=[IAM_SCOPE]) - if isinstance(bootstrap_credentials, google.oauth2.credentials.Credentials): - logging.info('Found OAuth2 credentials and skip SA auth.') - return None - elif isinstance(bootstrap_credentials, google.auth.app_engine.Credentials): - requests_toolbelt.adapters.appengine.monkeypatch() - - # For service account's using the Compute Engine metadata service, - # service_account_email isn't available until refresh is called. - bootstrap_credentials.refresh(Request()) - signer_email = bootstrap_credentials.service_account_email - if isinstance(bootstrap_credentials, - google.auth.compute_engine.credentials.Credentials): - # Since the Compute Engine metadata service doesn't expose the service - # account key, we use the IAM signBlob API to sign instead. - # In order for this to work: - # - # 1. Your VM needs the https://www.googleapis.com/auth/iam scope. - # You can specify this specific scope when creating a VM - # through the API or gcloud. When using Cloud Console, - # you'll need to specify the "full access to all Cloud APIs" - # scope. A VM's scopes can only be specified at creation time. - # - # 2. The VM's default service account needs the "Service Account Actor" - # role. This can be found under the "Project" category in Cloud - # Console, or roles/iam.serviceAccountActor in gcloud. - signer = google.auth.iam.Signer(Request(), bootstrap_credentials, - signer_email) - else: - # A Signer object can sign a JWT using the service account's key. - signer = bootstrap_credentials.signer - - # Construct OAuth 2.0 service account credentials using the signer - # and email acquired from the bootstrap credentials. - return google.oauth2.service_account.Credentials( - signer, - signer_email, - token_uri=OAUTH_TOKEN_URI, - additional_claims={'target_audience': client_id}) - - -def get_google_open_id_connect_token(service_account_credentials): - """Get an OpenID Connect token issued by Google for the service account. - - This function: - 1. Generates a JWT signed with the service account's private key - containing a special "target_audience" claim. - 2. Sends it to the OAUTH_TOKEN_URI endpoint. Because the JWT in #1 - has a target_audience claim, that endpoint will respond with - an OpenID Connect token for the service account -- in other words, - a JWT signed by *Google*. The aud claim in this JWT will be - set to the value from the target_audience claim in #1. - For more information, see - https://developers.google.com/identity/protocols/OAuth2ServiceAccount . - The HTTP/REST example on that page describes the JWT structure and - demonstrates how to call the token endpoint. (The example on that page - shows how to get an OAuth2 access token; this code is using a - modified version of it to get an OpenID Connect token.) - """ - - service_account_jwt = ( - service_account_credentials._make_authorization_grant_assertion()) - request = google.auth.transport.requests.Request() - body = { - 'assertion': service_account_jwt, - 'grant_type': google.oauth2._client._JWT_GRANT_TYPE, - } - token_response = google.oauth2._client._token_endpoint_request( - request, OAUTH_TOKEN_URI, body) - return token_response['id_token'] - - -def get_refresh_token_from_client_id(client_id, client_secret): - """Obtain the ID token for provided Client ID with user accounts. - - Flow: get authorization code -> exchange for refresh token -> obtain and return ID token - """ - auth_code = get_auth_code(client_id) - return get_refresh_token_from_code(auth_code, client_id, client_secret) - - -def get_auth_code(client_id): - auth_url = "https://accounts.google.com/o/oauth2/v2/auth?client_id=%s&response_type=code&scope=openid%%20email&access_type=offline&redirect_uri=urn:ietf:wg:oauth:2.0:oob" % client_id - print(auth_url) - open_new_tab(auth_url) - return input( - "If there's no browser window prompt, please direct to the URL above, then copy and paste the authorization code here: " - ) - - -def get_refresh_token_from_code(auth_code, client_id, client_secret): - payload = { - "code": auth_code, - "client_id": client_id, - "client_secret": client_secret, - "redirect_uri": "urn:ietf:wg:oauth:2.0:oob", - "grant_type": "authorization_code" - } - res = requests.post(OAUTH_TOKEN_URI, data=payload) - res.raise_for_status() - return str(json.loads(res.text)[u"refresh_token"]) - - -def id_token_from_refresh_token(client_id, client_secret, refresh_token, - audience): - payload = { - "client_id": client_id, - "client_secret": client_secret, - "refresh_token": refresh_token, - "grant_type": "refresh_token", - "audience": audience - } - res = requests.post(OAUTH_TOKEN_URI, data=payload) - res.raise_for_status() - return str(json.loads(res.text)[u"id_token"]) diff --git a/sdk/python/kfp/deprecated/_client.py b/sdk/python/kfp/deprecated/_client.py deleted file mode 100644 index 5d57dfe7825..00000000000 --- a/sdk/python/kfp/deprecated/_client.py +++ /dev/null @@ -1,1529 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import time -import logging -import json -import os -import re -import tarfile -import tempfile -import warnings -import yaml -import zipfile -import datetime -import copy -from typing import Mapping, Callable, Optional - -import kfp_server_api - -from kfp.deprecated import dsl -from kfp.deprecated.compiler import compiler -from kfp.deprecated.compiler._k8s_helper import sanitize_k8s_name - -from kfp.deprecated._auth import get_auth_token, get_gcp_access_token -from kfp_server_api import ApiException - -# Operators on scalar values. Only applies to one of |int_value|, -# |long_value|, |string_value| or |timestamp_value|. -_FILTER_OPERATIONS = { - "UNKNOWN": 0, - "EQUALS": 1, - "NOT_EQUALS": 2, - "GREATER_THAN": 3, - "GREATER_THAN_EQUALS": 5, - "LESS_THAN": 6, - "LESS_THAN_EQUALS": 7 -} - - -def _add_generated_apis(target_struct, api_module, api_client): - """Initializes a hierarchical API object based on the generated API module. - - PipelineServiceApi.create_pipeline becomes - target_struct.pipelines.create_pipeline - """ - Struct = type('Struct', (), {}) - - def camel_case_to_snake_case(name): - import re - return re.sub('([a-z0-9])([A-Z])', r'\1_\2', name).lower() - - for api_name in dir(api_module): - if not api_name.endswith('ServiceApi'): - continue - - short_api_name = camel_case_to_snake_case( - api_name[0:-len('ServiceApi')]) + 's' - api_struct = Struct() - setattr(target_struct, short_api_name, api_struct) - service_api = getattr(api_module.api, api_name) - initialized_service_api = service_api(api_client) - for member_name in dir(initialized_service_api): - if member_name.startswith('_') or member_name.endswith( - '_with_http_info'): - continue - - bound_member = getattr(initialized_service_api, member_name) - setattr(api_struct, member_name, bound_member) - models_struct = Struct() - for member_name in dir(api_module.models): - if not member_name[0].islower(): - setattr(models_struct, member_name, - getattr(api_module.models, member_name)) - target_struct.api_models = models_struct - - -KF_PIPELINES_ENDPOINT_ENV = 'KF_PIPELINES_ENDPOINT' -KF_PIPELINES_UI_ENDPOINT_ENV = 'KF_PIPELINES_UI_ENDPOINT' -KF_PIPELINES_DEFAULT_EXPERIMENT_NAME = 'KF_PIPELINES_DEFAULT_EXPERIMENT_NAME' -KF_PIPELINES_OVERRIDE_EXPERIMENT_NAME = 'KF_PIPELINES_OVERRIDE_EXPERIMENT_NAME' -KF_PIPELINES_IAP_OAUTH2_CLIENT_ID_ENV = 'KF_PIPELINES_IAP_OAUTH2_CLIENT_ID' -KF_PIPELINES_APP_OAUTH2_CLIENT_ID_ENV = 'KF_PIPELINES_APP_OAUTH2_CLIENT_ID' -KF_PIPELINES_APP_OAUTH2_CLIENT_SECRET_ENV = 'KF_PIPELINES_APP_OAUTH2_CLIENT_SECRET' - - -class Client(object): - """API Client for KubeFlow Pipeline. - - Args: - host: The host name to use to talk to Kubeflow Pipelines. If not set, the in-cluster - service DNS name will be used, which only works if the current environment is a pod - in the same cluster (such as a Jupyter instance spawned by Kubeflow's - JupyterHub). - Set the host based on https://www.kubeflow.org/docs/components/pipelines/sdk/connect-api/. - client_id: The client ID used by Identity-Aware Proxy. - namespace: The namespace where the kubeflow pipeline system is run. - other_client_id: The client ID used to obtain the auth codes and refresh tokens. - Reference: https://cloud.google.com/iap/docs/authentication-howto#authenticating_from_a_desktop_app. - other_client_secret: The client secret used to obtain the auth codes and refresh tokens. - existing_token: Pass in token directly, it's used for cases better get token outside of SDK, e.x. GCP Cloud Functions - or caller already has a token - cookies: CookieJar object containing cookies that will be passed to the pipelines API. - proxy: HTTP or HTTPS proxy server - ssl_ca_cert: Cert for proxy - kube_context: String name of context within kubeconfig to use, defaults to the current-context set within kubeconfig. - credentials: A TokenCredentialsBase object which provides the logic to - populate the requests with credentials to authenticate against the API - server. - ui_host: Base url to use to open the Kubeflow Pipelines UI. This is used when running the client from a notebook to generate and - print links. - verify_ssl: A boolean indication to verify the servers TLS certificate or not. - """ - - # in-cluster DNS name of the pipeline service - IN_CLUSTER_DNS_NAME = 'ml-pipeline.{}.svc.cluster.local:8888' - KUBE_PROXY_PATH = 'api/v1/namespaces/{}/services/ml-pipeline:http/proxy/' - - # Auto populated path in pods - # https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod - # https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#serviceaccount-admission-controller - NAMESPACE_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/namespace' - - LOCAL_KFP_CONTEXT = os.path.expanduser('~/.config/kfp/context.json') - - # TODO: Wrap the configurations for different authentication methods. - def __init__(self, - host=None, - client_id=None, - namespace='kubeflow', - other_client_id=None, - other_client_secret=None, - existing_token=None, - cookies=None, - proxy=None, - ssl_ca_cert=None, - kube_context=None, - credentials=None, - ui_host=None, - verify_ssl=None): - """Create a new instance of kfp client.""" - host = host or os.environ.get(KF_PIPELINES_ENDPOINT_ENV) - self._uihost = os.environ.get(KF_PIPELINES_UI_ENDPOINT_ENV, ui_host or - host) - client_id = client_id or os.environ.get( - KF_PIPELINES_IAP_OAUTH2_CLIENT_ID_ENV) - other_client_id = other_client_id or os.environ.get( - KF_PIPELINES_APP_OAUTH2_CLIENT_ID_ENV) - other_client_secret = other_client_secret or os.environ.get( - KF_PIPELINES_APP_OAUTH2_CLIENT_SECRET_ENV) - config = self._load_config(host, client_id, namespace, other_client_id, - other_client_secret, existing_token, proxy, - ssl_ca_cert, kube_context, credentials, - verify_ssl) - # Save the loaded API client configuration, as a reference if update is - # needed. - self._load_context_setting_or_default() - - # If custom namespace provided, overwrite the loaded or default one in - # context settings for current client instance - if namespace != 'kubeflow': - self._context_setting['namespace'] = namespace - - self._existing_config = config - if cookies is None: - cookies = self._context_setting.get('client_authentication_cookie') - api_client = kfp_server_api.api_client.ApiClient( - config, - cookie=cookies, - header_name=self._context_setting.get( - 'client_authentication_header_name'), - header_value=self._context_setting.get( - 'client_authentication_header_value')) - _add_generated_apis(self, kfp_server_api, api_client) - self._job_api = kfp_server_api.api.job_service_api.JobServiceApi( - api_client) - self._run_api = kfp_server_api.api.run_service_api.RunServiceApi( - api_client) - self._experiment_api = kfp_server_api.api.experiment_service_api.ExperimentServiceApi( - api_client) - self._pipelines_api = kfp_server_api.api.pipeline_service_api.PipelineServiceApi( - api_client) - self._upload_api = kfp_server_api.api.PipelineUploadServiceApi( - api_client) - self._healthz_api = kfp_server_api.api.healthz_service_api.HealthzServiceApi( - api_client) - if not self._context_setting['namespace'] and self.get_kfp_healthz( - ).multi_user is True: - try: - with open(Client.NAMESPACE_PATH, 'r') as f: - current_namespace = f.read() - self.set_user_namespace(current_namespace) - except FileNotFoundError: - logging.info( - 'Failed to automatically set namespace.', exc_info=False) - - def _load_config(self, host, client_id, namespace, other_client_id, - other_client_secret, existing_token, proxy, ssl_ca_cert, - kube_context, credentials, verify_ssl): - config = kfp_server_api.configuration.Configuration() - - if proxy: - # https://github.com/kubeflow/pipelines/blob/c6ac5e0b1fd991e19e96419f0f508ec0a4217c29/backend/api/python_http_client/kfp_server_api/rest.py#L100 - config.proxy = proxy - if verify_ssl is not None: - config.verify_ssl = verify_ssl - - if ssl_ca_cert: - config.ssl_ca_cert = ssl_ca_cert - - host = host or '' - - # Defaults to 'https' if host does not contain 'http' or 'https' protocol. - if host and not host.startswith('http'): - warnings.warn( - 'The host %s does not contain the "http" or "https" protocol.' - ' Defaults to "https".' % host) - host = 'https://' + host - - # Preprocess the host endpoint to prevent some common user mistakes. - if not client_id: - # always preserving the protocol (http://localhost requires it) - host = host.rstrip('/') - - if host: - config.host = host - - token = None - - # "existing_token" is designed to accept token generated outside of SDK. Here is an example. - # - # https://cloud.google.com/functions/docs/securing/function-identity - # https://cloud.google.com/endpoints/docs/grpc/service-account-authentication - # - # import requests - # import kfp - # - # def get_access_token(): - # url = 'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token' - # r = requests.get(url, headers={'Metadata-Flavor': 'Google'}) - # r.raise_for_status() - # access_token = r.json()['access_token'] - # return access_token - # - # client = kfp.Client(host='', existing_token=get_access_token()) - # - if existing_token: - token = existing_token - self._is_refresh_token = False - elif client_id: - token = get_auth_token(client_id, other_client_id, - other_client_secret) - self._is_refresh_token = True - elif self._is_inverse_proxy_host(host): - token = get_gcp_access_token() - self._is_refresh_token = False - elif credentials: - config.api_key['authorization'] = 'placeholder' - config.api_key_prefix['authorization'] = 'Bearer' - config.refresh_api_key_hook = credentials.refresh_api_key_hook - - if token: - config.api_key['authorization'] = token - config.api_key_prefix['authorization'] = 'Bearer' - return config - - if host: - # if host is explicitly set with auth token, it's probably a port forward address. - return config - - import kubernetes as k8s - in_cluster = True - try: - k8s.config.load_incluster_config() - except: - in_cluster = False - - if in_cluster: - config.host = Client.IN_CLUSTER_DNS_NAME.format(namespace) - config = self._get_config_with_default_credentials(config) - return config - - try: - k8s.config.load_kube_config( - client_configuration=config, context=kube_context) - except: - print('Failed to load kube config.') - return config - - if config.host: - config.host = config.host + '/' + Client.KUBE_PROXY_PATH.format( - namespace) - return config - - def _is_inverse_proxy_host(self, host): - if host: - return re.match(r'\S+.googleusercontent.com/{0,1}$', host) - if re.match(r'\w+', host): - warnings.warn( - 'The received host is %s, please include the full endpoint address ' - '(with ".(pipelines/notebooks).googleusercontent.com")' % host) - return False - - def _is_ipython(self): - """Returns whether we are running in notebook.""" - try: - import IPython - ipy = IPython.get_ipython() - if ipy is None: - return False - except ImportError: - return False - - return True - - def _get_url_prefix(self): - if self._uihost: - # User's own connection. - if self._uihost.startswith('http://') or self._uihost.startswith( - 'https://'): - return self._uihost - else: - return 'http://' + self._uihost - - # In-cluster pod. We could use relative URL. - return '/pipeline' - - def _load_context_setting_or_default(self): - if os.path.exists(Client.LOCAL_KFP_CONTEXT): - with open(Client.LOCAL_KFP_CONTEXT, 'r') as f: - self._context_setting = json.load(f) - else: - self._context_setting = { - 'namespace': '', - } - - def _refresh_api_client_token(self): - """Refreshes the existing token associated with the kfp_api_client.""" - if getattr(self, '_is_refresh_token', None): - return - - new_token = get_gcp_access_token() - self._existing_config.api_key['authorization'] = new_token - - def _get_config_with_default_credentials(self, config): - """Apply default credentials to the configuration object. - - This method accepts a Configuration object and extends it with - some default credentials interface. - """ - # XXX: The default credentials are audience-based service account tokens - # projected by the kubelet (ServiceAccountTokenVolumeCredentials). As we - # implement more and more credentials, we can have some heuristic and - # choose from a number of options. - # See https://github.com/kubeflow/pipelines/pull/5287#issuecomment-805654121 - from kfp.deprecated import auth - credentials = auth.ServiceAccountTokenVolumeCredentials() - config_copy = copy.deepcopy(config) - - try: - credentials.refresh_api_key_hook(config_copy) - except Exception: - logging.warning("Failed to set up default credentials. Proceeding" - " without credentials...") - return config - - config.refresh_api_key_hook = credentials.refresh_api_key_hook - config.api_key_prefix['authorization'] = 'Bearer' - config.refresh_api_key_hook(config) - return config - - def set_user_namespace(self, namespace: str): - """Set user namespace into local context setting file. - - This function should only be used when Kubeflow Pipelines is in the multi-user mode. - - Args: - namespace: kubernetes namespace the user has access to. - """ - self._context_setting['namespace'] = namespace - if not os.path.exists(os.path.dirname(Client.LOCAL_KFP_CONTEXT)): - os.makedirs(os.path.dirname(Client.LOCAL_KFP_CONTEXT)) - with open(Client.LOCAL_KFP_CONTEXT, 'w') as f: - json.dump(self._context_setting, f) - - def get_kfp_healthz(self) -> kfp_server_api.ApiGetHealthzResponse: - """Gets healthz info of KFP deployment. - - Returns: - response: json formatted response from the healtz endpoint. - """ - count = 0 - response = None - max_attempts = 5 - while not response: - count += 1 - if count > max_attempts: - raise TimeoutError( - 'Failed getting healthz endpoint after {} attempts.'.format( - max_attempts)) - try: - response = self._healthz_api.get_healthz() - return response - # ApiException, including network errors, is the only type that may - # recover after retry. - except kfp_server_api.ApiException: - # logging.exception also logs detailed info about the ApiException - logging.exception( - 'Failed to get healthz info attempt {} of 5.'.format(count)) - time.sleep(5) - - def get_user_namespace(self) -> str: - """Get user namespace in context config. - - Returns: - namespace: kubernetes namespace from the local context file or empty if it wasn't set. - """ - return self._context_setting['namespace'] - - def create_experiment( - self, - name: str, - description: str = None, - namespace: str = None) -> kfp_server_api.ApiExperiment: - """Create a new experiment. - - Args: - name: The name of the experiment. - description: Description of the experiment. - namespace: Kubernetes namespace where the experiment should be created. - For single user deployment, leave it as None; - For multi user, input a namespace where the user is authorized. - - Returns: - An Experiment object. Most important field is id. - """ - namespace = namespace or self.get_user_namespace() - experiment = None - try: - experiment = self.get_experiment( - experiment_name=name, namespace=namespace) - except ValueError as error: - # Ignore error if the experiment does not exist. - if not str(error).startswith('No experiment is found with name'): - raise error - - if not experiment: - logging.info('Creating experiment {}.'.format(name)) - - resource_references = [] - if namespace: - key = kfp_server_api.models.ApiResourceKey( - id=namespace, - type=kfp_server_api.models.ApiResourceType.NAMESPACE) - reference = kfp_server_api.models.ApiResourceReference( - key=key, - relationship=kfp_server_api.models.ApiRelationship.OWNER) - resource_references.append(reference) - - experiment = kfp_server_api.models.ApiExperiment( - name=name, - description=description, - resource_references=resource_references) - experiment = self._experiment_api.create_experiment(body=experiment) - - if self._is_ipython(): - import IPython - html = \ - ('Experiment details.' - % (self._get_url_prefix(), experiment.id)) - IPython.display.display(IPython.display.HTML(html)) - return experiment - - def get_pipeline_id(self, name) -> Optional[str]: - """Find the id of a pipeline by name. - - Args: - name: Pipeline name. - - Returns: - Returns the pipeline id if a pipeline with the name exists. - """ - pipeline_filter = json.dumps({ - "predicates": [{ - "op": _FILTER_OPERATIONS["EQUALS"], - "key": "name", - "stringValue": name, - }] - }) - result = self._pipelines_api.list_pipelines(filter=pipeline_filter) - if result.pipelines is None: - return None - if len(result.pipelines) == 1: - return result.pipelines[0].id - elif len(result.pipelines) > 1: - raise ValueError( - "Multiple pipelines with the name: {} found, the name needs to be unique" - .format(name)) - return None - - def list_experiments( - self, - page_token: str = '', - page_size: int = 10, - sort_by: str = '', - namespace: Optional[str] = None, - filter: Optional[str] = None - ) -> kfp_server_api.ApiListExperimentsResponse: - """List experiments. - - Args: - page_token: Token for starting of the page. - page_size: Size of the page. - sort_by: Can be '[field_name]', '[field_name] desc'. For example, 'name desc'. - namespace: Kubernetes namespace where the experiment was created. - For single user deployment, leave it as None; - For multi user, input a namespace where the user is authorized. - filter: A url-encoded, JSON-serialized Filter protocol buffer - (see [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/filter.proto)). - - An example filter string would be: - - # For the list of filter operations please see: - # https://github.com/kubeflow/pipelines/blob/master/sdk/python/kfp/_client.py#L40 - json.dumps({ - "predicates": [{ - "op": _FILTER_OPERATIONS["EQUALS"], - "key": "name", - "stringValue": "my-name", - }] - }) - - Returns: - A response object including a list of experiments and next page token. - """ - namespace = namespace or self.get_user_namespace() - response = self._experiment_api.list_experiment( - page_token=page_token, - page_size=page_size, - sort_by=sort_by, - resource_reference_key_type=kfp_server_api.models.api_resource_type - .ApiResourceType.NAMESPACE, - resource_reference_key_id=namespace, - filter=filter) - return response - - def get_experiment(self, - experiment_id=None, - experiment_name=None, - namespace=None) -> kfp_server_api.ApiExperiment: - """Get details of an experiment. - - Either experiment_id or experiment_name is required - - Args: - experiment_id: Id of the experiment. (Optional) - experiment_name: Name of the experiment. (Optional) - namespace: Kubernetes namespace where the experiment was created. - For single user deployment, leave it as None; - For multi user, input the namespace where the user is authorized. - - Returns: - A response object including details of a experiment. - - Raises: - kfp_server_api.ApiException: If experiment is not found or None of the arguments is provided - """ - namespace = namespace or self.get_user_namespace() - if experiment_id is None and experiment_name is None: - raise ValueError( - 'Either experiment_id or experiment_name is required') - if experiment_id is not None: - return self._experiment_api.get_experiment(id=experiment_id) - experiment_filter = json.dumps({ - "predicates": [{ - "op": _FILTER_OPERATIONS["EQUALS"], - "key": "name", - "stringValue": experiment_name, - }] - }) - if namespace: - result = self._experiment_api.list_experiment( - filter=experiment_filter, - resource_reference_key_type=kfp_server_api.models - .api_resource_type.ApiResourceType.NAMESPACE, - resource_reference_key_id=namespace) - else: - result = self._experiment_api.list_experiment( - filter=experiment_filter) - if not result.experiments: - raise ValueError( - 'No experiment is found with name {}.'.format(experiment_name)) - if len(result.experiments) > 1: - raise ValueError( - 'Multiple experiments is found with name {}.'.format( - experiment_name)) - return result.experiments[0] - - def archive_experiment(self, experiment_id: str): - """Archive experiment. - - Args: - experiment_id: id of the experiment. - - Raises: - kfp_server_api.ApiException: If experiment is not found. - """ - self._experiment_api.archive_experiment(experiment_id) - - def delete_experiment(self, experiment_id): - """Delete experiment. - - Args: - experiment_id: id of the experiment. - - Returns: - Object. If the method is called asynchronously, returns the request thread. - - Raises: - kfp_server_api.ApiException: If experiment is not found. - """ - return self._experiment_api.delete_experiment(id=experiment_id) - - def _extract_pipeline_yaml(self, package_file): - - def _choose_pipeline_yaml_file(file_list) -> str: - yaml_files = [file for file in file_list if file.endswith('.yaml')] - if len(yaml_files) == 0: - raise ValueError( - 'Invalid package. Missing pipeline yaml file in the package.' - ) - - if 'pipeline.yaml' in yaml_files: - return 'pipeline.yaml' - else: - if len(yaml_files) == 1: - return yaml_files[0] - raise ValueError( - 'Invalid package. There is no pipeline.yaml file and there are multiple yaml files.' - ) - - if package_file.endswith('.tar.gz') or package_file.endswith('.tgz'): - with tarfile.open(package_file, "r:gz") as tar: - file_names = [member.name for member in tar if member.isfile()] - pipeline_yaml_file = _choose_pipeline_yaml_file(file_names) - with tar.extractfile(tar.getmember(pipeline_yaml_file)) as f: - return yaml.safe_load(f) - elif package_file.endswith('.zip'): - with zipfile.ZipFile(package_file, 'r') as zip: - pipeline_yaml_file = _choose_pipeline_yaml_file(zip.namelist()) - with zip.open(pipeline_yaml_file) as f: - return yaml.safe_load(f) - elif package_file.endswith('.yaml') or package_file.endswith('.yml'): - with open(package_file, 'r') as f: - return yaml.safe_load(f) - else: - raise ValueError( - 'The package_file ' + package_file + - ' should end with one of the following formats: [.tar.gz, .tgz, .zip, .yaml, .yml]' - ) - - def _override_caching_options(self, workflow: dict, enable_caching: bool): - templates = workflow['spec']['templates'] - for template in templates: - if 'metadata' in template \ - and 'labels' in template['metadata'] \ - and 'pipelines.kubeflow.org/enable_caching' in template['metadata']['labels']: - template['metadata']['labels'][ - 'pipelines.kubeflow.org/enable_caching'] = str( - enable_caching).lower() - - def list_pipelines( - self, - page_token: str = '', - page_size: int = 10, - sort_by: str = '', - filter: Optional[str] = None - ) -> kfp_server_api.ApiListPipelinesResponse: - """List pipelines. - - Args: - page_token: Token for starting of the page. - page_size: Size of the page. - sort_by: one of 'field_name', 'field_name desc'. For example, 'name desc'. - filter: A url-encoded, JSON-serialized Filter protocol buffer - (see [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/filter.proto)). - - An example filter string would be: - - # For the list of filter operations please see: - # https://github.com/kubeflow/pipelines/blob/master/sdk/python/kfp/_client.py#L40 - json.dumps({ - "predicates": [{ - "op": _FILTER_OPERATIONS["EQUALS"], - "key": "name", - "stringValue": "my-name", - }] - }) - - Returns: - A response object including a list of pipelines and next page token. - """ - return self._pipelines_api.list_pipelines( - page_token=page_token, - page_size=page_size, - sort_by=sort_by, - filter=filter) - - # TODO: provide default namespace, similar to kubectl default namespaces. - def run_pipeline( - self, - experiment_id: str, - job_name: str, - pipeline_package_path: Optional[str] = None, - params: Optional[dict] = None, - pipeline_id: Optional[str] = None, - version_id: Optional[str] = None, - pipeline_root: Optional[str] = None, - enable_caching: Optional[str] = None, - service_account: Optional[str] = None, - ) -> kfp_server_api.ApiRun: - """Run a specified pipeline. - - Args: - experiment_id: The id of an experiment. - job_name: Name of the job. - pipeline_package_path: Local path of the pipeline package(the filename should end with one of the following .tar.gz, .tgz, .zip, .yaml, .yml). - params: A dictionary with key (string) as param name and value (string) as as param value. - pipeline_id: The id of a pipeline. - version_id: The id of a pipeline version. - If both pipeline_id and version_id are specified, version_id will take precendence. - If only pipeline_id is specified, the default version of this pipeline is used to create the run. - pipeline_root: The root path of the pipeline outputs. This argument should - be used only for pipeline compiled with - dsl.PipelineExecutionMode.V2_COMPATIBLE or - dsl.PipelineExecutionMode.V2_ENGINGE mode. - enable_caching: Optional. Whether or not to enable caching for the run. - This setting affects v2 compatible mode and v2 mode only. - If not set, defaults to the compile time settings, which are True for all - tasks by default, while users may specify different caching options for - individual tasks. - If set, the setting applies to all tasks in the pipeline -- overrides - the compile time settings. - service_account: Optional. Specifies which Kubernetes service account this - run uses. - - Returns: - A run object. Most important field is id. - """ - if params is None: - params = {} - - if pipeline_root is not None: - params[dsl.ROOT_PARAMETER_NAME] = pipeline_root - - job_config = self._create_job_config( - experiment_id=experiment_id, - params=params, - pipeline_package_path=pipeline_package_path, - pipeline_id=pipeline_id, - version_id=version_id, - enable_caching=enable_caching, - ) - run_body = kfp_server_api.models.ApiRun( - pipeline_spec=job_config.spec, - resource_references=job_config.resource_references, - name=job_name, - service_account=service_account) - - response = self._run_api.create_run(body=run_body) - - if self._is_ipython(): - import IPython - html = ( - 'Run details.' - % (self._get_url_prefix(), response.run.id)) - IPython.display.display(IPython.display.HTML(html)) - return response.run - - def create_recurring_run( - self, - experiment_id: str, - job_name: str, - description: Optional[str] = None, - start_time: Optional[str] = None, - end_time: Optional[str] = None, - interval_second: Optional[int] = None, - cron_expression: Optional[str] = None, - max_concurrency: Optional[int] = 1, - no_catchup: Optional[bool] = None, - params: Optional[dict] = None, - pipeline_package_path: Optional[str] = None, - pipeline_id: Optional[str] = None, - version_id: Optional[str] = None, - enabled: bool = True, - enable_caching: Optional[bool] = None, - service_account: Optional[str] = None, - ) -> kfp_server_api.ApiJob: - """Create a recurring run. - - Args: - experiment_id: The string id of an experiment. - job_name: Name of the job. - description: An optional job description. - start_time: The RFC3339 time string of the time when to start the job. - end_time: The RFC3339 time string of the time when to end the job. - interval_second: Integer indicating the seconds between two recurring runs in for a periodic schedule. - cron_expression: A cron expression representing a set of times, using 6 space-separated fields, e.g. "0 0 9 ? * 2-6". - See `here `_ for details of the cron expression format. - max_concurrency: Integer indicating how many jobs can be run in parallel. - no_catchup: Whether the recurring run should catch up if behind schedule. - For example, if the recurring run is paused for a while and re-enabled - afterwards. If no_catchup=False, the scheduler will catch up on (backfill) each - missed interval. Otherwise, it only schedules the latest interval if more than one interval - is ready to be scheduled. - Usually, if your pipeline handles backfill internally, you should turn catchup - off to avoid duplicate backfill. (default: {False}) - pipeline_package_path: Local path of the pipeline package(the filename should end with one of the following .tar.gz, .tgz, .zip, .yaml, .yml). - params: A dictionary with key (string) as param name and value (string) as param value. - pipeline_id: The id of a pipeline. - version_id: The id of a pipeline version. - If both pipeline_id and version_id are specified, version_id will take precendence. - If only pipeline_id is specified, the default version of this pipeline is used to create the run. - enabled: A bool indicating whether the recurring run is enabled or disabled. - enable_caching: Optional. Whether or not to enable caching for the run. - This setting affects v2 compatible mode and v2 mode only. - If not set, defaults to the compile time settings, which are True for all - tasks by default, while users may specify different caching options for - individual tasks. - If set, the setting applies to all tasks in the pipeline -- overrides - the compile time settings. - service_account: Optional. Specifies which Kubernetes service account this - recurring run uses. - - Returns: - A Job object. Most important field is id. - - Raises: - ValueError: If required parameters are not supplied. - """ - - job_config = self._create_job_config( - experiment_id=experiment_id, - params=params, - pipeline_package_path=pipeline_package_path, - pipeline_id=pipeline_id, - version_id=version_id, - enable_caching=enable_caching, - ) - - if all([interval_second, cron_expression - ]) or not any([interval_second, cron_expression]): - raise ValueError( - 'Either interval_second or cron_expression is required') - if interval_second is not None: - trigger = kfp_server_api.models.ApiTrigger( - periodic_schedule=kfp_server_api.models.ApiPeriodicSchedule( - start_time=start_time, - end_time=end_time, - interval_second=interval_second)) - if cron_expression is not None: - trigger = kfp_server_api.models.ApiTrigger( - cron_schedule=kfp_server_api.models.ApiCronSchedule( - start_time=start_time, - end_time=end_time, - cron=cron_expression)) - - job_body = kfp_server_api.models.ApiJob( - enabled=enabled, - pipeline_spec=job_config.spec, - resource_references=job_config.resource_references, - name=job_name, - description=description, - no_catchup=no_catchup, - trigger=trigger, - max_concurrency=max_concurrency, - service_account=service_account) - return self._job_api.create_job(body=job_body) - - def _create_job_config( - self, - experiment_id: str, - params: Optional[dict], - pipeline_package_path: Optional[str], - pipeline_id: Optional[str], - version_id: Optional[str], - enable_caching: Optional[bool], - ): - """Create a JobConfig with spec and resource_references. - - Args: - experiment_id: The id of an experiment. - pipeline_package_path: Local path of the pipeline package(the filename should end with one of the following .tar.gz, .tgz, .zip, .yaml, .yml). - params: A dictionary with key (string) as param name and value (string) as param value. - pipeline_id: The id of a pipeline. - version_id: The id of a pipeline version. - If both pipeline_id and version_id are specified, version_id will take precendence. - If only pipeline_id is specified, the default version of this pipeline is used to create the run. - enable_caching: Whether or not to enable caching for the run. - This setting affects v2 compatible mode and v2 mode only. - If not set, defaults to the compile time settings, which are True for all - tasks by default, while users may specify different caching options for - individual tasks. - If set, the setting applies to all tasks in the pipeline -- overrides - the compile time settings. - - Returns: - A JobConfig object with attributes spec and resource_reference. - """ - - class JobConfig: - - def __init__(self, spec, resource_references): - self.spec = spec - self.resource_references = resource_references - - params = params or {} - pipeline_json_string = None - if pipeline_package_path: - pipeline_obj = self._extract_pipeline_yaml(pipeline_package_path) - - # Caching option set at submission time overrides the compile time settings. - if enable_caching is not None: - self._override_caching_options(pipeline_obj, enable_caching) - - pipeline_json_string = json.dumps(pipeline_obj) - api_params = [ - kfp_server_api.ApiParameter( - name=sanitize_k8s_name(name=k, allow_capital_underscore=True), - value=str(v) if type(v) not in (list, dict) else json.dumps(v)) - for k, v in params.items() - ] - resource_references = [] - key = kfp_server_api.models.ApiResourceKey( - id=experiment_id, - type=kfp_server_api.models.ApiResourceType.EXPERIMENT) - reference = kfp_server_api.models.ApiResourceReference( - key=key, relationship=kfp_server_api.models.ApiRelationship.OWNER) - resource_references.append(reference) - - if version_id: - key = kfp_server_api.models.ApiResourceKey( - id=version_id, - type=kfp_server_api.models.ApiResourceType.PIPELINE_VERSION) - reference = kfp_server_api.models.ApiResourceReference( - key=key, - relationship=kfp_server_api.models.ApiRelationship.CREATOR) - resource_references.append(reference) - - spec = kfp_server_api.models.ApiPipelineSpec( - pipeline_id=pipeline_id, - workflow_manifest=pipeline_json_string, - parameters=api_params) - return JobConfig(spec=spec, resource_references=resource_references) - - def create_run_from_pipeline_func( - self, - pipeline_func: Callable, - arguments: Mapping[str, str], - run_name: Optional[str] = None, - experiment_name: Optional[str] = None, - pipeline_conf: Optional[dsl.PipelineConf] = None, - namespace: Optional[str] = None, - mode: dsl.PipelineExecutionMode = dsl.PipelineExecutionMode.V1_LEGACY, - launcher_image: Optional[str] = None, - pipeline_root: Optional[str] = None, - enable_caching: Optional[bool] = None, - service_account: Optional[str] = None, - ): - """Runs pipeline on KFP-enabled Kubernetes cluster. - - This command compiles the pipeline function, creates or gets an experiment and submits the pipeline for execution. - - Args: - pipeline_func: A function that describes a pipeline by calling components and composing them into execution graph. - arguments: Arguments to the pipeline function provided as a dict. - run_name: Optional. Name of the run to be shown in the UI. - experiment_name: Optional. Name of the experiment to add the run to. - pipeline_conf: Optional. Pipeline configuration ops that will be applied - to all the ops in the pipeline func. - namespace: Kubernetes namespace where the pipeline runs are created. - For single user deployment, leave it as None; - For multi user, input a namespace where the user is authorized - mode: The PipelineExecutionMode to use when compiling and running - pipeline_func. - launcher_image: The launcher image to use if the mode is specified as - PipelineExecutionMode.V2_COMPATIBLE. Should only be needed for tests - or custom deployments right now. - pipeline_root: The root path of the pipeline outputs. This argument should - be used only for pipeline compiled with - dsl.PipelineExecutionMode.V2_COMPATIBLE or - dsl.PipelineExecutionMode.V2_ENGINGE mode. - enable_caching: Optional. Whether or not to enable caching for the run. - This setting affects v2 compatible mode and v2 mode only. - If not set, defaults to the compile time settings, which are True for all - tasks by default, while users may specify different caching options for - individual tasks. - If set, the setting applies to all tasks in the pipeline -- overrides - the compile time settings. - service_account: Optional. Specifies which Kubernetes service account this - run uses. - """ - if pipeline_root is not None and mode == dsl.PipelineExecutionMode.V1_LEGACY: - raise ValueError('`pipeline_root` should not be used with ' - 'dsl.PipelineExecutionMode.V1_LEGACY mode.') - - #TODO: Check arguments against the pipeline function - pipeline_name = pipeline_func.__name__ - run_name = run_name or pipeline_name + ' ' + datetime.datetime.now( - ).strftime('%Y-%m-%d %H-%M-%S') - with tempfile.TemporaryDirectory() as tmpdir: - pipeline_package_path = os.path.join(tmpdir, 'pipeline.yaml') - compiler.Compiler( - mode=mode, launcher_image=launcher_image).compile( - pipeline_func=pipeline_func, - package_path=pipeline_package_path, - pipeline_conf=pipeline_conf) - - return self.create_run_from_pipeline_package( - pipeline_file=pipeline_package_path, - arguments=arguments, - run_name=run_name, - experiment_name=experiment_name, - namespace=namespace, - pipeline_root=pipeline_root, - enable_caching=enable_caching, - service_account=service_account, - ) - - def create_run_from_pipeline_package( - self, - pipeline_file: str, - arguments: Mapping[str, str], - run_name: Optional[str] = None, - experiment_name: Optional[str] = None, - namespace: Optional[str] = None, - pipeline_root: Optional[str] = None, - enable_caching: Optional[bool] = None, - service_account: Optional[str] = None, - ): - """Runs pipeline on KFP-enabled Kubernetes cluster. - - This command takes a local pipeline package, creates or gets an experiment - and submits the pipeline for execution. - - Args: - pipeline_file: A compiled pipeline package file. - arguments: Arguments to the pipeline function provided as a dict. - run_name: Optional. Name of the run to be shown in the UI. - experiment_name: Optional. Name of the experiment to add the run to. - namespace: Kubernetes namespace where the pipeline runs are created. - For single user deployment, leave it as None; - For multi user, input a namespace where the user is authorized - pipeline_root: The root path of the pipeline outputs. This argument should - be used only for pipeline compiled with - dsl.PipelineExecutionMode.V2_COMPATIBLE or - dsl.PipelineExecutionMode.V2_ENGINGE mode. - enable_caching: Optional. Whether or not to enable caching for the run. - This setting affects v2 compatible mode and v2 mode only. - If not set, defaults to the compile time settings, which are True for all - tasks by default, while users may specify different caching options for - individual tasks. - If set, the setting applies to all tasks in the pipeline -- overrides - the compile time settings. - service_account: Optional. Specifies which Kubernetes service account this - run uses. - """ - - class RunPipelineResult: - - def __init__(self, client, run_info): - self._client = client - self.run_info = run_info - self.run_id = run_info.id - - def wait_for_run_completion(self, timeout=None): - timeout = timeout or datetime.timedelta.max - return self._client.wait_for_run_completion( - self.run_id, timeout) - - def __repr__(self): - return 'RunPipelineResult(run_id={})'.format(self.run_id) - - #TODO: Check arguments against the pipeline function - pipeline_name = os.path.basename(pipeline_file) - experiment_name = experiment_name or os.environ.get( - KF_PIPELINES_DEFAULT_EXPERIMENT_NAME, None) - overridden_experiment_name = os.environ.get( - KF_PIPELINES_OVERRIDE_EXPERIMENT_NAME, experiment_name) - if overridden_experiment_name != experiment_name: - import warnings - warnings.warn('Changing experiment name from "{}" to "{}".'.format( - experiment_name, overridden_experiment_name)) - experiment_name = overridden_experiment_name or 'Default' - run_name = run_name or ( - pipeline_name + ' ' + - datetime.datetime.now().strftime('%Y-%m-%d %H-%M-%S')) - experiment = self.create_experiment( - name=experiment_name, namespace=namespace) - run_info = self.run_pipeline( - experiment_id=experiment.id, - job_name=run_name, - pipeline_package_path=pipeline_file, - params=arguments, - pipeline_root=pipeline_root, - enable_caching=enable_caching, - service_account=service_account, - ) - return RunPipelineResult(self, run_info) - - def delete_job(self, job_id: str): - """Deletes a job. - - Args: - job_id: id of the job. - - Returns: - Object. If the method is called asynchronously, returns the request thread. - - Raises: - kfp_server_api.ApiException: If the job is not found. - """ - return self._job_api.delete_job(id=job_id) - - def disable_job(self, job_id: str): - """Disables a job. - - Args: - job_id: id of the job. - - Returns: - Object. If the method is called asynchronously, returns the request thread. - - Raises: - ApiException: If the job is not found. - """ - return self._job_api.disable_job(id=job_id) - - def enable_job(self, job_id: str): - """Enables a job. - - Args: - job_id: id of the job. - - Returns: - Object. If the method is called asynchronously, returns the request thread. - - Raises: - ApiException: If the job is not found. - """ - return self._job_api.enable_job(id=job_id) - - def list_runs( - self, - page_token: str = '', - page_size: int = 10, - sort_by: str = '', - experiment_id: Optional[str] = None, - namespace: Optional[str] = None, - filter: Optional[str] = None) -> kfp_server_api.ApiListRunsResponse: - """List runs, optionally can be filtered by experiment or namespace. - - Args: - page_token: Token for starting of the page. - page_size: Size of the page. - sort_by: One of 'field_name', 'field_name desc'. For example, 'name desc'. - experiment_id: Experiment id to filter upon - namespace: Kubernetes namespace to filter upon. - For single user deployment, leave it as None; - For multi user, input a namespace where the user is authorized. - filter: A url-encoded, JSON-serialized Filter protocol buffer - (see [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/filter.proto)). - - An example filter string would be: - - # For the list of filter operations please see: - # https://github.com/kubeflow/pipelines/blob/master/sdk/python/kfp/_client.py#L40 - json.dumps({ - "predicates": [{ - "op": _FILTER_OPERATIONS["EQUALS"], - "key": "name", - "stringValue": "my-name", - }] - }) - - Returns: - A response object including a list of experiments and next page token. - """ - namespace = namespace or self.get_user_namespace() - if experiment_id is not None: - response = self._run_api.list_runs( - page_token=page_token, - page_size=page_size, - sort_by=sort_by, - resource_reference_key_type=kfp_server_api.models - .api_resource_type.ApiResourceType.EXPERIMENT, - resource_reference_key_id=experiment_id, - filter=filter) - elif namespace: - response = self._run_api.list_runs( - page_token=page_token, - page_size=page_size, - sort_by=sort_by, - resource_reference_key_type=kfp_server_api.models - .api_resource_type.ApiResourceType.NAMESPACE, - resource_reference_key_id=namespace, - filter=filter) - else: - response = self._run_api.list_runs( - page_token=page_token, - page_size=page_size, - sort_by=sort_by, - filter=filter) - return response - - def list_recurring_runs( - self, - page_token: str = '', - page_size: int = 10, - sort_by: str = '', - experiment_id: Optional[str] = None, - filter: Optional[str] = None) -> kfp_server_api.ApiListJobsResponse: - """List recurring runs. - - Args: - page_token: Token for starting of the page. - page_size: Size of the page. - sort_by: One of 'field_name', 'field_name desc'. For example, 'name desc'. - experiment_id: Experiment id to filter upon. - filter: A url-encoded, JSON-serialized Filter protocol buffer - (see [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/filter.proto)). - - An example filter string would be: - - # For the list of filter operations please see: - # https://github.com/kubeflow/pipelines/blob/master/sdk/python/kfp/_client.py#L40 - json.dumps({ - "predicates": [{ - "op": _FILTER_OPERATIONS["EQUALS"], - "key": "name", - "stringValue": "my-name", - }] - }) - - Returns: - A response object including a list of recurring_runs and next page token. - """ - if experiment_id is not None: - response = self._job_api.list_jobs( - page_token=page_token, - page_size=page_size, - sort_by=sort_by, - resource_reference_key_type=kfp_server_api.models - .api_resource_type.ApiResourceType.EXPERIMENT, - resource_reference_key_id=experiment_id, - filter=filter) - else: - response = self._job_api.list_jobs( - page_token=page_token, - page_size=page_size, - sort_by=sort_by, - filter=filter) - return response - - def get_recurring_run(self, job_id: str) -> kfp_server_api.ApiJob: - """Get recurring_run details. - - Args: - job_id: id of the recurring_run. - - Returns: - A response object including details of a recurring_run. - - Raises: - kfp_server_api.ApiException: If recurring_run is not found. - """ - return self._job_api.get_job(id=job_id) - - def get_run(self, run_id: str) -> kfp_server_api.ApiRun: - """Get run details. - - Args: - run_id: id of the run. - - Returns: - A response object including details of a run. - - Raises: - kfp_server_api.ApiException: If run is not found. - """ - return self._run_api.get_run(run_id=run_id) - - def wait_for_run_completion(self, run_id: str, timeout: int): - """Waits for a run to complete. - - Args: - run_id: Run id, returned from run_pipeline. - timeout: Timeout in seconds. - - Returns: - A run detail object: Most important fields are run and pipeline_runtime. - - Raises: - TimeoutError: if the pipeline run failed to finish before the specified timeout. - """ - status = 'Running:' - start_time = datetime.datetime.now() - if isinstance(timeout, datetime.timedelta): - timeout = timeout.total_seconds() - is_valid_token = False - while (status is None or status.lower() - not in ['succeeded', 'failed', 'skipped', 'error']): - try: - get_run_response = self._run_api.get_run(run_id=run_id) - is_valid_token = True - except ApiException as api_ex: - # if the token is valid but receiving 401 Unauthorized error - # then refresh the token - if is_valid_token and api_ex.status == 401: - logging.info('Access token has expired !!! Refreshing ...') - self._refresh_api_client_token() - continue - else: - raise api_ex - status = get_run_response.run.status - elapsed_time = (datetime.datetime.now() - - start_time).total_seconds() - logging.info('Waiting for the job to complete...') - if elapsed_time > timeout: - raise TimeoutError('Run timeout') - time.sleep(5) - return get_run_response - - def _get_workflow_json(self, run_id): - """Get the workflow json. - - Args: - run_id: run id, returned from run_pipeline. - - Returns: - workflow: Json workflow - """ - get_run_response = self._run_api.get_run(run_id=run_id) - workflow = get_run_response.pipeline_runtime.workflow_manifest - workflow_json = json.loads(workflow) - return workflow_json - - def upload_pipeline( - self, - pipeline_package_path: str = None, - pipeline_name: str = None, - description: str = None, - ) -> kfp_server_api.ApiPipeline: - """Uploads the pipeline to the Kubeflow Pipelines cluster. - - Args: - pipeline_package_path: Local path to the pipeline package. - pipeline_name: Optional. Name of the pipeline to be shown in the UI. - description: Optional. Description of the pipeline to be shown in the UI. - - Returns: - Server response object containing pipleine id and other information. - """ - - response = self._upload_api.upload_pipeline( - pipeline_package_path, name=pipeline_name, description=description) - if self._is_ipython(): - import IPython - html = 'Pipeline details.' % ( - self._get_url_prefix(), response.id) - IPython.display.display(IPython.display.HTML(html)) - return response - - def upload_pipeline_version( - self, - pipeline_package_path, - pipeline_version_name: str, - pipeline_id: Optional[str] = None, - pipeline_name: Optional[str] = None, - description: Optional[str] = None, - ) -> kfp_server_api.ApiPipelineVersion: - """Uploads a new version of the pipeline to the Kubeflow Pipelines - cluster. - - Args: - pipeline_package_path: Local path to the pipeline package. - pipeline_version_name: Name of the pipeline version to be shown in the UI. - pipeline_id: Optional. Id of the pipeline. - pipeline_name: Optional. Name of the pipeline. - description: Optional. Description of the pipeline version to be shown in the UI. - - Returns: - Server response object containing pipleine id and other information. - - Raises: - ValueError when none or both of pipeline_id or pipeline_name are specified - kfp_server_api.ApiException: If pipeline id is not found. - """ - - if all([pipeline_id, pipeline_name - ]) or not any([pipeline_id, pipeline_name]): - raise ValueError('Either pipeline_id or pipeline_name is required') - - if pipeline_name: - pipeline_id = self.get_pipeline_id(pipeline_name) - kwargs = dict( - name=pipeline_version_name, - pipelineid=pipeline_id, - ) - - if description: - kwargs['description'] = description - try: - response = self._upload_api.upload_pipeline_version( - pipeline_package_path, **kwargs) - except kfp_server_api.exceptions.ApiTypeError as e: - # ToDo: Remove this once we drop support for kfp_server_api < 1.7.1 - if 'description' in e.message and 'unexpected keyword argument' in e.message: - raise NotImplementedError( - 'Pipeline version description is not supported in current kfp-server-api pypi package. Upgrade to 1.7.1 or above' - ) - else: - raise e - - if self._is_ipython(): - import IPython - html = 'Pipeline details.' % ( - self._get_url_prefix(), response.id) - IPython.display.display(IPython.display.HTML(html)) - return response - - def get_pipeline(self, pipeline_id: str) -> kfp_server_api.ApiPipeline: - """Get pipeline details. - - Args: - pipeline_id: id of the pipeline. - - Returns: - A response object including details of a pipeline. - - Raises: - kfp_server_api.ApiException: If pipeline is not found. - """ - return self._pipelines_api.get_pipeline(id=pipeline_id) - - def delete_pipeline(self, pipeline_id): - """Delete pipeline. - - Args: - pipeline_id: id of the pipeline. - - Returns: - Object. If the method is called asynchronously, returns the request thread. - - Raises: - kfp_server_api.ApiException: If pipeline is not found. - """ - return self._pipelines_api.delete_pipeline(id=pipeline_id) - - def list_pipeline_versions( - self, - pipeline_id: str, - page_token: str = '', - page_size: int = 10, - sort_by: str = '', - filter: Optional[str] = None - ) -> kfp_server_api.ApiListPipelineVersionsResponse: - """Lists pipeline versions. - - Args: - pipeline_id: Id of the pipeline to list versions - page_token: Token for starting of the page. - page_size: Size of the page. - sort_by: One of 'field_name', 'field_name desc'. For example, 'name desc'. - filter: A url-encoded, JSON-serialized Filter protocol buffer - (see [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/filter.proto)). - - An example filter string would be: - - # For the list of filter operations please see: - # https://github.com/kubeflow/pipelines/blob/master/sdk/python/kfp/_client.py#L40 - json.dumps({ - "predicates": [{ - "op": _FILTER_OPERATIONS["EQUALS"], - "key": "name", - "stringValue": "my-name", - }] - }) - - Returns: - A response object including a list of versions and next page token. - - Raises: - kfp_server_api.ApiException: If pipeline is not found. - """ - - return self._pipelines_api.list_pipeline_versions( - page_token=page_token, - page_size=page_size, - sort_by=sort_by, - resource_key_type=kfp_server_api.models.api_resource_type - .ApiResourceType.PIPELINE, - resource_key_id=pipeline_id, - filter=filter) - - def delete_pipeline_version(self, version_id: str): - """Delete pipeline version. - - Args: - version_id: id of the pipeline version. - - Returns: - Object. If the method is called asynchronously, returns the request thread. - - Raises: - Exception if pipeline version is not found. - """ - return self._pipelines_api.delete_pipeline_version( - version_id=version_id) diff --git a/sdk/python/kfp/deprecated/_config.py b/sdk/python/kfp/deprecated/_config.py deleted file mode 100644 index 297c667c966..00000000000 --- a/sdk/python/kfp/deprecated/_config.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#TODO: wrap the DSL level configuration into one Config -TYPE_CHECK = True -# COMPILING_FOR_V2 is True when using kfp.compiler or use (v1) kfp.compiler -# with V2_COMPATIBLE or V2_ENGINE mode -COMPILING_FOR_V2 = False diff --git a/sdk/python/kfp/deprecated/_local_client.py b/sdk/python/kfp/deprecated/_local_client.py deleted file mode 100644 index da94e07ea8d..00000000000 --- a/sdk/python/kfp/deprecated/_local_client.py +++ /dev/null @@ -1,546 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import json -import logging -import os -import re -import subprocess -import tempfile -import warnings -from collections import deque -from typing import Any, Callable, Dict, List, Mapping, Optional, Union, cast - -from . import dsl -from .compiler.compiler import sanitize_k8s_name - - -class _Dag: - """DAG stands for Direct Acyclic Graph. - - DAG here is used to decide the order to execute pipeline ops. - - For more information on DAG, please refer to `wiki `_. - """ - - def __init__(self, nodes: List[str]) -> None: - """ - - Args:: - nodes: List of DAG nodes, each node is identified by an unique name. - """ - self._graph = {node: [] for node in nodes} - self._reverse_graph = {node: [] for node in nodes} - - @property - def graph(self): - return self._graph - - @property - def reverse_graph(self): - return self._reverse_graph - - def add_edge(self, edge_source: str, edge_target: str) -> None: - """Add an edge between DAG nodes. - - Args:: - edge_source: the source node of the edge - edge_target: the target node of the edge - """ - self._graph[edge_source].append(edge_target) - self._reverse_graph[edge_target].append(edge_source) - - def get_follows(self, source_node: str) -> List[str]: - """Get all target nodes start from the specified source node. - - Args:: - source_node: the source node - """ - return self._graph.get(source_node, []) - - def get_dependencies(self, target_node: str) -> List[str]: - """Get all source nodes end with the specified target node. - - Args:: - target_node: the target node - """ - return self._reverse_graph.get(target_node, []) - - def topological_sort(self) -> List[str]: - """List DAG nodes in topological order.""" - - in_degree = {node: 0 for node in self._graph.keys()} - - for i in self._graph: - for j in self._graph[i]: - in_degree[j] += 1 - - queue = deque() - for node, degree in in_degree.items(): - if degree == 0: - queue.append(node) - - sorted_nodes = [] - - while queue: - u = queue.popleft() - sorted_nodes.append(u) - - for node in self._graph[u]: - in_degree[node] -= 1 - - if in_degree[node] == 0: - queue.append(node) - - return sorted_nodes - - -def _extract_pipeline_param(param: str) -> dsl.PipelineParam: - """Extract PipelineParam from string.""" - matches = re.findall(r"{{pipelineparam:op=([\w\s_-]*);name=([\w\s_-]+)}}", - param) - op_dependency_name = matches[0][0] - output_file_name = matches[0][1] - return dsl.PipelineParam(output_file_name, op_dependency_name) - - -def _get_op(ops: List[dsl.ContainerOp], - op_name: str) -> Union[dsl.ContainerOp, None]: - """Get the first op with specified op name.""" - return next(filter(lambda op: op.name == op_name, ops), None) - - -def _get_subgroup(groups: List[dsl.OpsGroup], - group_name: str) -> Union[dsl.OpsGroup, None]: - """Get the first OpsGroup with specified group name.""" - return next(filter(lambda g: g.name == group_name, groups), None) - - -class LocalClient: - - class ExecutionMode: - """Configuration to decide whether the client executes a component in - docker or in local process.""" - - DOCKER = "docker" - LOCAL = "local" - - def __init__( - self, - mode: str = DOCKER, - images_to_exclude: List[str] = [], - ops_to_exclude: List[str] = [], - docker_options: List[str] = [], - ) -> None: - """Constructor. - - Args: - mode: Default execution mode, default 'docker' - images_to_exclude: If the image of op is in images_to_exclude, the op is - executed in the mode different from default_mode. - ops_to_exclude: If the name of op is in ops_to_exclude, the op is - executed in the mode different from default_mode. - docker_options: Docker options used in docker mode, - e.g. docker_options=["-e", "foo=bar"]. - """ - if mode not in [self.DOCKER, self.LOCAL]: - raise Exception( - "Invalid execution mode, must be docker of local") - self._mode = mode - self._images_to_exclude = images_to_exclude - self._ops_to_exclude = ops_to_exclude - self._docker_options = docker_options - - @property - def mode(self) -> str: - return self._mode - - @property - def images_to_exclude(self) -> List[str]: - return self._images_to_exclude - - @property - def ops_to_exclude(self) -> List[str]: - return self._ops_to_exclude - - @property - def docker_options(self) -> List[str]: - return self._docker_options - - def __init__(self, pipeline_root: Optional[str] = None) -> None: - """Construct the instance of LocalClient. - - Args： - pipeline_root: The root directory where the output artifact of component - will be saved. - """ - warnings.warn( - 'LocalClient is an Alpha[1] feature. It may be deprecated in the future.\n' - '[1] https://github.com/kubeflow/pipelines/blob/master/docs/release/feature-stages.md#alpha', - category=FutureWarning, - ) - - pipeline_root = pipeline_root or tempfile.tempdir - self._pipeline_root = pipeline_root - - def _find_base_group(self, groups: List[dsl.OpsGroup], - op_name: str) -> Union[dsl.OpsGroup, None]: - """Find the base group of op in candidate group list.""" - if groups is None or len(groups) == 0: - return None - for group in groups: - if _get_op(group.ops, op_name): - return group - else: - _parent_group = self._find_base_group(group.groups, op_name) - if _parent_group: - return group - - return None - - def _create_group_dag(self, pipeline_dag: _Dag, - group: dsl.OpsGroup) -> _Dag: - """Create DAG within current group, it's a DAG of direct ops and direct - subgroups. - - Each node of the DAG is either an op or a subgroup. For each - node in current group, if one of its DAG follows is also an op - in current group, add an edge to this follow op, otherwise, if - this follow belongs to subgroups, add an edge to its subgroup. - If this node has dependency from subgroups, then add an edge - from this subgroup to current node. - """ - group_dag = _Dag([op.name for op in group.ops] + - [g.name for g in group.groups]) - - for op in group.ops: - for follow in pipeline_dag.get_follows(op.name): - if _get_op(group.ops, follow) is not None: - # add edge between direct ops - group_dag.add_edge(op.name, follow) - else: - _base_group = self._find_base_group(group.groups, follow) - if _base_group: - # add edge to direct subgroup - group_dag.add_edge(op.name, _base_group.name) - - for dependency in pipeline_dag.get_dependencies(op.name): - if _get_op(group.ops, dependency) is None: - _base_group = self._find_base_group(group.groups, - dependency) - if _base_group: - # add edge from direct subgroup - group_dag.add_edge(_base_group.name, op.name) - - return group_dag - - def _create_op_dag(self, p: dsl.Pipeline) -> _Dag: - """Create the DAG of the pipeline ops.""" - dag = _Dag(p.ops.keys()) - - for op in p.ops.values(): - # dependencies defined by inputs - for input_value in op.inputs: - if isinstance(input_value, dsl.PipelineParam): - input_param = _extract_pipeline_param(input_value.pattern) - if input_param.op_name: - dag.add_edge(input_param.op_name, op.name) - else: - logging.debug("%s depend on pipeline param", op.name) - - # explicit dependencies of current op - for dependent in op.dependent_names: - dag.add_edge(dependent, op.name) - return dag - - def _make_output_file_path_unique(self, run_name: str, op_name: str, - output_file: str) -> str: - """Alter the file path of output artifact to make sure it's unique in - local runner. - - kfp compiler will bound a tmp file for each component output, - which is unique in kfp runtime, but not unique in local runner. - We alter the file path of the name of current run and op, to - make it unique in local runner. - """ - if not output_file.startswith("/tmp/"): - return output_file - return f'{self._pipeline_root}/{run_name}/{op_name.lower()}/{output_file[len("/tmp/"):]}' - - def _get_output_file_path( - self, - run_name: str, - pipeline: dsl.Pipeline, - op_name: str, - output_name: str = None, - ) -> str: - """Get the file path of component output.""" - - op_dependency = pipeline.ops[op_name] - if output_name is None and len(op_dependency.file_outputs) == 1: - output_name = next(iter(op_dependency.file_outputs.keys())) - output_file = op_dependency.file_outputs[output_name] - unique_output_file = self._make_output_file_path_unique( - run_name, op_name, output_file) - return unique_output_file - - def _generate_cmd_for_subprocess_execution( - self, - run_name: str, - pipeline: dsl.Pipeline, - op: dsl.ContainerOp, - stack: Dict[str, Any], - ) -> List[str]: - """Generate shell command to run the op locally.""" - cmd = op.command + op.arguments - - # In debug mode, for `python -c cmd` format command, pydev will insert code before - # `cmd`, but there is no newline at the end of the inserted code, which will cause - # syntax error, so we add newline before `cmd`. - for i in range(len(cmd)): - if cmd[i] == "-c": - cmd[i + 1] = "\n" + cmd[i + 1] - - for index, cmd_item in enumerate(cmd): - if cmd_item in stack: # Argument is LoopArguments item - cmd[index] = str(stack[cmd_item]) - elif cmd_item in op.file_outputs.values( - ): # Argument is output file - output_name = next( - filter(lambda item: item[1] == cmd_item, - op.file_outputs.items()))[0] - output_param = op.outputs[output_name] - output_file = cmd_item - output_file = self._make_output_file_path_unique( - run_name, output_param.op_name, output_file) - - os.makedirs(os.path.dirname(output_file), exist_ok=True) - cmd[index] = output_file - elif (cmd_item in op.input_artifact_paths.values() - ): # Argument is input artifact file - input_name = next( - filter( - lambda item: item[1] == cmd_item, - op.input_artifact_paths.items(), - ))[0] - input_param_pattern = op.artifact_arguments[input_name] - pipeline_param = _extract_pipeline_param(input_param_pattern) - input_file = self._get_output_file_path(run_name, pipeline, - pipeline_param.op_name, - pipeline_param.name) - - cmd[index] = input_file - - return cmd - - def _generate_cmd_for_docker_execution( - self, - run_name: str, - pipeline: dsl.Pipeline, - op: dsl.ContainerOp, - stack: Dict[str, Any], - docker_options: List[str] = []) -> List[str]: - """Generate the command to run the op in docker locally.""" - cmd = self._generate_cmd_for_subprocess_execution( - run_name, pipeline, op, stack) - - docker_cmd = [ - "docker", - "run", - *docker_options, - "-v", - "{pipeline_root}:{pipeline_root}".format( - pipeline_root=self._pipeline_root), - op.image, - ] + cmd - return docker_cmd - - def _run_group_dag( - self, - run_name: str, - pipeline: dsl.Pipeline, - pipeline_dag: _Dag, - current_group: dsl.OpsGroup, - stack: Dict[str, Any], - execution_mode: ExecutionMode, - ) -> bool: - """Run ops in current group in topological order. - - Args: - pipeline: kfp.dsl.Pipeline - pipeline_dag: DAG of pipeline ops - current_group: current ops group - stack: stack to trace `LoopArguments` - execution_mode: Configuration to decide whether the client executes - component in docker or in local process. - Returns: - True if succeed to run the group dag. - """ - group_dag = self._create_group_dag(pipeline_dag, current_group) - - for node in group_dag.topological_sort(): - subgroup = _get_subgroup(current_group.groups, node) - if subgroup is not None: # Node of DAG is subgroup - success = self._run_group(run_name, pipeline, pipeline_dag, - subgroup, stack, execution_mode) - if not success: - return False - else: # Node of DAG is op - op = _get_op(current_group.ops, node) - - execution_mode = ( - execution_mode - if execution_mode else LocalClient.ExecutionMode()) - can_run_locally = execution_mode.mode == LocalClient.ExecutionMode.LOCAL - exclude = ( - op.image in execution_mode.images_to_exclude or - op.name in execution_mode.ops_to_exclude) - if exclude: - can_run_locally = not can_run_locally - - if can_run_locally: - cmd = self._generate_cmd_for_subprocess_execution( - run_name, pipeline, op, stack) - else: - cmd = self._generate_cmd_for_docker_execution( - run_name, pipeline, op, stack, - execution_mode.docker_options) - process = subprocess.Popen( - cmd, - shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - # TODO support async process - logging.info("start task：%s", op.name) - stdout, stderr = process.communicate() - if stdout: - logging.info(stdout) - if stderr: - logging.error(stderr) - if process.returncode != 0: - logging.error(cmd) - return False - - def _run_group( - self, - run_name: str, - pipeline: dsl.Pipeline, - pipeline_dag: _Dag, - current_group: dsl.OpsGroup, - stack: Dict[str, Any], - execution_mode: ExecutionMode, - ) -> bool: - """Run all ops in current group. - - Args: - run_name: str, the name of this run, can be used to query the run result - pipeline: kfp.dsl.Pipeline - pipeline_dag: DAG of pipeline ops - current_group: current ops group - stack: stack to trace `LoopArguments` - execution_mode: Configuration to decide whether the client executes - component in docker or in local process. - Returns: - True if succeed to run the group. - """ - if current_group.type == dsl.ParallelFor.TYPE_NAME: - current_group = cast(dsl.ParallelFor, current_group) - - if current_group.items_is_pipeline_param: - _loop_args = current_group.loop_args - _param_name = _loop_args.name[:-len(_loop_args - .LOOP_ITEM_NAME_BASE) - 1] - - _op_dependency = pipeline.ops[_loop_args.op_name] - _list_file = _op_dependency.file_outputs[_param_name] - _altered_list_file = self._make_output_file_path_unique( - run_name, _loop_args.op_name, _list_file) - with open(_altered_list_file, "r") as f: - _param_values = json.load(f) - for index, _param_value in enumerate(_param_values): - if isinstance(_param_values, (dict, list)): - _param_value = json.dumps(_param_value) - stack[_loop_args.pattern] = _param_value - loop_run_name = "{run_name}/{loop_index}".format( - run_name=run_name, loop_index=index) - success = self._run_group_dag( - loop_run_name, - pipeline, - pipeline_dag, - current_group, - stack, - execution_mode, - ) - del stack[_loop_args.pattern] - if not success: - return False - return True - else: - raise Exception("Not implemented") - else: - return self._run_group_dag(run_name, pipeline, pipeline_dag, - current_group, stack, execution_mode) - - def create_run_from_pipeline_func( - self, - pipeline_func: Callable, - arguments: Mapping[str, str], - execution_mode: ExecutionMode = ExecutionMode(), - ): - """Runs a pipeline locally, either using Docker or in a local process. - - Parameters: - pipeline_func: pipeline function - arguments: Arguments to the pipeline function provided as a dict, reference - to `kfp.client.create_run_from_pipeline_func` - execution_mode: Configuration to decide whether the client executes component - in docker or in local process. - """ - - class RunPipelineResult: - - def __init__(self, client: LocalClient, pipeline: dsl.Pipeline, - run_id: str, success: bool): - self._client = client - self._pipeline = pipeline - self.run_id = run_id - self._success = success - - def get_output_file(self, op_name: str, output: str = None): - return self._client._get_output_file_path( - self.run_id, self._pipeline, op_name, output) - - def success(self) -> bool: - return self._success - - def __repr__(self): - return "RunPipelineResult(run_id={})".format(self.run_id) - - pipeline_name = sanitize_k8s_name( - getattr(pipeline_func, "_component_human_name", None) or - pipeline_func.__name__) - with dsl.Pipeline(pipeline_name) as pipeline: - pipeline_func(**arguments) - - run_version = datetime.datetime.now().strftime("%Y%m%d%H%M%S") - run_name = pipeline.name.replace(" ", "_").lower() + "_" + run_version - - pipeline_dag = self._create_op_dag(pipeline) - success = self._run_group(run_name, pipeline, pipeline_dag, - pipeline.groups[0], {}, execution_mode) - - return RunPipelineResult(self, pipeline, run_name, success=success) diff --git a/sdk/python/kfp/deprecated/_runners.py b/sdk/python/kfp/deprecated/_runners.py deleted file mode 100644 index 54a83176b46..00000000000 --- a/sdk/python/kfp/deprecated/_runners.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__all__ = [ - "run_pipeline_func_on_cluster", - "run_pipeline_func_locally", -] - -from typing import Callable, Mapping, Optional - -from . import Client, LocalClient, dsl - - -def run_pipeline_func_on_cluster( - pipeline_func: Callable, - arguments: Mapping[str, str], - run_name: str = None, - experiment_name: str = None, - kfp_client: Client = None, - pipeline_conf: dsl.PipelineConf = None, -): - """Runs pipeline on KFP-enabled Kubernetes cluster. - - This command compiles the pipeline function, creates or gets an experiment - and submits the pipeline for execution. - - Feature stage: - [Alpha](https://github.com/kubeflow/pipelines/blob/07328e5094ac2981d3059314cc848fbb71437a76/docs/release/feature-stages.md#alpha) - - Args: - pipeline_func: A function that describes a pipeline by calling components - and composing them into execution graph. - arguments: Arguments to the pipeline function provided as a dict. - run_name: Optional. Name of the run to be shown in the UI. - experiment_name: Optional. Name of the experiment to add the run to. - kfp_client: Optional. An instance of kfp.Client configured for the desired - KFP cluster. - pipeline_conf: Optional. kfp.dsl.PipelineConf instance. Can specify op - transforms, image pull secrets and other pipeline-level configuration - options. - """ - kfp_client = kfp_client or Client() - return kfp_client.create_run_from_pipeline_func(pipeline_func, arguments, - run_name, experiment_name, - pipeline_conf) - - -def run_pipeline_func_locally( - pipeline_func: Callable, - arguments: Mapping[str, str], - local_client: Optional[LocalClient] = None, - pipeline_root: Optional[str] = None, - execution_mode: LocalClient.ExecutionMode = LocalClient.ExecutionMode(), -): - """Runs a pipeline locally, either using Docker or in a local process. - - Feature stage: - [Alpha](https://github.com/kubeflow/pipelines/blob/master/docs/release/feature-stages.md#alpha) - - In this alpha implementation, we support: - * Control flow: Condition, ParallelFor - * Data passing: InputValue, InputPath, OutputPath - - And we don't support: - * Control flow: ExitHandler, Graph, SubGraph - * ContainerOp with environment variables, init containers, sidecars, pvolumes - * ResourceOp - * VolumeOp - * Caching - - Args: - pipeline_func: A function that describes a pipeline by calling components - and composing them into execution graph. - arguments: Arguments to the pipeline function provided as a dict. - reference to `kfp.client.create_run_from_pipeline_func`. - local_client: Optional. An instance of kfp.LocalClient. - pipeline_root: Optional. The root directory where the output artifact of component - will be saved. - execution_mode: Configuration to decide whether the client executes component - in docker or in local process. - """ - local_client = local_client or LocalClient(pipeline_root) - return local_client.create_run_from_pipeline_func( - pipeline_func, arguments, execution_mode=execution_mode) diff --git a/sdk/python/kfp/deprecated/auth/__init__.py b/sdk/python/kfp/deprecated/auth/__init__.py deleted file mode 100644 index cc90d9d27ca..00000000000 --- a/sdk/python/kfp/deprecated/auth/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2021 Arrikto Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ._tokencredentialsbase import TokenCredentialsBase, read_token_from_file -from ._satvolumecredentials import ServiceAccountTokenVolumeCredentials - -KF_PIPELINES_SA_TOKEN_ENV = "KF_PIPELINES_SA_TOKEN_PATH" -KF_PIPELINES_SA_TOKEN_PATH = "/var/run/secrets/kubeflow/pipelines/token" diff --git a/sdk/python/kfp/deprecated/auth/_satvolumecredentials.py b/sdk/python/kfp/deprecated/auth/_satvolumecredentials.py deleted file mode 100644 index cb31a342a30..00000000000 --- a/sdk/python/kfp/deprecated/auth/_satvolumecredentials.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2021 Arrikto Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import logging - -from kubernetes.client import configuration - -from kfp.deprecated import auth - - -class ServiceAccountTokenVolumeCredentials(auth.TokenCredentialsBase): - """Audience-bound ServiceAccountToken in the local filesystem. - - This is a credentials interface for audience-bound ServiceAccountTokens - found in the local filesystem, that get refreshed by the kubelet. - - The constructor of the class expects a filesystem path. - If not provided, it uses the path stored in the environment variable - defined in ``auth.KF_PIPELINES_SA_TOKEN_ENV``. - If the environment variable is also empty, it falls back to the path - specified in ``auth.KF_PIPELINES_SA_TOKEN_PATH``. - - This method of authentication is meant for use inside a Kubernetes cluster. - - Relevant documentation: - https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#service-account-token-volume-projection - """ - - def __init__(self, path=None): - self._token_path = ( - path or os.getenv(auth.KF_PIPELINES_SA_TOKEN_ENV) or - auth.KF_PIPELINES_SA_TOKEN_PATH) - - def _get_token(self): - token = None - try: - token = auth.read_token_from_file(self._token_path) - except OSError as e: - logging.error("Failed to read a token from file '%s' (%s).", - self._token_path, str(e)) - raise - return token - - def refresh_api_key_hook(self, config: configuration.Configuration): - """Refresh the api key. - - This is a helper function for registering token refresh with swagger - generated clients. - - Args: - config (kubernetes.client.configuration.Configuration): - The configuration object that the client uses. - - The Configuration object of the kubernetes client's is the same - with kfp_server_api.configuration.Configuration. - """ - config.api_key["authorization"] = self._get_token() diff --git a/sdk/python/kfp/deprecated/auth/_tokencredentialsbase.py b/sdk/python/kfp/deprecated/auth/_tokencredentialsbase.py deleted file mode 100644 index 6589ebd35eb..00000000000 --- a/sdk/python/kfp/deprecated/auth/_tokencredentialsbase.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2021 Arrikto Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import abc - -from kubernetes.client import configuration - - -class TokenCredentialsBase(abc.ABC): - - @abc.abstractmethod - def refresh_api_key_hook(self, config: configuration.Configuration): - """Refresh the api key. - - This is a helper function for registering token refresh with swagger - generated clients. - - All classes that inherit from TokenCredentialsBase must implement this - method to refresh the credentials. - - Args: - config (kubernetes.client.configuration.Configuration): - The configuration object that the client uses. - - The Configuration object of the kubernetes client's is the same - with kfp_server_api.configuration.Configuration. - """ - raise NotImplementedError() - - -def read_token_from_file(path=None): - """Read a token found in some file.""" - token = None - with open(path, "r") as f: - token = f.read().strip() - return token diff --git a/sdk/python/kfp/deprecated/aws.py b/sdk/python/kfp/deprecated/aws.py deleted file mode 100644 index 7b76c17393d..00000000000 --- a/sdk/python/kfp/deprecated/aws.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def use_aws_secret(secret_name='aws-secret', - aws_access_key_id_name='AWS_ACCESS_KEY_ID', - aws_secret_access_key_name='AWS_SECRET_ACCESS_KEY', - aws_region=None): - """An operator that configures the container to use AWS credentials. - - AWS doesn't create secret along with kubeflow deployment and it requires users - to manually create credential secret with proper permissions. - - :: - - apiVersion: v1 - kind: Secret - metadata: - name: aws-secret - type: Opaque - data: - AWS_ACCESS_KEY_ID: BASE64_YOUR_AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: BASE64_YOUR_AWS_SECRET_ACCESS_KEY - """ - - def _use_aws_secret(task): - from kubernetes import client as k8s_client - task.container \ - .add_env_variable( - k8s_client.V1EnvVar( - name='AWS_ACCESS_KEY_ID', - value_from=k8s_client.V1EnvVarSource( - secret_key_ref=k8s_client.V1SecretKeySelector( - name=secret_name, - key=aws_access_key_id_name - ) - ) - ) - ) \ - .add_env_variable( - k8s_client.V1EnvVar( - name='AWS_SECRET_ACCESS_KEY', - value_from=k8s_client.V1EnvVarSource( - secret_key_ref=k8s_client.V1SecretKeySelector( - name=secret_name, - key=aws_secret_access_key_name - ) - ) - ) - ) - - if aws_region: - task.container \ - .add_env_variable( - k8s_client.V1EnvVar( - name='AWS_REGION', - value=aws_region - ) - ) - return task - - return _use_aws_secret diff --git a/sdk/python/kfp/deprecated/azure.py b/sdk/python/kfp/deprecated/azure.py deleted file mode 100644 index 6693f18cb3d..00000000000 --- a/sdk/python/kfp/deprecated/azure.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -def use_azure_secret(secret_name='azcreds'): - """An operator that configures the container to use Azure user credentials. - - The azcreds secret is created as part of the kubeflow deployment that - stores the client ID and secrets for the kubeflow azure service principal. - - With this service principal, the container has a range of Azure APIs to access to. - """ - - def _use_azure_secret(task): - from kubernetes import client as k8s_client - (task.container.add_env_variable( - k8s_client.V1EnvVar( - name='AZ_SUBSCRIPTION_ID', - value_from=k8s_client.V1EnvVarSource( - secret_key_ref=k8s_client.V1SecretKeySelector( - name=secret_name, key='AZ_SUBSCRIPTION_ID'))) - ).add_env_variable( - k8s_client.V1EnvVar( - name='AZ_TENANT_ID', - value_from=k8s_client.V1EnvVarSource( - secret_key_ref=k8s_client.V1SecretKeySelector( - name=secret_name, key='AZ_TENANT_ID'))) - ).add_env_variable( - k8s_client.V1EnvVar( - name='AZ_CLIENT_ID', - value_from=k8s_client.V1EnvVarSource( - secret_key_ref=k8s_client.V1SecretKeySelector( - name=secret_name, key='AZ_CLIENT_ID')))) - .add_env_variable( - k8s_client.V1EnvVar( - name='AZ_CLIENT_SECRET', - value_from=k8s_client.V1EnvVarSource( - secret_key_ref=k8s_client.V1SecretKeySelector( - name=secret_name, key='AZ_CLIENT_SECRET'))))) - return task - - return _use_azure_secret diff --git a/sdk/python/kfp/deprecated/cli/__init__.py b/sdk/python/kfp/deprecated/cli/__init__.py deleted file mode 100644 index c55179a49ed..00000000000 --- a/sdk/python/kfp/deprecated/cli/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/sdk/python/kfp/deprecated/cli/cli.py b/sdk/python/kfp/deprecated/cli/cli.py deleted file mode 100644 index 7bac24ac8f9..00000000000 --- a/sdk/python/kfp/deprecated/cli/cli.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import sys - -import click -import typer - -from kfp.deprecated._client import Client -from kfp.deprecated.cli.run import run -from kfp.deprecated.cli.recurring_run import recurring_run -from kfp.deprecated.cli.pipeline import pipeline -from kfp.deprecated.cli.diagnose_me_cli import diagnose_me -from kfp.deprecated.cli.experiment import experiment -from kfp.deprecated.cli.output import OutputFormat -from kfp.deprecated.cli import components - -_NO_CLIENT_COMMANDS = ['diagnose_me', 'components'] - - -@click.group() -@click.option('--endpoint', help='Endpoint of the KFP API service to connect.') -@click.option('--iap-client-id', help='Client ID for IAP protected endpoint.') -@click.option( - '-n', - '--namespace', - default='kubeflow', - show_default=True, - help='Kubernetes namespace to connect to the KFP API.') -@click.option( - '--other-client-id', - help='Client ID for IAP protected endpoint to obtain the refresh token.') -@click.option( - '--other-client-secret', - help='Client ID for IAP protected endpoint to obtain the refresh token.') -@click.option( - '--output', - type=click.Choice(list(map(lambda x: x.name, OutputFormat))), - default=OutputFormat.table.name, - show_default=True, - help='The formatting style for command output.') -@click.pass_context -def cli(ctx: click.Context, endpoint: str, iap_client_id: str, namespace: str, - other_client_id: str, other_client_secret: str, output: OutputFormat): - """kfp is the command line interface to KFP service. - - Feature stage: - [Alpha](https://github.com/kubeflow/pipelines/blob/07328e5094ac2981d3059314cc848fbb71437a76/docs/release/feature-stages.md#alpha) - """ - if ctx.invoked_subcommand in _NO_CLIENT_COMMANDS: - # Do not create a client for these subcommands - return - ctx.obj['client'] = Client(endpoint, iap_client_id, namespace, - other_client_id, other_client_secret) - ctx.obj['namespace'] = namespace - ctx.obj['output'] = output - - -def main(): - logging.basicConfig(format='%(message)s', level=logging.INFO) - cli.add_command(run) - cli.add_command(recurring_run) - cli.add_command(pipeline) - cli.add_command(diagnose_me, 'diagnose_me') - cli.add_command(experiment) - cli.add_command(typer.main.get_command(components.app)) - try: - cli(obj={}, auto_envvar_prefix='KFP') - except Exception as e: - click.echo(str(e), err=True) - sys.exit(1) diff --git a/sdk/python/kfp/deprecated/cli/components.py b/sdk/python/kfp/deprecated/cli/components.py deleted file mode 100644 index 21f11293b6f..00000000000 --- a/sdk/python/kfp/deprecated/cli/components.py +++ /dev/null @@ -1,408 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import contextlib -import enum -import pathlib -import shutil -import subprocess -import tempfile -from typing import Any, List, Optional - -_DOCKER_IS_PRESENT = True -try: - import docker -except ImportError: - _DOCKER_IS_PRESENT = False - -import typer - -import kfp.deprecated as kfp -from kfp.components import component_factory, kfp_config, utils - -_REQUIREMENTS_TXT = 'requirements.txt' - -_DOCKERFILE = 'Dockerfile' - -_DOCKERFILE_TEMPLATE = ''' -FROM {base_image} - -WORKDIR {component_root_dir} -COPY requirements.txt requirements.txt -RUN pip install --no-cache-dir -r requirements.txt -{maybe_copy_kfp_package} -RUN pip install --no-cache-dir {kfp_package_path} -COPY . . -''' - -_DOCKERIGNORE = '.dockerignore' - -# Location in which to write out shareable YAML for components. -_COMPONENT_METADATA_DIR = 'component_metadata' - -_DOCKERIGNORE_TEMPLATE = ''' -{}/ -'''.format(_COMPONENT_METADATA_DIR) - -# Location at which v2 Python function-based components will stored -# in containerized components. -_COMPONENT_ROOT_DIR = pathlib.Path('/usr/local/src/kfp/components') - - -@contextlib.contextmanager -def _registered_modules(): - registered_modules = {} - component_factory.REGISTERED_MODULES = registered_modules - try: - yield registered_modules - finally: - component_factory.REGISTERED_MODULES = None - - -class _Engine(str, enum.Enum): - """Supported container build engines.""" - DOCKER = 'docker' - KANIKO = 'kaniko' - CLOUD_BUILD = 'cloudbuild' - - -app = typer.Typer() - - -def _info(message: Any): - info = typer.style('INFO', fg=typer.colors.GREEN) - typer.echo('{}: {}'.format(info, message)) - - -def _warning(message: Any): - info = typer.style('WARNING', fg=typer.colors.YELLOW) - typer.echo('{}: {}'.format(info, message)) - - -def _error(message: Any): - info = typer.style('ERROR', fg=typer.colors.RED) - typer.echo('{}: {}'.format(info, message)) - - -class _ComponentBuilder(): - """Helper class for building containerized v2 KFP components.""" - - def __init__( - self, - context_directory: pathlib.Path, - kfp_package_path: Optional[pathlib.Path] = None, - component_filepattern: str = '**/*.py', - ): - """ComponentBuilder builds containerized components. - - Args: - context_directory: Directory containing one or more Python files - with one or more KFP v2 components. - kfp_package_path: Path to a pip-installable location for KFP. - This can either be pointing to KFP SDK root directory located in - a local clone of the KFP repo, or a git+https location. - If left empty, defaults to KFP on PyPi. - """ - self._context_directory = context_directory - self._dockerfile = self._context_directory / _DOCKERFILE - self._component_filepattern = component_filepattern - self._components: List[ - component_factory.component_factory.ComponentInfo] = [] - - # This is only set if we need to install KFP from local copy. - self._maybe_copy_kfp_package = '' - - if kfp_package_path is None: - self._kfp_package_path = 'kfp=={}'.format(kfp.__version__) - elif kfp_package_path.is_dir(): - _info('Building KFP package from local directory {}'.format( - typer.style(str(kfp_package_path), fg=typer.colors.CYAN))) - temp_dir = pathlib.Path(tempfile.mkdtemp()) - try: - subprocess.run([ - 'python3', - kfp_package_path / 'setup.py', - 'bdist_wheel', - '--dist-dir', - str(temp_dir), - ], - cwd=kfp_package_path) - wheel_files = list(temp_dir.glob('*.whl')) - if len(wheel_files) != 1: - _error('Failed to find built KFP wheel under {}'.format( - temp_dir)) - raise typer.Exit(1) - - wheel_file = wheel_files[0] - shutil.copy(wheel_file, self._context_directory) - self._kfp_package_path = wheel_file.name - self._maybe_copy_kfp_package = 'COPY {wheel_name} {wheel_name}'.format( - wheel_name=self._kfp_package_path) - except subprocess.CalledProcessError as e: - _error('Failed to build KFP wheel locally:\n{}'.format(e)) - raise typer.Exit(1) - finally: - _info('Cleaning up temporary directory {}'.format(temp_dir)) - shutil.rmtree(temp_dir) - else: - self._kfp_package_path = kfp_package_path - - _info('Building component using KFP package path: {}'.format( - typer.style(str(self._kfp_package_path), fg=typer.colors.CYAN))) - - self._context_directory_files = [ - file.name - for file in self._context_directory.glob('*') - if file.is_file() - ] - - self._component_files = [ - file for file in self._context_directory.glob( - self._component_filepattern) if file.is_file() - ] - - self._base_image = None - self._target_image = None - self._load_components() - - def _load_components(self): - if not self._component_files: - _error( - 'No component files found matching pattern `{}` in directory {}' - .format(self._component_filepattern, self._context_directory)) - raise typer.Exit(1) - - for python_file in self._component_files: - with _registered_modules() as component_modules: - module_name = python_file.name[:-len('.py')] - module_directory = python_file.parent - utils.load_module( - module_name=module_name, module_directory=module_directory) - - formatted_module_file = typer.style( - str(python_file), fg=typer.colors.CYAN) - if not component_modules: - _error('No KFP components found in file {}'.format( - formatted_module_file)) - raise typer.Exit(1) - - _info('Found {} component(s) in file {}:'.format( - len(component_modules), formatted_module_file)) - for name, component in component_modules.items(): - _info('{}: {}'.format(name, component)) - self._components.append(component) - - base_images = set([info.base_image for info in self._components]) - target_images = set([info.target_image for info in self._components]) - - if len(base_images) != 1: - _error('Found {} unique base_image values {}. Components' - ' must specify the same base_image and target_image.'.format( - len(base_images), base_images)) - raise typer.Exit(1) - - self._base_image = base_images.pop() - if self._base_image is None: - _error('Did not find a base_image specified in any of the' - ' components. A base_image must be specified in order to' - ' build the component.') - raise typer.Exit(1) - _info('Using base image: {}'.format( - typer.style(self._base_image, fg=typer.colors.YELLOW))) - - if len(target_images) != 1: - _error('Found {} unique target_image values {}. Components' - ' must specify the same base_image and' - ' target_image.'.format(len(target_images), target_images)) - raise typer.Exit(1) - - self._target_image = target_images.pop() - if self._target_image is None: - _error('Did not find a target_image specified in any of the' - ' components. A target_image must be specified in order' - ' to build the component.') - raise typer.Exit(1) - _info('Using target image: {}'.format( - typer.style(self._target_image, fg=typer.colors.YELLOW))) - - def _maybe_write_file(self, - filename: str, - contents: str, - overwrite: bool = False): - formatted_filename = typer.style(filename, fg=typer.colors.CYAN) - if filename in self._context_directory_files: - _info('Found existing file {} under {}.'.format( - formatted_filename, self._context_directory)) - if not overwrite: - _info('Leaving this file untouched.') - return - else: - _warning( - 'Overwriting existing file {}'.format(formatted_filename)) - else: - _warning('{} not found under {}. Creating one.'.format( - formatted_filename, self._context_directory)) - - filepath = self._context_directory / filename - with open(filepath, 'w') as f: - f.write('# Generated by KFP.\n{}'.format(contents)) - _info('Generated file {}.'.format(filepath)) - - def maybe_generate_requirements_txt(self): - self._maybe_write_file(_REQUIREMENTS_TXT, '') - - def maybe_generate_dockerignore(self): - self._maybe_write_file(_DOCKERIGNORE, _DOCKERIGNORE_TEMPLATE) - - def write_component_files(self): - for component_info in self._components: - filename = ( - component_info.output_component_file or - component_info.function_name + '.yaml') - container_filename = ( - self._context_directory / _COMPONENT_METADATA_DIR / filename) - container_filename.parent.mkdir(exist_ok=True, parents=True) - component_info.component_spec.save_to_component_yaml( - str(container_filename)) - - def generate_kfp_config(self): - config = kfp_config.KFPConfig(config_directory=self._context_directory) - for component_info in self._components: - relative_path = component_info.module_path.relative_to( - self._context_directory) - config.add_component( - function_name=component_info.function_name, path=relative_path) - config.save() - - def maybe_generate_dockerfile(self, overwrite_dockerfile: bool = False): - dockerfile_contents = _DOCKERFILE_TEMPLATE.format( - base_image=self._base_image, - maybe_copy_kfp_package=self._maybe_copy_kfp_package, - component_root_dir=_COMPONENT_ROOT_DIR, - kfp_package_path=self._kfp_package_path) - - self._maybe_write_file(_DOCKERFILE, dockerfile_contents, - overwrite_dockerfile) - - def build_image(self, push_image: bool = True): - _info('Building image {} using Docker...'.format( - typer.style(self._target_image, fg=typer.colors.YELLOW))) - client = docker.from_env() - - docker_log_prefix = typer.style('Docker', fg=typer.colors.CYAN) - - try: - context = str(self._context_directory) - logs = client.api.build( - path=context, - dockerfile='Dockerfile', - tag=self._target_image, - decode=True, - ) - for log in logs: - message = log.get('stream', '').rstrip('\n') - if message: - _info('{}: {}'.format(docker_log_prefix, message)) - - except docker.errors.BuildError as e: - for log in e.build_log: - message = log.get('message', '').rstrip('\n') - if message: - _error('{}: {}'.format(docker_log_prefix, message)) - _error('{}: {}'.format(docker_log_prefix, e)) - raise typer.Exit(1) - - if not push_image: - return - - _info('Pushing image {}...'.format( - typer.style(self._target_image, fg=typer.colors.YELLOW))) - - try: - response = client.images.push( - self._target_image, stream=True, decode=True) - for log in response: - status = log.get('status', '').rstrip('\n') - layer = log.get('id', '') - if status: - _info('{}: {} {}'.format(docker_log_prefix, layer, status)) - except docker.errors.BuildError as e: - _error('{}: {}'.format(docker_log_prefix, e)) - raise e - - _info('Built and pushed component container {}'.format( - typer.style(self._target_image, fg=typer.colors.YELLOW))) - - -@app.callback() -def components(): - """Builds shareable, containerized components.""" - - -@app.command() -def build(components_directory: pathlib.Path = typer.Argument( - ..., - help="Path to a directory containing one or more Python" - " files with KFP v2 components. The container will be built" - " with this directory as the context."), - component_filepattern: str = typer.Option( - '**/*.py', - help="Filepattern to use when searching for KFP components. The" - " default searches all Python files in the specified directory."), - engine: _Engine = typer.Option( - _Engine.DOCKER, - help="Engine to use to build the component's container."), - kfp_package_path: Optional[pathlib.Path] = typer.Option( - None, help="A pip-installable path to the KFP package."), - overwrite_dockerfile: bool = typer.Option( - False, - help="Set this to true to always generate a Dockerfile" - " as part of the build process"), - push_image: bool = typer.Option( - True, help="Push the built image to its remote repository.")): - """Builds containers for KFP v2 Python-based components.""" - components_directory = components_directory.resolve() - if not components_directory.is_dir(): - _error('{} does not seem to be a valid directory.'.format( - components_directory)) - raise typer.Exit(1) - - if engine != _Engine.DOCKER: - _error('Currently, only `docker` is supported for --engine.') - raise typer.Exit(1) - - if engine == _Engine.DOCKER: - if not _DOCKER_IS_PRESENT: - _error( - 'The `docker` Python package was not found in the current' - ' environment. Please run `pip install docker` to install it.' - ' Optionally, you can also install KFP with all of its' - ' optional dependencies by running `pip install kfp[all]`.') - raise typer.Exit(1) - - builder = _ComponentBuilder( - context_directory=components_directory, - kfp_package_path=kfp_package_path, - component_filepattern=component_filepattern, - ) - builder.write_component_files() - builder.generate_kfp_config() - - builder.maybe_generate_requirements_txt() - builder.maybe_generate_dockerignore() - builder.maybe_generate_dockerfile(overwrite_dockerfile=overwrite_dockerfile) - builder.build_image(push_image=push_image) - - -if __name__ == '__main__': - app() diff --git a/sdk/python/kfp/deprecated/cli/components_test.py b/sdk/python/kfp/deprecated/cli/components_test.py deleted file mode 100644 index 6d2e6cea0e3..00000000000 --- a/sdk/python/kfp/deprecated/cli/components_test.py +++ /dev/null @@ -1,492 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for `components` command group in KFP CLI.""" -import contextlib -import pathlib -import sys -import textwrap -import unittest -from typing import List, Optional, Union -from unittest import mock - -from typer import testing - -# Docker is an optional install, but we need the import to succeed for tests. -# So we patch it before importing kfp.cli.components. -try: - import docker # pylint: disable=unused-import -except ImportError: - sys.modules['docker'] = mock.Mock() -from kfp.deprecated.cli import components - -_COMPONENT_TEMPLATE = ''' -from kfp.dsl import * - -@component( - base_image={base_image}, - target_image={target_image}, - output_component_file={output_component_file}) -def {func_name}(): - pass -''' - - -def _make_component(func_name: str, - base_image: Optional[str] = None, - target_image: Optional[str] = None, - output_component_file: Optional[str] = None) -> str: - return textwrap.dedent(''' - from kfp.dsl import * - - @component( - base_image={base_image}, - target_image={target_image}, - output_component_file={output_component_file}) - def {func_name}(): - pass - ''').format( - base_image=repr(base_image), - target_image=repr(target_image), - output_component_file=repr(output_component_file), - func_name=func_name) - - -def _write_file(filename: str, file_contents: str): - filepath = pathlib.Path(filename) - filepath.parent.mkdir(exist_ok=True, parents=True) - filepath.write_text(file_contents) - - -def _write_components(filename: str, component_template: Union[List[str], str]): - if isinstance(component_template, list): - file_contents = '\n\n'.join(component_template) - else: - file_contents = component_template - _write_file(filename=filename, file_contents=file_contents) - - -class Test(unittest.TestCase): - - def setUp(self) -> None: - self._runner = testing.CliRunner() - components._DOCKER_IS_PRESENT = True - - patcher = mock.patch('docker.from_env') - self._docker_client = patcher.start().return_value - self._docker_client.images.build.return_value = [{ - 'stream': 'Build logs' - }] - self._docker_client.images.push.return_value = [{'status': 'Pushed'}] - self.addCleanup(patcher.stop) - - self._app = components.app - with contextlib.ExitStack() as stack: - stack.enter_context(self._runner.isolated_filesystem()) - self._working_dir = pathlib.Path.cwd() - self.addCleanup(stack.pop_all().close) - - return super().setUp() - - def assertFileExists(self, path: str): - path_under_test_dir = self._working_dir / path - self.assertTrue(path_under_test_dir, f'File {path} does not exist!') - - def assertFileExistsAndContains(self, path: str, expected_content: str): - self.assertFileExists(path) - path_under_test_dir = self._working_dir / path - got_content = path_under_test_dir.read_text() - self.assertEqual(got_content, expected_content) - - def testKFPConfigForSingleFile(self): - preprocess_component = _make_component( - func_name='preprocess', target_image='custom-image') - train_component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', - [preprocess_component, train_component]) - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir)], - ) - self.assertEqual(result.exit_code, 0) - - self.assertFileExistsAndContains( - 'kfp_config.ini', - textwrap.dedent('''\ - [Components] - preprocess = components.py - train = components.py - - ''')) - - def testKFPConfigForSingleFileUnderNestedDirectory(self): - preprocess_component = _make_component( - func_name='preprocess', target_image='custom-image') - train_component = _make_component( - func_name='train', target_image='custom-image') - _write_components('dir1/dir2/dir3/components.py', - [preprocess_component, train_component]) - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir)], - ) - self.assertEqual(result.exit_code, 0) - - self.assertFileExistsAndContains( - 'kfp_config.ini', - textwrap.dedent('''\ - [Components] - preprocess = dir1/dir2/dir3/components.py - train = dir1/dir2/dir3/components.py - - ''')) - - def testKFPConfigForMultipleFiles(self): - component = _make_component( - func_name='preprocess', target_image='custom-image') - _write_components('preprocess_component.py', component) - - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('train_component.py', component) - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir)], - ) - self.assertEqual(result.exit_code, 0) - - self.assertFileExistsAndContains( - 'kfp_config.ini', - textwrap.dedent('''\ - [Components] - preprocess = preprocess_component.py - train = train_component.py - - ''')) - - def testKFPConfigForMultipleFilesUnderNestedDirectories(self): - component = _make_component( - func_name='preprocess', target_image='custom-image') - _write_components('preprocess/preprocess_component.py', component) - - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('train/train_component.py', component) - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir)], - ) - self.assertEqual(result.exit_code, 0) - - self.assertFileExistsAndContains( - 'kfp_config.ini', - textwrap.dedent('''\ - [Components] - preprocess = preprocess/preprocess_component.py - train = train/train_component.py - - ''')) - - def testTargetImageMustBeTheSameInAllComponents(self): - component_one = _make_component(func_name='one', target_image='image-1') - component_two = _make_component(func_name='two', target_image='image-1') - _write_components('one_two/one_two.py', [component_one, component_two]) - - component_three = _make_component( - func_name='three', target_image='image-2') - component_four = _make_component( - func_name='four', target_image='image-3') - _write_components('three_four/three_four.py', - [component_three, component_four]) - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir)], - ) - self.assertEqual(result.exit_code, 1) - - def testTargetImageMustBeTheSameInAllComponents(self): - component_one = _make_component( - func_name='one', base_image='image-1', target_image='target-image') - component_two = _make_component( - func_name='two', base_image='image-1', target_image='target-image') - _write_components('one_two/one_two.py', [component_one, component_two]) - - component_three = _make_component( - func_name='three', - base_image='image-2', - target_image='target-image') - component_four = _make_component( - func_name='four', base_image='image-3', target_image='target-image') - _write_components('three_four/three_four.py', - [component_three, component_four]) - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir)], - ) - self.assertEqual(result.exit_code, 1) - - def testComponentFilepatternCanBeUsedToRestrictDiscovery(self): - component = _make_component( - func_name='preprocess', target_image='custom-image') - _write_components('preprocess/preprocess_component.py', component) - - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('train/train_component.py', component) - - result = self._runner.invoke( - self._app, - [ - 'build', - str(self._working_dir), '--component-filepattern=train/*' - ], - ) - self.assertEqual(result.exit_code, 0) - - self.assertFileExistsAndContains( - 'kfp_config.ini', - textwrap.dedent('''\ - [Components] - train = train/train_component.py - - ''')) - - def testEmptyRequirementsTxtFileIsGenerated(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - - result = self._runner.invoke(self._app, - ['build', str(self._working_dir)]) - self.assertEqual(result.exit_code, 0) - self.assertFileExistsAndContains('requirements.txt', - '# Generated by KFP.\n') - - def testExistingRequirementsTxtFileIsUnchanged(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - - _write_file('requirements.txt', 'Some pre-existing content') - - result = self._runner.invoke(self._app, - ['build', str(self._working_dir)]) - self.assertEqual(result.exit_code, 0) - self.assertFileExistsAndContains('requirements.txt', - 'Some pre-existing content') - - def testDockerignoreFileIsGenerated(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - - result = self._runner.invoke(self._app, - ['build', str(self._working_dir)]) - self.assertEqual(result.exit_code, 0) - self.assertFileExistsAndContains( - '.dockerignore', - textwrap.dedent('''\ - # Generated by KFP. - - component_metadata/ - ''')) - - def testExistingDockerignoreFileIsUnchanged(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - - _write_file('.dockerignore', 'Some pre-existing content') - - result = self._runner.invoke(self._app, - ['build', str(self._working_dir)]) - self.assertEqual(result.exit_code, 0) - self.assertFileExistsAndContains('.dockerignore', - 'Some pre-existing content') - - def testDockerEngineIsSupported(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir), '--engine=docker']) - self.assertEqual(result.exit_code, 0) - self._docker_client.api.build.assert_called_once() - self._docker_client.images.push.assert_called_once_with( - 'custom-image', stream=True, decode=True) - - def testKanikoEngineIsNotSupported(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir), '--engine=kaniko'], - ) - self.assertEqual(result.exit_code, 1) - self._docker_client.api.build.assert_not_called() - self._docker_client.images.push.assert_not_called() - - def testCloudBuildEngineIsNotSupported(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir), '--engine=cloudbuild'], - ) - self.assertEqual(result.exit_code, 1) - self._docker_client.api.build.assert_not_called() - self._docker_client.images.push.assert_not_called() - - def testDockerClientIsCalledToBuildAndPushByDefault(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir)], - ) - self.assertEqual(result.exit_code, 0) - - self._docker_client.api.build.assert_called_once() - self._docker_client.images.push.assert_called_once_with( - 'custom-image', stream=True, decode=True) - - def testDockerClientIsCalledToBuildButSkipsPushing(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir), '--no-push-image'], - ) - self.assertEqual(result.exit_code, 0) - - self._docker_client.api.build.assert_called_once() - self._docker_client.images.push.assert_not_called() - - @mock.patch('kfp.__version__', '1.2.3') - def testDockerfileIsCreatedCorrectly(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir)], - ) - self.assertEqual(result.exit_code, 0) - self._docker_client.api.build.assert_called_once() - self.assertFileExistsAndContains( - 'Dockerfile', - textwrap.dedent('''\ - # Generated by KFP. - - FROM python:3.9 - - WORKDIR /usr/local/src/kfp/components - COPY requirements.txt requirements.txt - RUN pip install --no-cache-dir -r requirements.txt - - RUN pip install --no-cache-dir kfp==1.8.11 - COPY . . - ''')) - - def testExistingDockerfileIsUnchangedByDefault(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - _write_file('Dockerfile', 'Existing Dockerfile contents') - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir)], - ) - self.assertEqual(result.exit_code, 0) - self._docker_client.api.build.assert_called_once() - self.assertFileExistsAndContains('Dockerfile', - 'Existing Dockerfile contents') - - @mock.patch('kfp.__version__', '1.2.3') - def testExistingDockerfileCanBeOverwritten(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - _write_file('Dockerfile', 'Existing Dockerfile contents') - - result = self._runner.invoke( - self._app, - ['build', str(self._working_dir), '--overwrite-dockerfile'], - ) - self.assertEqual(result.exit_code, 0) - self._docker_client.api.build.assert_called_once() - self.assertFileExistsAndContains( - 'Dockerfile', - textwrap.dedent('''\ - # Generated by KFP. - - FROM python:3.9 - - WORKDIR /usr/local/src/kfp/components - COPY requirements.txt requirements.txt - RUN pip install --no-cache-dir -r requirements.txt - - RUN pip install --no-cache-dir kfp==1.8.11 - COPY . . - ''')) - - def testDockerfileCanContainCustomKFPPackage(self): - component = _make_component( - func_name='train', target_image='custom-image') - _write_components('components.py', component) - - result = self._runner.invoke( - self._app, - [ - 'build', - str(self._working_dir), - '--kfp-package-path=/Some/localdir/containing/kfp/source' - ], - ) - self.assertEqual(result.exit_code, 0) - self._docker_client.api.build.assert_called_once() - self.assertFileExistsAndContains( - 'Dockerfile', - textwrap.dedent('''\ - # Generated by KFP. - - FROM python:3.9 - - WORKDIR /usr/local/src/kfp/components - COPY requirements.txt requirements.txt - RUN pip install --no-cache-dir -r requirements.txt - - RUN pip install --no-cache-dir /Some/localdir/containing/kfp/source - COPY . . - ''')) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me/__init__.py b/sdk/python/kfp/deprecated/cli/diagnose_me/__init__.py deleted file mode 100644 index 5d32951430f..00000000000 --- a/sdk/python/kfp/deprecated/cli/diagnose_me/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2019 The Kubeflow Authors. All Rights Reserved. -# -# Licensed under the Apache License,Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing,software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env.py b/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env.py deleted file mode 100644 index 9a5267590c2..00000000000 --- a/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env.py +++ /dev/null @@ -1,71 +0,0 @@ -# Lint as: python3 -# Copyright 2019 The Kubeflow Authors. All Rights Reserved. -# -# Licensed under the Apache License,Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing,software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Functions for diagnostic data collection from development development.""" - -import enum -from kfp.deprecated.cli.diagnose_me import utility - - -class Commands(enum.Enum): - """Enum for gcloud and gsutil commands.""" - PIP3_LIST = 1 - PYTHON3_PIP_LIST = 2 - PIP3_VERSION = 3 - PYHYON3_PIP_VERSION = 4 - WHICH_PYHYON3 = 5 - WHICH_PIP3 = 6 - - -_command_string = { - Commands.PIP3_LIST: 'pip3 list', - Commands.PYTHON3_PIP_LIST: 'python3 -m pip list', - Commands.PIP3_VERSION: 'pip3 -V', - Commands.PYHYON3_PIP_VERSION: 'python3 -m pip -V', - Commands.WHICH_PYHYON3: 'which python3', - Commands.WHICH_PIP3: 'which pip3', -} - - -def get_dev_env_configuration( - configuration: Commands, - human_readable: bool = False) -> utility.ExecutorResponse: - """Captures the specified environment configuration. - - Captures the developement environment configuration including PIP version and - Phython version as specifeid by configuration - - Args: - configuration: Commands for specific information to be retrieved - - PIP3LIST: captures pip3 freeze results - - PYTHON3PIPLIST: captuers python3 -m pip freeze results - - PIP3VERSION: captuers pip3 -V results - - PYHYON3PIPVERSION: captuers python3 -m pip -V results - human_readable: If true all output will be in human readable form insted of - Json. - - Returns: - A utility.ExecutorResponse with the output results for the specified - command. - """ - command_list = _command_string[configuration].split(' ') - if not human_readable and configuration not in ( - Commands.PIP3_VERSION, - Commands.PYHYON3_PIP_VERSION, - Commands.WHICH_PYHYON3, - Commands.WHICH_PIP3, - ): - command_list.extend(['--format', 'json']) - - return utility.ExecutorResponse().execute_command(command_list) diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env_test.py b/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env_test.py deleted file mode 100644 index 66d304d3d95..00000000000 --- a/sdk/python/kfp/deprecated/cli/diagnose_me/dev_env_test.py +++ /dev/null @@ -1,63 +0,0 @@ -# Lint as: python3 -# Copyright 2019 The Kubeflow Authors. All Rights Reserved. -# -# Licensed under the Apache License,Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing,software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Integration tests for diagnose_me.dev_env.""" - -from typing import Text -import unittest -from unittest import mock -from kfp.deprecated.cli.diagnose_me import dev_env -from kfp.deprecated.cli.diagnose_me import utility - - -class DevEnvTest(unittest.TestCase): - - def test_Commands(self): - """Verify commands are formaated properly.""" - for command in dev_env.Commands: - self.assertIsInstance(dev_env._command_string[command], Text) - self.assertNotIn('\t', dev_env._command_string[command]) - self.assertNotIn('\n', dev_env._command_string[command]) - - @mock.patch.object(utility, 'ExecutorResponse', autospec=True) - def test_dev_env_configuration(self, mock_executor_response): - """Tests dev_env command execution.""" - dev_env.get_dev_env_configuration(dev_env.Commands.PIP3_LIST) - mock_executor_response().execute_command.assert_called_with( - ['pip3', 'list', '--format', 'json']) - - @mock.patch.object(utility, 'ExecutorResponse', autospec=True) - def test_dev_env_configuration_human_readable(self, mock_executor_response): - """Tests dev_env command execution.""" - dev_env.get_dev_env_configuration( - dev_env.Commands.PIP3_LIST, human_readable=True) - mock_executor_response().execute_command.assert_called_with( - ['pip3', 'list']) - - @mock.patch.object(utility, 'ExecutorResponse', autospec=True) - def test_dev_env_configuration_version(self, mock_executor_response): - """Tests dev_env command execution.""" - # human readable = false should not set format flag for version calls - dev_env.get_dev_env_configuration( - dev_env.Commands.PIP3_VERSION, human_readable=False) - mock_executor_response().execute_command.assert_called_with( - ['pip3', '-V']) - dev_env.get_dev_env_configuration( - dev_env.Commands.PYHYON3_PIP_VERSION, human_readable=False) - mock_executor_response().execute_command.assert_called_with( - ['python3', '-m', 'pip', '-V']) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me/gcp.py b/sdk/python/kfp/deprecated/cli/diagnose_me/gcp.py deleted file mode 100644 index be643320f81..00000000000 --- a/sdk/python/kfp/deprecated/cli/diagnose_me/gcp.py +++ /dev/null @@ -1,152 +0,0 @@ -# Lint as: python3 -# Copyright 2019 The Kubeflow Authors. All Rights Reserved. -# -# Licensed under the Apache License,Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing,software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Functions for collecting GCP related environment configurations.""" - -import enum -from typing import List, Text, Optional -from kfp.deprecated.cli.diagnose_me import utility - - -class Commands(enum.Enum): - """Enum for gcloud and gsutil commands.""" - GET_APIS = 1 - GET_CONTAINER_CLUSTERS = 2 - GET_CONTAINER_IMAGES = 3 - GET_DISKS = 4 - GET_GCLOUD_DEFAULT = 5 - GET_NETWORKS = 6 - GET_QUOTAS = 7 - GET_SCOPES = 8 - GET_SERVICE_ACCOUNTS = 9 - GET_STORAGE_BUCKETS = 10 - GET_GCLOUD_VERSION = 11 - GET_AUTH_LIST = 12 - - -_command_string = { - Commands.GET_APIS: 'services list', - Commands.GET_CONTAINER_CLUSTERS: 'container clusters list', - Commands.GET_CONTAINER_IMAGES: 'container images list', - Commands.GET_DISKS: 'compute disks list', - Commands.GET_GCLOUD_DEFAULT: 'config list --all', - Commands.GET_NETWORKS: 'compute networks list', - Commands.GET_QUOTAS: 'compute regions list', - Commands.GET_SCOPES: 'compute instances list', - Commands.GET_SERVICE_ACCOUNTS: 'iam service-accounts list', - Commands.GET_STORAGE_BUCKETS: 'ls', - Commands.GET_GCLOUD_VERSION: 'version', - Commands.GET_AUTH_LIST: 'auth list', -} - - -def execute_gcloud_command( - gcloud_command_list: List[Text], - project_id: Optional[Text] = None, - human_readable: Optional[bool] = False) -> utility.ExecutorResponse: - """Function for invoking gcloud command. - - Args: - gcloud_command_list: a command string list to be past to gcloud example - format is ['config', 'list', '--all'] - project_id: specificies the project to run the commands against if not - provided provided will use gcloud default project if one is configured - otherwise will return an error message. - human_readable: If false sets parameter --format json for all calls, - otherwie output will be in human readable format. - - Returns: - utility.ExecutorResponse with outputs from stdout,stderr and execution code. - """ - command_list = ['gcloud'] - command_list.extend(gcloud_command_list) - if not human_readable: - command_list.extend(['--format', 'json']) - - if project_id is not None: - command_list.extend(['--project', project_id]) - - return utility.ExecutorResponse().execute_command(command_list) - - -def execute_gsutil_command( - gsutil_command_list: List[Text], - project_id: Optional[Text] = None) -> utility.ExecutorResponse: - """Function for invoking gsutil command. - - This function takes in a gsutil parameter list and returns the results as a - list of dictionaries. - Args: - gsutil_command_list: a command string list to be past to gsutil example - format is ['config', 'list', '--all'] - project_id: specific project to check the QUOTASs for,if no project id is - provided will use gcloud default project if one is configured otherwise - will return an erro massage. - - Returns: - utility.ExecutorResponse with outputs from stdout,stderr and execution code. - """ - command_list = ['gsutil'] - command_list.extend(gsutil_command_list) - - if project_id is not None: - command_list.extend(['-p', project_id]) - - return utility.ExecutorResponse().execute_command(command_list) - - -def get_gcp_configuration( - configuration: Commands, - project_id: Optional[Text] = None, - human_readable: Optional[bool] = False) -> utility.ExecutorResponse: - """Captures the specified environment configuration. - - Captures the environment configuration for the specified setting such as - NETWORKSing configuration, project QUOTASs, etc. - - Args: - configuration: Commands for specific information to be retrieved - - APIS: Captures a complete list of enabled APISs and their configuration - details under the specified project. - - CONTAINER_CLUSTERS: List all visible k8 clusters under the project. - - CONTAINER_IMAGES: List of all container images under the project - container repo. - - DISKS: List of storage allocated by the project including notebook - instances as well as k8 pds with corresponding state. - - GCLOUD_DEFAULT: Environment default configuration for gcloud - - NETWORKS: List all NETWORKSs and their configuration under the project. - - QUOTAS: Captures a complete list of QUOTASs for project per - region,returns the results as a list of dictionaries. - - SCOPES: list of SCOPESs for each compute resources in the project. - - SERVICE_ACCOUNTS: List of all service accounts that are enabled under - this project. - - STORAGE_BUCKETS: list of buckets and corresponding access information. - project_id: specific project to check the QUOTASs for,if no project id is - provided will use gcloud default project if one is configured otherwise - will return an error message. - human_readable: If true all output will be in human readable form insted of - Json. - - Returns: - A utility.ExecutorResponse with the output results for the specified - command. - """ - # storage bucket call requires execute_gsutil_command - if configuration is Commands.GET_STORAGE_BUCKETS: - return execute_gsutil_command( - [_command_string[Commands.GET_STORAGE_BUCKETS]], project_id) - - # For all other cases can execute the command directly - return execute_gcloud_command(_command_string[configuration].split(' '), - project_id, human_readable) diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me/gcp_test.py b/sdk/python/kfp/deprecated/cli/diagnose_me/gcp_test.py deleted file mode 100644 index 630708675aa..00000000000 --- a/sdk/python/kfp/deprecated/cli/diagnose_me/gcp_test.py +++ /dev/null @@ -1,87 +0,0 @@ -# Lint as: python3 -# Copyright 2019 The Kubeflow Authors. All Rights Reserved. -# -# Licensed under the Apache License,Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing,software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for diagnose_me.gcp.""" - -from typing import Text -import unittest -from unittest import mock -from kfp.deprecated.cli.diagnose_me import gcp -from kfp.deprecated.cli.diagnose_me import utility - - -class GoogleCloudTest(unittest.TestCase): - - @mock.patch.object(gcp, 'execute_gcloud_command', autospec=True) - def test_project_configuration_gcloud(self, mock_execute_gcloud_command): - """Tests gcloud commands.""" - gcp.get_gcp_configuration(gcp.Commands.GET_APIS) - mock_execute_gcloud_command.assert_called_once_with( - ['services', 'list'], project_id=None, human_readable=False) - - @mock.patch.object(gcp, 'execute_gsutil_command', autospec=True) - def test_project_configuration_gsutil(self, mock_execute_gsutil_command): - """Test Gsutil commands.""" - gcp.get_gcp_configuration(gcp.Commands.GET_STORAGE_BUCKETS) - mock_execute_gsutil_command.assert_called_once_with(['ls'], - project_id=None) - - def test_Commands(self): - """Verify commands are formaated properly.""" - for command in gcp.Commands: - self.assertIsInstance(gcp._command_string[command], Text) - self.assertNotIn('\t', gcp._command_string[command]) - self.assertNotIn('\n', gcp._command_string[command]) - - @mock.patch.object(utility, 'ExecutorResponse', autospec=True) - def test_execute_gsutil_command(self, mock_executor_response): - """Test execute_gsutil_command.""" - gcp.execute_gsutil_command( - [gcp._command_string[gcp.Commands.GET_STORAGE_BUCKETS]]) - mock_executor_response().execute_command.assert_called_once_with( - ['gsutil', 'ls']) - - gcp.execute_gsutil_command( - [gcp._command_string[gcp.Commands.GET_STORAGE_BUCKETS]], - project_id='test_project') - mock_executor_response().execute_command.assert_called_with( - ['gsutil', 'ls', '-p', 'test_project']) - - @mock.patch.object(utility, 'ExecutorResponse', autospec=True) - def test_execute_gcloud_command(self, mock_executor_response): - """Test execute_gcloud_command.""" - gcp.execute_gcloud_command( - gcp._command_string[gcp.Commands.GET_APIS].split(' ')) - mock_executor_response().execute_command.assert_called_once_with( - ['gcloud', 'services', 'list', '--format', 'json']) - - gcp.execute_gcloud_command( - gcp._command_string[gcp.Commands.GET_APIS].split(' '), - project_id='test_project') - # verify project id is added correctly - mock_executor_response().execute_command.assert_called_with([ - 'gcloud', 'services', 'list', '--format', 'json', '--project', - 'test_project' - ]) - # verify human_readable removes json fromat flag - gcp.execute_gcloud_command( - gcp._command_string[gcp.Commands.GET_APIS].split(' '), - project_id='test_project', - human_readable=True) - mock_executor_response().execute_command.assert_called_with( - ['gcloud', 'services', 'list', '--project', 'test_project']) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster.py b/sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster.py deleted file mode 100644 index 43d44adfdbd..00000000000 --- a/sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster.py +++ /dev/null @@ -1,124 +0,0 @@ -# Lint as: python3 -# Copyright 2019 The Kubeflow Authors. All Rights Reserved. -# -# Licensed under the Apache License,Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing,software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Functions for collecting diagnostic information on Kubernetes cluster.""" - -import enum -from typing import List, Text -from kfp.deprecated.cli.diagnose_me import utility - - -class Commands(enum.Enum): - """Enum for kubernetes commands.""" - GET_CONFIGURED_CONTEXT = 1 - GET_PODS = 2 - GET_PVCS = 3 - GET_PVS = 4 - GET_SECRETS = 5 - GET_SERVICES = 6 - GET_KUBECTL_VERSION = 7 - GET_CONFIG_MAPS = 8 - - -_command_string = { - Commands.GET_CONFIGURED_CONTEXT: 'config view', - Commands.GET_PODS: 'get pods', - Commands.GET_PVCS: 'get pvc', - Commands.GET_PVS: 'get pv', - Commands.GET_SECRETS: 'get secrets', - Commands.GET_SERVICES: 'get services', - Commands.GET_KUBECTL_VERSION: 'version', - Commands.GET_CONFIG_MAPS: 'get configmaps', -} - - -def execute_kubectl_command( - kubectl_command_list: List[Text], - human_readable: bool = False) -> utility.ExecutorResponse: - """Invokes the kubectl command. - - Args: - kubectl_command_list: a command string list to be past to kubectl example - format is ['config', 'view'] - human_readable: If false sets parameter -o json for all calls, otherwie - output will be in human readable format. - - Returns: - utility.ExecutorResponse with outputs from stdout,stderr and execution code. - """ - command_list = ['kubectl'] - command_list.extend(kubectl_command_list) - if not human_readable: - command_list.extend(['-o', 'json']) - - return utility.ExecutorResponse().execute_command(command_list) - - -def get_kubectl_configuration( - configuration: Commands, - kubernetes_context: Text = None, - namespace: Text = None, - human_readable: bool = False) -> utility.ExecutorResponse: - """Captures the specified environment configuration. - - Captures the environment state for the specified setting such as current - context, active pods, etc and returns it in as a dictionary format. if no - context is specified the system will use the current_context or error out of - none is specified. - - Args: - configuration: - - K8_CONFIGURED_CONTEXT: returns all k8 configuration available in the - current env including current_context. - - PODS: returns all pods and their status details. - - PVCS: returns all PersistentVolumeClaim and their status details. - - SECRETS: returns all accessible k8 secrests. - - PVS: returns all PersistentVolume and their status details. - - SERVICES: returns all services and their status details. - kubernetes_context: Context to use to retrieve cluster specific commands, if - set to None calls will rely on current_context configured. - namespace: default name space to be used for the commaand, if not specifeid - --all-namespaces will be used. - human_readable: If true all output will be in human readable form insted of - Json. - - Returns: - A list of dictionaries matching gcloud / gsutil output for the specified - configuration,or an error message if any occurs during execution. - """ - - if configuration in (Commands.GET_CONFIGURED_CONTEXT, - Commands.GET_KUBECTL_VERSION): - return execute_kubectl_command( - (_command_string[configuration]).split(' '), human_readable) - - execution_command = _command_string[configuration].split(' ') - if kubernetes_context: - execution_command.extend(['--context', kubernetes_context]) - if namespace: - execution_command.extend(['--namespace', namespace]) - else: - execution_command.extend(['--all-namespaces']) - - return execute_kubectl_command(execution_command, human_readable) - - -def _get_kfp_runtime() -> Text: - """Captures the current version of kpf in k8 cluster. - - Returns: - Returns the run-time version of kfp in as a string. - """ - # TODO(chavoshi) needs to be implemented. - raise NotImplementedError diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster_test.py b/sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster_test.py deleted file mode 100644 index e2409797b22..00000000000 --- a/sdk/python/kfp/deprecated/cli/diagnose_me/kubernetes_cluster_test.py +++ /dev/null @@ -1,76 +0,0 @@ -# Lint as: python3 -# Copyright 2019 The Kubeflow Authors. All Rights Reserved. -# -# Licensed under the Apache License,Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing,software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for diagnose_me.kubernetes_cluster.""" - -from typing import Text -import unittest -from unittest import mock -from kfp.deprecated.cli.diagnose_me import kubernetes_cluster as dkc -from kfp.deprecated.cli.diagnose_me import utility - - -class KubernetesClusterTest(unittest.TestCase): - - @mock.patch.object(dkc, 'execute_kubectl_command', autospec=True) - def test_project_configuration_gcloud(self, mock_execute_kubectl_command): - """Tests gcloud commands.""" - dkc.get_kubectl_configuration(dkc.Commands.GET_PODS) - mock_execute_kubectl_command.assert_called_once_with( - ['get', 'pods', '--all-namespaces'], human_readable=False) - - dkc.get_kubectl_configuration(dkc.Commands.GET_CONFIGURED_CONTEXT) - mock_execute_kubectl_command.assert_called_with(['config', 'view'], - human_readable=False) - - dkc.get_kubectl_configuration(dkc.Commands.GET_KUBECTL_VERSION) - mock_execute_kubectl_command.assert_called_with(['version'], - human_readable=False) - - dkc.get_kubectl_configuration( - dkc.Commands.GET_PODS, kubernetes_context='test_context') - mock_execute_kubectl_command.assert_called_with( - ['get', 'pods', '--context', 'test_context', '--all-namespaces'], - human_readable=False) - - dkc.get_kubectl_configuration( - dkc.Commands.GET_PODS, kubernetes_context='test_context') - mock_execute_kubectl_command.assert_called_with( - ['get', 'pods', '--context', 'test_context', '--all-namespaces'], - human_readable=False) - - def test_Commands(self): - """Verify commands are formaated properly.""" - for command in dkc.Commands: - self.assertIsInstance(dkc._command_string[command], Text) - self.assertNotIn('\t', dkc._command_string[command]) - self.assertNotIn('\n', dkc._command_string[command]) - - @mock.patch.object(utility, 'ExecutorResponse', autospec=True) - def test_execute_kubectl_command(self, mock_executor_response): - """Test execute_gsutil_command.""" - dkc.execute_kubectl_command( - [dkc._command_string[dkc.Commands.GET_KUBECTL_VERSION]]) - mock_executor_response().execute_command.assert_called_once_with( - ['kubectl', 'version', '-o', 'json']) - - dkc.execute_kubectl_command( - [dkc._command_string[dkc.Commands.GET_KUBECTL_VERSION]], - human_readable=True) - mock_executor_response().execute_command.assert_called_with( - ['kubectl', 'version']) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me/utility.py b/sdk/python/kfp/deprecated/cli/diagnose_me/utility.py deleted file mode 100644 index f83984a091f..00000000000 --- a/sdk/python/kfp/deprecated/cli/diagnose_me/utility.py +++ /dev/null @@ -1,91 +0,0 @@ -# Lint as: python3 -# Copyright 2019 The Kubeflow Authors. All Rights Reserved. -# -# Licensed under the Apache License,Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing,software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Supporting tools and classes for diagnose_me.""" - -import json -import subprocess -from typing import List, Text - - -class ExecutorResponse(object): - """Class for keeping track of output of _executor methods. - - Data model for executing commands and capturing their response. This class - defines the data model layer for execution results, based on MVC design - pattern. - - TODO() This class should be extended to contain data structure to better - represent the underlying data instaed of dict for various response types. - """ - - def execute_command(self, command_list: List[Text]): - """Executes the command in command_list. - - sets values for _stdout,_std_err, and returncode accordingly. - - TODO(): This method is kept in ExecutorResponse for simplicity, however this - deviates from MVP design pattern. It should be factored out in future. - - Args: - command_list: A List of strings that represts the command and parameters - to be executed. - - Returns: - Instance of utility.ExecutorResponse. - """ - - try: - # TODO() switch to process.run to simplify the code. - process = subprocess.Popen( - command_list, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - self._stdout = stdout.decode('utf-8') - self._stderr = stderr.decode('utf-8') - self._returncode = process.returncode - except OSError as e: - self._stderr = e - self._stdout = '' - self._returncode = e.errno - self._parse_raw_input() - return self - - def _parse_raw_input(self): - """Parses the raw input and popluates _json and _parsed properies.""" - try: - self._parsed_output = json.loads(self._stdout) - self._json = self._stdout - except json.JSONDecodeError: - self._json = json.dumps(self._stdout) - self._parsed_output = self._stdout - - @property - def parsed_output(self) -> Text: - """Json load results of stdout or raw results if stdout was not - Json.""" - return self._parsed_output - - @property - def has_error(self) -> bool: - """Returns true if execution error code was not 0.""" - return self._returncode != 0 - - @property - def json_output(self) -> Text: - """Run results in stdout in json format.""" - return self._parsed_output - - @property - def stderr(self): - return self._stderr diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me/utility_test.py b/sdk/python/kfp/deprecated/cli/diagnose_me/utility_test.py deleted file mode 100644 index f339d711429..00000000000 --- a/sdk/python/kfp/deprecated/cli/diagnose_me/utility_test.py +++ /dev/null @@ -1,43 +0,0 @@ -# Lint as: python3 -# Copyright 2019 The Kubeflow Authors. All Rights Reserved. -# -# Licensed under the Apache License,Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing,software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for diagnose_me.utility.""" - -import unittest -from kfp.deprecated.cli.diagnose_me import utility - - -class UtilityTest(unittest.TestCase): - - def test_parse_raw_input_json(self): - """Testing json stdout is correctly parsed.""" - response = utility.ExecutorResponse() - response._stdout = '{"key":"value"}' - response._parse_raw_input() - - self.assertEqual(response._json, '{"key":"value"}') - self.assertEqual(response._parsed_output, {'key': 'value'}) - - def test_parse_raw_input_text(self): - """Testing non-json stdout is correctly parsed.""" - response = utility.ExecutorResponse() - response._stdout = 'non-json string' - response._parse_raw_input() - - self.assertEqual(response._json, '"non-json string"') - self.assertEqual(response._parsed_output, 'non-json string') - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/cli/diagnose_me_cli.py b/sdk/python/kfp/deprecated/cli/diagnose_me_cli.py deleted file mode 100644 index b23f86b1f48..00000000000 --- a/sdk/python/kfp/deprecated/cli/diagnose_me_cli.py +++ /dev/null @@ -1,106 +0,0 @@ -# Lint as: python3 -"""CLI interface for KFP diagnose_me tool.""" - -import json as json_library -import sys -from typing import Dict, Text -import click -from kfp.deprecated.cli.diagnose_me import dev_env -from kfp.deprecated.cli.diagnose_me import gcp -from kfp.deprecated.cli.diagnose_me import kubernetes_cluster as k8 -from kfp.deprecated.cli.diagnose_me import utility - - -@click.group() -def diagnose_me(): - """Prints diagnoses information for KFP environment.""" - - -@diagnose_me.command() -@click.option( - '-j', - '--json', - is_flag=True, - help='Output in Json format, human readable format is set by default.') -@click.option( - '-p', - '--project-id', - type=Text, - help='Target project id. It will use environment default if not specified.') -@click.option( - '-n', - '--namespace', - type=Text, - help='Namespace to use for Kubernetes cluster.all-namespaces is used if not specified.' -) -@click.pass_context -def diagnose_me(ctx: click.Context, json: bool, project_id: str, - namespace: str): - """Runs environment diagnostic with specified parameters. - - Feature stage: - [Alpha](https://github.com/kubeflow/pipelines/blob/07328e5094ac2981d3059314cc848fbb71437a76/docs/release/feature-stages.md#alpha) - """ - # validate kubectl, gcloud , and gsutil exist - local_env_gcloud_sdk = gcp.get_gcp_configuration( - gcp.Commands.GET_GCLOUD_VERSION, - project_id=project_id, - human_readable=False) - for app in ['Google Cloud SDK', 'gsutil', 'kubectl']: - if app not in local_env_gcloud_sdk.json_output: - raise RuntimeError( - '%s is not installed, gcloud, gsutil and kubectl are required ' - % app + 'for this app to run. Please follow instructions at ' + - 'https://cloud.google.com/sdk/install to install the SDK.') - - click.echo('Collecting diagnostic information ...', file=sys.stderr) - - # default behaviour dump all configurations - results = {} - for gcp_command in gcp.Commands: - results[gcp_command] = gcp.get_gcp_configuration( - gcp_command, project_id=project_id, human_readable=not json) - - for k8_command in k8.Commands: - results[k8_command] = k8.get_kubectl_configuration( - k8_command, human_readable=not json) - - for dev_env_command in dev_env.Commands: - results[dev_env_command] = dev_env.get_dev_env_configuration( - dev_env_command, human_readable=not json) - - print_to_sdtout(results, not json) - - -def print_to_sdtout(results: Dict[str, utility.ExecutorResponse], - human_readable: bool): - """Viewer to print the ExecutorResponse results to stdout. - - Args: - results: A dictionary with key:command names and val: Execution response - human_readable: Print results in human readable format. If set to True - command names will be printed as visual delimiters in new lines. If False - results are printed as a dictionary with command as key. - """ - - output_dict = {} - human_readable_result = [] - for key, val in results.items(): - if val.has_error: - output_dict[ - key. - name] = 'Following error occurred during the diagnoses: %s' % val.stderr - continue - - output_dict[key.name] = val.json_output - human_readable_result.append('================ %s ===================' % - (key.name)) - human_readable_result.append(val.parsed_output) - - if human_readable: - result = '\n'.join(human_readable_result) - else: - result = json_library.dumps( - output_dict, sort_keys=True, indent=2, separators=(',', ': ')) - - click.echo(result) diff --git a/sdk/python/kfp/deprecated/cli/experiment.py b/sdk/python/kfp/deprecated/cli/experiment.py deleted file mode 100644 index 543f66c584f..00000000000 --- a/sdk/python/kfp/deprecated/cli/experiment.py +++ /dev/null @@ -1,143 +0,0 @@ -import click -import json -from typing import List - -from kfp.deprecated.cli.output import print_output, OutputFormat -import kfp_server_api -from kfp_server_api.models.api_experiment import ApiExperiment - - -@click.group() -def experiment(): - """Manage experiment resources.""" - - -@experiment.command() -@click.option('-d', '--description', help="Description of the experiment.") -@click.argument("name") -@click.pass_context -def create(ctx: click.Context, description: str, name: str): - """Create an experiment.""" - client = ctx.obj["client"] - output_format = ctx.obj["output"] - - response = client.create_experiment(name, description=description) - _display_experiment(response, output_format) - - -@experiment.command() -@click.option( - '--page-token', default='', help="Token for starting of the page.") -@click.option( - '-m', '--max-size', default=100, help="Max size of the listed experiments.") -@click.option( - '--sort-by', - default="created_at desc", - help="Can be '[field_name]', '[field_name] desc'. For example, 'name desc'." -) -@click.option( - '--filter', - help=( - "filter: A url-encoded, JSON-serialized Filter protocol buffer " - "(see [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/filter.proto))." - )) -@click.pass_context -def list(ctx: click.Context, page_token: str, max_size: int, sort_by: str, - filter: str): - """List experiments.""" - client = ctx.obj['client'] - output_format = ctx.obj['output'] - - response = client.list_experiments( - page_token=page_token, - page_size=max_size, - sort_by=sort_by, - filter=filter) - if response.experiments: - _display_experiments(response.experiments, output_format) - else: - if output_format == OutputFormat.json.name: - msg = json.dumps([]) - else: - msg = "No experiments found" - click.echo(msg) - - -@experiment.command() -@click.argument("experiment-id") -@click.pass_context -def get(ctx: click.Context, experiment_id: str): - """Get detailed information about an experiment.""" - client = ctx.obj["client"] - output_format = ctx.obj["output"] - - response = client.get_experiment(experiment_id) - _display_experiment(response, output_format) - - -@experiment.command() -@click.argument("experiment-id") -@click.pass_context -def delete(ctx: click.Context, experiment_id: str): - """Delete an experiment.""" - - confirmation = "Caution. The RunDetails page could have an issue" \ - " when it renders a run that has no experiment." \ - " Do you want to continue?" - if not click.confirm(confirmation): - return - - client = ctx.obj["client"] - - client.delete_experiment(experiment_id) - click.echo("{} is deleted.".format(experiment_id)) - - -def _display_experiments(experiments: List[ApiExperiment], - output_format: OutputFormat): - headers = ["Experiment ID", "Name", "Created at"] - data = [ - [exp.id, exp.name, exp.created_at.isoformat()] for exp in experiments - ] - print_output(data, headers, output_format, table_format="grid") - - -def _display_experiment(exp: kfp_server_api.ApiExperiment, - output_format: OutputFormat): - table = [ - ["ID", exp.id], - ["Name", exp.name], - ["Description", exp.description], - ["Created at", exp.created_at.isoformat()], - ] - if output_format == OutputFormat.table.name: - print_output([], ["Experiment Details"], output_format) - print_output(table, [], output_format, table_format="plain") - elif output_format == OutputFormat.json.name: - print_output(dict(table), [], output_format) - - -@experiment.command() -@click.option( - "--experiment-id", - default=None, - help="The ID of the experiment to archive, can only supply either an experiment ID or name." -) -@click.option( - "--experiment-name", - default=None, - help="The name of the experiment to archive, can only supply either an experiment ID or name." -) -@click.pass_context -def archive(ctx: click.Context, experiment_id: str, experiment_name: str): - """Archive an experiment.""" - client = ctx.obj["client"] - - if (experiment_id is None) == (experiment_name is None): - raise ValueError('Either experiment_id or experiment_name is required') - - if not experiment_id: - experiment = client.get_experiment(experiment_name=experiment_name) - experiment_id = experiment.id - - client.archive_experiment(experiment_id=experiment_id) diff --git a/sdk/python/kfp/deprecated/cli/output.py b/sdk/python/kfp/deprecated/cli/output.py deleted file mode 100644 index a5e4440ccc3..00000000000 --- a/sdk/python/kfp/deprecated/cli/output.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import click -import json -from enum import Enum, unique -from typing import Union - -from tabulate import tabulate - - -@unique -class OutputFormat(Enum): - """Enumerated class with the allowed output format constants.""" - table = "table" - json = "json" - - -def print_output(data: Union[list, dict], - headers: list, - output_format: str, - table_format: str = "simple"): - """Prints the output from the cli command execution based on the specified - format. - - Args: - data (Union[list, dict]): Nested list of values representing the rows to be printed. - headers (list): List of values representing the column names to be printed - for the ``data``. - output_format (str): The desired formatting of the text from the command output. - table_format (str): The formatting for the table ``output_format``. - Default value set to ``simple``. - - Returns: - None: Prints the output. - - Raises: - NotImplementedError: If the ``output_format`` is unknown. - """ - if output_format == OutputFormat.table.name: - click.echo(tabulate(data, headers=headers, tablefmt=table_format)) - elif output_format == OutputFormat.json.name: - if not headers: - output = data - else: - output = [] - for row in data: - output.append(dict(zip(headers, row))) - click.echo(json.dumps(output, indent=4)) - else: - raise NotImplementedError( - "Unknown Output Format: {}".format(output_format)) diff --git a/sdk/python/kfp/deprecated/cli/pipeline.py b/sdk/python/kfp/deprecated/cli/pipeline.py deleted file mode 100644 index 2d03f7b1893..00000000000 --- a/sdk/python/kfp/deprecated/cli/pipeline.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import click -import json -from typing import List, Optional - -import kfp_server_api -from kfp.deprecated.cli.output import print_output, OutputFormat - - -@click.group() -def pipeline(): - """Manage pipeline resources.""" - - -@pipeline.command() -@click.option("-p", "--pipeline-name", help="Name of the pipeline.") -@click.option("-d", "--description", help="Description for the pipeline.") -@click.argument("package-file") -@click.pass_context -def upload(ctx: click.Context, - pipeline_name: str, - package_file: str, - description: str = None): - """Upload a KFP pipeline.""" - client = ctx.obj["client"] - output_format = ctx.obj["output"] - if not pipeline_name: - pipeline_name = package_file.split(".")[0] - - pipeline = client.upload_pipeline(package_file, pipeline_name, description) - _display_pipeline(pipeline, output_format) - - -@pipeline.command() -@click.option("-p", "--pipeline-id", help="ID of the pipeline", required=False) -@click.option("-n", "--pipeline-name", help="Name of pipeline", required=False) -@click.option( - "-v", - "--pipeline-version", - help="Name of the pipeline version", - required=True) -@click.argument("package-file") -@click.pass_context -def upload_version(ctx: click.Context, - package_file: str, - pipeline_version: str, - pipeline_id: Optional[str] = None, - pipeline_name: Optional[str] = None): - """Upload a version of the KFP pipeline.""" - client = ctx.obj["client"] - output_format = ctx.obj["output"] - if bool(pipeline_id) == bool(pipeline_name): - raise ValueError("Need to supply 'pipeline-name' or 'pipeline-id'") - if pipeline_name is not None: - pipeline_id = client.get_pipeline_id(name=pipeline_name) - if pipeline_id is None: - raise ValueError("Can't find a pipeline with name: %s" % - pipeline_name) - version = client.pipeline_uploads.upload_pipeline_version( - package_file, name=pipeline_version, pipelineid=pipeline_id) - _display_pipeline_version(version, output_format) - - -@pipeline.command() -@click.option( - '--page-token', default='', help="Token for starting of the page.") -@click.option( - '-m', '--max-size', default=100, help="Max size of the listed pipelines.") -@click.option( - '--sort-by', - default="created_at desc", - help="Can be '[field_name]', '[field_name] desc'. For example, 'name desc'." -) -@click.option( - '--filter', - help=( - "filter: A url-encoded, JSON-serialized Filter protocol buffer " - "(see [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/filter.proto))." - )) -@click.pass_context -def list(ctx: click.Context, page_token: str, max_size: int, sort_by: str, - filter: str): - """List uploaded KFP pipelines.""" - client = ctx.obj["client"] - output_format = ctx.obj["output"] - - response = client.list_pipelines( - page_token=page_token, - page_size=max_size, - sort_by=sort_by, - filter=filter) - if response.pipelines: - _print_pipelines(response.pipelines, output_format) - else: - if output_format == OutputFormat.json.name: - msg = json.dumps([]) - else: - msg = "No pipelines found" - click.echo(msg) - - -@pipeline.command() -@click.argument("pipeline-id") -@click.option( - '--page-token', default='', help="Token for starting of the page.") -@click.option( - '-m', - '--max-size', - default=100, - help="Max size of the listed pipeline versions.") -@click.option( - '--sort-by', - default="created_at desc", - help="Can be '[field_name]', '[field_name] desc'. For example, 'name desc'." -) -@click.option( - '--filter', - help=( - "filter: A url-encoded, JSON-serialized Filter protocol buffer " - "(see [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/filter.proto))." - )) -@click.pass_context -def list_versions(ctx: click.Context, pipeline_id: str, page_token: str, - max_size: int, sort_by: str, filter: str): - """List versions of an uploaded KFP pipeline.""" - client = ctx.obj["client"] - output_format = ctx.obj["output"] - - response = client.list_pipeline_versions( - pipeline_id, - page_token=page_token, - page_size=max_size, - sort_by=sort_by, - filter=filter) - if response.versions: - _print_pipeline_versions(response.versions, output_format) - else: - if output_format == OutputFormat.json.name: - msg = json.dumps([]) - else: - msg = "No pipeline or version found" - click.echo(msg) - - -@pipeline.command() -@click.argument("version-id") -@click.pass_context -def delete_version(ctx: click.Context, version_id: str): - """Delete pipeline version. - - Args: - version_id: id of the pipeline version. - - Returns: - Object. If the method is called asynchronously, returns the request thread. - - Throws: - Exception if pipeline version is not found. - """ - client = ctx.obj["client"] - return client.delete_pipeline_version(version_id) - - -@pipeline.command() -@click.argument("pipeline-id") -@click.pass_context -def get(ctx: click.Context, pipeline_id: str): - """Get detailed information about an uploaded KFP pipeline.""" - client = ctx.obj["client"] - output_format = ctx.obj["output"] - - pipeline = client.get_pipeline(pipeline_id) - _display_pipeline(pipeline, output_format) - - -@pipeline.command() -@click.argument("pipeline-id") -@click.pass_context -def delete(ctx: click.Context, pipeline_id: str): - """Delete an uploaded KFP pipeline.""" - client = ctx.obj["client"] - - client.delete_pipeline(pipeline_id) - click.echo(f"{pipeline_id} is deleted") - - -def _print_pipelines(pipelines: List[kfp_server_api.ApiPipeline], - output_format: OutputFormat): - headers = ["Pipeline ID", "Name", "Uploaded at"] - data = [[pipeline.id, pipeline.name, - pipeline.created_at.isoformat()] for pipeline in pipelines] - print_output(data, headers, output_format, table_format="grid") - - -def _print_pipeline_versions(versions: List[kfp_server_api.ApiPipelineVersion], - output_format: OutputFormat): - headers = ["Version ID", "Version name", "Uploaded at", "Pipeline ID"] - data = [[ - version.id, version.name, - version.created_at.isoformat(), - next(rr - for rr in version.resource_references - if rr.key.type == kfp_server_api.ApiResourceType.PIPELINE).key.id - ] - for version in versions] - print_output(data, headers, output_format, table_format="grid") - - -def _display_pipeline(pipeline: kfp_server_api.ApiPipeline, - output_format: OutputFormat): - # Pipeline information - table = [["Pipeline ID", pipeline.id], ["Name", pipeline.name], - ["Description", pipeline.description], - ["Uploaded at", pipeline.created_at.isoformat()], - ["Version ID", pipeline.default_version.id]] - - # Pipeline parameter details - headers = ["Parameter Name", "Default Value"] - data = [] - if pipeline.parameters is not None: - data = [[param.name, param.value] for param in pipeline.parameters] - - if output_format == OutputFormat.table.name: - print_output([], ["Pipeline Details"], output_format) - print_output(table, [], output_format, table_format="plain") - print_output(data, headers, output_format, table_format="grid") - elif output_format == OutputFormat.json.name: - output = dict() - output["Pipeline Details"] = dict(table) - params = [] - for item in data: - params.append(dict(zip(headers, item))) - output["Pipeline Parameters"] = params - print_output(output, [], output_format) - - -def _display_pipeline_version(version: kfp_server_api.ApiPipelineVersion, - output_format: OutputFormat): - pipeline_id = next( - rr for rr in version.resource_references - if rr.key.type == kfp_server_api.ApiResourceType.PIPELINE).key.id - table = [["Pipeline ID", pipeline_id], ["Version name", version.name], - ["Uploaded at", version.created_at.isoformat()], - ["Version ID", version.id]] - - if output_format == OutputFormat.table.name: - print_output([], ["Pipeline Version Details"], output_format) - print_output(table, [], output_format, table_format="plain") - elif output_format == OutputFormat.json.name: - print_output(dict(table), [], output_format) diff --git a/sdk/python/kfp/deprecated/cli/recurring_run.py b/sdk/python/kfp/deprecated/cli/recurring_run.py deleted file mode 100644 index 8126e5f0479..00000000000 --- a/sdk/python/kfp/deprecated/cli/recurring_run.py +++ /dev/null @@ -1,214 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Any, Dict, List, Optional -import json -import click -from kfp.deprecated.cli.output import print_output, OutputFormat -import kfp_server_api - - -@click.group() -def recurring_run(): - """Manage recurring-run resources.""" - - -@recurring_run.command() -@click.option( - '--catchup/--no-catchup', - help='Whether the recurring run should catch up if behind schedule.', - type=bool) -@click.option( - '--cron-expression', - help='A cron expression representing a set of times, using 6 space-separated fields, e.g. "0 0 9 ? * 2-6".' -) -@click.option('--description', help='The description of the recurring run.') -@click.option( - '--enable-caching/--disable-caching', - help='Optional. Whether or not to enable caching for the run.', - type=bool) -@click.option( - '--enabled/--disabled', - help='A bool indicating whether the recurring run is enabled or disabled.', - type=bool) -@click.option( - '--end-time', - help='The RFC3339 time string of the time when to end the job.') -@click.option( - '--experiment-id', - help='The ID of the experiment to create the recurring run under, can only supply either an experiment ID or name.' -) -@click.option( - '--experiment-name', - help='The name of the experiment to create the recurring run under, can only supply either an experiment ID or name.' -) -@click.option('--job-name', help='The name of the recurring run.') -@click.option( - '--interval-second', - help='Integer indicating the seconds between two recurring runs in for a periodic schedule.' -) -@click.option( - '--max-concurrency', - help='Integer indicating how many jobs can be run in parallel.', - type=int) -@click.option( - '--pipeline-id', - help='The ID of the pipeline to use to create the recurring run.') -@click.option( - '--pipeline-package-path', - help='Local path of the pipeline package(the filename should end with one of the following .tar.gz, .tgz, .zip, .yaml, .yml).' -) -@click.option( - '--start-time', - help='The RFC3339 time string of the time when to start the job.') -@click.option('--version-id', help='The id of a pipeline version.') -@click.argument("args", nargs=-1) -@click.pass_context -def create(ctx: click.Context, - job_name: str, - experiment_id: Optional[str] = None, - experiment_name: Optional[str] = None, - catchup: Optional[bool] = None, - cron_expression: Optional[str] = None, - enabled: Optional[bool] = None, - description: Optional[str] = None, - enable_caching: Optional[bool] = None, - end_time: Optional[str] = None, - interval_second: Optional[int] = None, - max_concurrency: Optional[int] = None, - params: Optional[dict] = None, - pipeline_package_path: Optional[str] = None, - pipeline_id: Optional[str] = None, - start_time: Optional[str] = None, - version_id: Optional[str] = None, - args: Optional[List[str]] = None): - """Create a recurring run.""" - client = ctx.obj["client"] - output_format = ctx.obj['output'] - - if (experiment_id is None) == (experiment_name is None): - raise ValueError('Either experiment_id or experiment_name is required') - if not experiment_id: - experiment = client.create_experiment(experiment_name) - experiment_id = experiment.id - - # Ensure we only split on the first equals char so the value can contain - # equals signs too. - split_args: List = [arg.split("=", 1) for arg in args or []] - params: Dict[str, Any] = dict(split_args) - recurring_run = client.create_recurring_run( - cron_expression=cron_expression, - description=description, - enabled=enabled, - enable_caching=enable_caching, - end_time=end_time, - experiment_id=experiment_id, - interval_second=interval_second, - job_name=job_name, - max_concurrency=max_concurrency, - no_catchup=not catchup, - params=params, - pipeline_package_path=pipeline_package_path, - pipeline_id=pipeline_id, - start_time=start_time, - version_id=version_id) - - _display_recurring_run(recurring_run, output_format) - - -@recurring_run.command() -@click.option( - '-e', - '--experiment-id', - help='Parent experiment ID of listed recurring runs.') -@click.option( - '--page-token', default='', help="Token for starting of the page.") -@click.option( - '-m', - '--max-size', - default=100, - help="Max size of the listed recurring runs.") -@click.option( - '--sort-by', - default="created_at desc", - help="Can be '[field_name]', '[field_name] desc'. For example, 'name desc'." -) -@click.option( - '--filter', - help=( - "filter: A url-encoded, JSON-serialized Filter protocol buffer " - "(see [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/filter.proto))." - )) -@click.pass_context -def list(ctx: click.Context, experiment_id: str, page_token: str, max_size: int, - sort_by: str, filter: str): - """List recurring runs.""" - client = ctx.obj['client'] - output_format = ctx.obj['output'] - - response = client.list_recurring_runs( - experiment_id=experiment_id, - page_token=page_token, - page_size=max_size, - sort_by=sort_by, - filter=filter) - if response.jobs: - _display_recurring_runs(response.jobs, output_format) - else: - if output_format == OutputFormat.json.name: - msg = json.dumps([]) - else: - msg = "No recurring runs found" - click.echo(msg) - - -@recurring_run.command() -@click.argument("job-id") -@click.pass_context -def get(ctx: click.Context, job_id: str): - """Get detailed information about an experiment.""" - client = ctx.obj["client"] - output_format = ctx.obj["output"] - - response = client.get_recurring_run(job_id) - _display_recurring_run(response, output_format) - - -@recurring_run.command() -@click.argument("job-id") -@click.pass_context -def delete(ctx: click.Context, job_id: str): - """Delete a recurring run.""" - client = ctx.obj["client"] - client.delete_job(job_id) - - -def _display_recurring_runs(recurring_runs: List[kfp_server_api.ApiJob], - output_format: OutputFormat): - headers = ["Recurring Run ID", "Name"] - data = [[rr.id, rr.name] for rr in recurring_runs] - print_output(data, headers, output_format, table_format="grid") - - -def _display_recurring_run(recurring_run: kfp_server_api.ApiJob, - output_format: OutputFormat): - table = [ - ["Recurring Run ID", recurring_run.id], - ["Name", recurring_run.name], - ] - if output_format == OutputFormat.table.name: - print_output([], ["Recurring Run Details"], output_format) - print_output(table, [], output_format, table_format="plain") - elif output_format == OutputFormat.json.name: - print_output(dict(table), [], output_format) diff --git a/sdk/python/kfp/deprecated/cli/run.py b/sdk/python/kfp/deprecated/cli/run.py deleted file mode 100644 index 8410a1239ab..00000000000 --- a/sdk/python/kfp/deprecated/cli/run.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import sys -import subprocess -import time -import json -import click -import shutil -import datetime -from typing import List - -import kfp_server_api -from kfp.deprecated._client import Client -from kfp.deprecated.cli.output import print_output, OutputFormat - - -@click.group() -def run(): - """manage run resources.""" - - -@run.command() -@click.option( - '-e', '--experiment-id', help='Parent experiment ID of listed runs.') -@click.option( - '--page-token', default='', help="Token for starting of the page.") -@click.option( - '-m', '--max-size', default=100, help="Max size of the listed runs.") -@click.option( - '--sort-by', - default="created_at desc", - help="Can be '[field_name]', '[field_name] desc'. For example, 'name desc'." -) -@click.option( - '--filter', - help=( - "filter: A url-encoded, JSON-serialized Filter protocol buffer " - "(see [filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/filter.proto))." - )) -@click.pass_context -def list(ctx: click.Context, experiment_id: str, page_token: str, max_size: int, - sort_by: str, filter: str): - """list recent KFP runs.""" - client = ctx.obj['client'] - output_format = ctx.obj['output'] - response = client.list_runs( - experiment_id=experiment_id, - page_token=page_token, - page_size=max_size, - sort_by=sort_by, - filter=filter) - if response and response.runs: - _print_runs(response.runs, output_format) - else: - if output_format == OutputFormat.json.name: - msg = json.dumps([]) - else: - msg = 'No runs found.' - click.echo(msg) - - -@run.command() -@click.option( - '-e', - '--experiment-name', - required=True, - help='Experiment name of the run.') -@click.option('-r', '--run-name', help='Name of the run.') -@click.option( - '-f', - '--package-file', - type=click.Path(exists=True, dir_okay=False), - help='Path of the pipeline package file.') -@click.option('-p', '--pipeline-id', help='ID of the pipeline template.') -@click.option('-n', '--pipeline-name', help='Name of the pipeline template.') -@click.option( - '-w', - '--watch', - is_flag=True, - default=False, - help='Watch the run status until it finishes.') -@click.option('-v', '--version', help='ID of the pipeline version.') -@click.option( - '-t', - '--timeout', - default=0, - help='Wait for a run to complete until timeout in seconds.', - type=int) -@click.argument('args', nargs=-1) -@click.pass_context -def submit(ctx: click.Context, experiment_name: str, run_name: str, - package_file: str, pipeline_id: str, pipeline_name: str, watch: bool, - timeout: int, version: str, args: List[str]): - """submit a KFP run.""" - client = ctx.obj['client'] - namespace = ctx.obj['namespace'] - output_format = ctx.obj['output'] - if not run_name: - run_name = experiment_name - - if not pipeline_id and pipeline_name: - pipeline_id = client.get_pipeline_id(name=pipeline_name) - - if not package_file and not pipeline_id and not version: - click.echo( - 'You must provide one of [package_file, pipeline_id, version].', - err=True) - sys.exit(1) - - arg_dict = dict(arg.split('=', maxsplit=1) for arg in args) - - experiment = client.create_experiment(experiment_name) - run = client.run_pipeline( - experiment.id, - run_name, - package_file, - arg_dict, - pipeline_id, - version_id=version) - if timeout > 0: - _wait_for_run_completion(client, run.id, timeout, output_format) - else: - _display_run(client, namespace, run.id, watch, output_format) - - -@run.command() -@click.option( - '-w', - '--watch', - is_flag=True, - default=False, - help='Watch the run status until it finishes.') -@click.option( - '-d', - '--detail', - is_flag=True, - default=False, - help='Get detailed information of the run in json format.') -@click.argument('run-id') -@click.pass_context -def get(ctx: click.Context, watch: bool, detail: bool, run_id: str): - """display the details of a KFP run.""" - client = ctx.obj['client'] - namespace = ctx.obj['namespace'] - output_format = ctx.obj['output'] - - _display_run(client, namespace, run_id, watch, output_format, detail) - - -def _display_run(client: click.Context, - namespace: str, - run_id: str, - watch: bool, - output_format: OutputFormat, - detail: bool = False): - run = client.get_run(run_id).run - - if detail: - data = { - key: - value.isoformat() if isinstance(value, datetime.datetime) else value - for key, value in run.to_dict().items() - if key not in ['pipeline_spec' - ] # useless but too much detailed field - } - click.echo(data) - return - - _print_runs([run], output_format) - if not watch: - return - argo_path = shutil.which('argo') - if not argo_path: - raise RuntimeError( - "argo isn't found in $PATH. It's necessary for watch. " - "Please make sure it's installed and available. " - "Installation instructions be found here - " - "https://github.com/argoproj/argo-workflows/releases") - - argo_workflow_name = None - while True: - time.sleep(1) - run_detail = client.get_run(run_id) - run = run_detail.run - if run_detail.pipeline_runtime and run_detail.pipeline_runtime.workflow_manifest: - manifest = json.loads(run_detail.pipeline_runtime.workflow_manifest) - if manifest['metadata'] and manifest['metadata']['name']: - argo_workflow_name = manifest['metadata']['name'] - break - if run_detail.run.status in ['Succeeded', 'Skipped', 'Failed', 'Error']: - click.echo('Run is finished with status {}.'.format( - run_detail.run.status)) - return - if argo_workflow_name: - subprocess.run( - [argo_path, 'watch', argo_workflow_name, '-n', namespace]) - _print_runs([run], output_format) - - -def _wait_for_run_completion(client: Client, run_id: str, timeout: int, - output_format: OutputFormat): - run_detail = client.wait_for_run_completion(run_id, timeout) - _print_runs([run_detail.run], output_format) - - -def _print_runs(runs: List[kfp_server_api.ApiRun], output_format: OutputFormat): - headers = ['run id', 'name', 'status', 'created at', 'experiment id'] - data = [[ - run.id, run.name, run.status, - run.created_at.isoformat(), - next(rr - for rr in run.resource_references - if rr.key.type == kfp_server_api.ApiResourceType.EXPERIMENT).key.id - ] - for run in runs] - print_output(data, headers, output_format, table_format='grid') diff --git a/sdk/python/kfp/deprecated/compiler/__init__.py b/sdk/python/kfp/deprecated/compiler/__init__.py deleted file mode 100644 index 248088cb6fd..00000000000 --- a/sdk/python/kfp/deprecated/compiler/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated.compiler.compiler import Compiler -from kfp.deprecated.containers._component_builder import build_python_component, build_docker_image, VersionedDependency diff --git a/sdk/python/kfp/deprecated/compiler/_data_passing_rewriter.py b/sdk/python/kfp/deprecated/compiler/_data_passing_rewriter.py deleted file mode 100644 index 48c39f8a720..00000000000 --- a/sdk/python/kfp/deprecated/compiler/_data_passing_rewriter.py +++ /dev/null @@ -1,475 +0,0 @@ -import copy -import json -import re -from typing import List, Optional, Set - - -def fix_big_data_passing(workflow: dict) -> dict: - '''fix_big_data_passing converts a workflow where some artifact data is passed as parameters and converts it to a workflow where this data is passed as artifacts. - Args: - workflow: The workflow to fix - Returns: - The fixed workflow - - Motivation: - DSL compiler only supports passing Argo parameters. - Due to the convoluted nature of the DSL compiler, the artifact consumption and passing has been implemented on top of that using parameter passing. The artifact data is passed as parameters and the consumer template creates an artifact/file out of that data. - Due to the limitations of Kubernetes and Argo this scheme cannot pass data larger than few kilobytes preventing any serious use of artifacts. - - This function rewrites the compiled workflow so that the data consumed as artifact is passed as artifact. - It also prunes the unused parameter outputs. This is important since if a big piece of data is ever returned through a file that is also output as parameter, the execution will fail. - This makes is possible to pass large amounts of data. - - Implementation: - 1. Index the DAGs to understand how data is being passed and which inputs/outputs are connected to each other. - 2. Search for direct data consumers in container/resource templates and some DAG task attributes (e.g. conditions and loops) to find out which inputs are directly consumed as parameters/artifacts. - 3. Propagate the consumption information upstream to all inputs/outputs all the way up to the data producers. - 4. Convert the inputs, outputs and arguments based on how they're consumed downstream. - ''' - workflow = copy.deepcopy(workflow) - - container_templates = [ - template for template in workflow['spec']['templates'] - if 'container' in template - ] - dag_templates = [ - template for template in workflow['spec']['templates'] - if 'dag' in template - ] - resource_templates = [ - template for template in workflow['spec']['templates'] - if 'resource' in template - ] # TODO: Handle these - resource_template_names = set( - template['name'] for template in resource_templates) - - # 1. Index the DAGs to understand how data is being passed and which inputs/outputs are connected to each other. - template_input_to_parent_dag_inputs = { - } # (task_template_name, task_input_name) -> Set[(dag_template_name, dag_input_name)] - template_input_to_parent_task_outputs = { - } # (task_template_name, task_input_name) -> Set[(upstream_template_name, upstream_output_name)] - template_input_to_parent_constant_arguments = { - } #(task_template_name, task_input_name) -> Set[argument_value] # Unused - dag_output_to_parent_template_outputs = { - } # (dag_template_name, output_name) -> Set[(upstream_template_name, upstream_output_name)] - - for template in dag_templates: - dag_template_name = template['name'] - # Indexing task arguments - dag_tasks = template['dag']['tasks'] - task_name_to_template_name = { - task['name']: task['template'] for task in dag_tasks - } - for task in dag_tasks: - task_template_name = task['template'] - parameter_arguments = task.get('arguments', - {}).get('parameters', {}) - for parameter_argument in parameter_arguments: - task_input_name = parameter_argument['name'] - argument_value = parameter_argument['value'] - - argument_placeholder_parts = deconstruct_single_placeholder( - argument_value) - if not argument_placeholder_parts: # Argument is considered to be constant string - template_input_to_parent_constant_arguments.setdefault( - (task_template_name, task_input_name), - set()).add(argument_value) - - placeholder_type = argument_placeholder_parts[0] - if placeholder_type not in ('inputs', 'outputs', 'tasks', - 'steps', 'workflow', 'pod', 'item'): - # Do not fail on Jinja or other double-curly-brace templates - continue - if placeholder_type == 'inputs': - assert argument_placeholder_parts[1] == 'parameters' - dag_input_name = argument_placeholder_parts[2] - template_input_to_parent_dag_inputs.setdefault( - (task_template_name, task_input_name), set()).add( - (dag_template_name, dag_input_name)) - elif placeholder_type == 'tasks': - upstream_task_name = argument_placeholder_parts[1] - assert argument_placeholder_parts[2] == 'outputs' - assert argument_placeholder_parts[3] == 'parameters' - upstream_output_name = argument_placeholder_parts[4] - upstream_template_name = task_name_to_template_name[ - upstream_task_name] - template_input_to_parent_task_outputs.setdefault( - (task_template_name, task_input_name), set()).add( - (upstream_template_name, upstream_output_name)) - elif placeholder_type == 'item' or placeholder_type == 'workflow' or placeholder_type == 'pod': - # Treat loop variables as constant values - # workflow.parameters.* placeholders are not supported, but the DSL compiler does not produce those. - template_input_to_parent_constant_arguments.setdefault( - (task_template_name, task_input_name), - set()).add(argument_value) - else: - raise AssertionError - - dag_input_name = extract_input_parameter_name(argument_value) - if dag_input_name: - template_input_to_parent_dag_inputs.setdefault( - (task_template_name, task_input_name), set()).add( - (dag_template_name, dag_input_name)) - else: - template_input_to_parent_constant_arguments.setdefault( - (task_template_name, task_input_name), - set()).add(argument_value) - - # Indexing DAG outputs (Does DSL compiler produce them?) - for dag_output in task.get('outputs', {}).get('parameters', {}): - dag_output_name = dag_output['name'] - output_value = dag_output['value'] - argument_placeholder_parts = deconstruct_single_placeholder( - output_value) - placeholder_type = argument_placeholder_parts[0] - if not argument_placeholder_parts: # Argument is considered to be constant string - raise RuntimeError( - 'Constant DAG output values are not supported for now.') - if placeholder_type == 'inputs': - raise RuntimeError( - 'Pass-through DAG inputs/outputs are not supported') - elif placeholder_type == 'tasks': - upstream_task_name = argument_placeholder_parts[1] - assert argument_placeholder_parts[2] == 'outputs' - assert argument_placeholder_parts[3] == 'parameters' - upstream_output_name = argument_placeholder_parts[4] - upstream_template_name = task_name_to_template_name[ - upstream_task_name] - dag_output_to_parent_template_outputs.setdefault( - (dag_template_name, dag_output_name), set()).add( - (upstream_template_name, upstream_output_name)) - elif placeholder_type == 'item' or placeholder_type == 'workflow' or placeholder_type == 'pod': - raise RuntimeError( - 'DAG output value "{}" is not supported.'.format( - output_value)) - else: - raise AssertionError( - 'Unexpected placeholder type "{}".'.format( - placeholder_type)) - # Finshed indexing the DAGs - - # 2. Search for direct data consumers in container/resource templates and some DAG task attributes (e.g. conditions and loops) to find out which inputs are directly consumed as parameters/artifacts. - inputs_directly_consumed_as_parameters = set() - inputs_directly_consumed_as_artifacts = set() - outputs_directly_consumed_as_parameters = set() - - # Searching for artifact input consumers in container template inputs - for template in container_templates: - template_name = template['name'] - for input_artifact in template.get('inputs', {}).get('artifacts', {}): - raw_data = input_artifact['raw']['data'] # The structure must exist - # The raw data must be a single input parameter reference. Otherwise (e.g. it's a string or a string with multiple inputs) we should not do the conversion to artifact passing. - input_name = extract_input_parameter_name(raw_data) - if input_name: - inputs_directly_consumed_as_artifacts.add( - (template_name, input_name)) - del input_artifact[ - 'raw'] # Deleting the "default value based" data passing hack so that it's replaced by the "argument based" way of data passing. - input_artifact[ - 'name'] = input_name # The input artifact name should be the same as the original input parameter name - - # Searching for parameter input consumers in DAG templates (.when, .withParam, etc) - for template in dag_templates: - template_name = template['name'] - dag_tasks = template['dag']['tasks'] - task_name_to_template_name = { - task['name']: task['template'] for task in dag_tasks - } - for task in template['dag']['tasks']: - # We do not care about the inputs mentioned in task arguments since we will be free to switch them from parameters to artifacts - # TODO: Handle cases where argument value is a string containing placeholders (not just consisting of a single placeholder) or the input name contains placeholder - task_without_arguments = task.copy() # Shallow copy - task_without_arguments.pop('arguments', None) - placeholders = extract_all_placeholders(task_without_arguments) - for placeholder in placeholders: - parts = placeholder.split('.') - placeholder_type = parts[0] - if placeholder_type not in ('inputs', 'outputs', 'tasks', - 'steps', 'workflow', 'pod', 'item'): - # Do not fail on Jinja or other double-curly-brace templates - continue - if placeholder_type == 'inputs': - if parts[1] == 'parameters': - input_name = parts[2] - inputs_directly_consumed_as_parameters.add( - (template_name, input_name)) - else: - raise AssertionError - elif placeholder_type == 'tasks': - upstream_task_name = parts[1] - assert parts[2] == 'outputs' - assert parts[3] == 'parameters' - upstream_output_name = parts[4] - upstream_template_name = task_name_to_template_name[ - upstream_task_name] - outputs_directly_consumed_as_parameters.add( - (upstream_template_name, upstream_output_name)) - elif placeholder_type == 'workflow' or placeholder_type == 'pod': - pass - elif placeholder_type == 'item': - raise AssertionError( - 'The "{{item}}" placeholder is not expected outside task arguments.' - ) - else: - raise AssertionError( - 'Unexpected placeholder type "{}".'.format( - placeholder_type)) - - # Searching for parameter input consumers in container and resource templates - for template in container_templates + resource_templates: - template_name = template['name'] - placeholders = extract_all_placeholders(template) - for placeholder in placeholders: - parts = placeholder.split('.') - placeholder_type = parts[0] - if placeholder_type not in ('inputs', 'outputs', 'tasks', 'steps', - 'workflow', 'pod', 'item'): - # Do not fail on Jinja or other double-curly-brace templates - continue - - if placeholder_type == 'workflow' or placeholder_type == 'pod': - pass - elif placeholder_type == 'inputs': - if parts[1] == 'parameters': - input_name = parts[2] - inputs_directly_consumed_as_parameters.add( - (template_name, input_name)) - elif parts[1] == 'artifacts': - raise AssertionError( - 'Found unexpected Argo input artifact placeholder in container template: {}' - .format(placeholder)) - else: - raise AssertionError( - 'Found unexpected Argo input placeholder in container template: {}' - .format(placeholder)) - else: - raise AssertionError( - 'Found unexpected Argo placeholder in container template: {}' - .format(placeholder)) - - # Finished indexing data consumers - - # 3. Propagate the consumption information upstream to all inputs/outputs all the way up to the data producers. - inputs_consumed_as_parameters = set() - inputs_consumed_as_artifacts = set() - - outputs_consumed_as_parameters = set() - outputs_consumed_as_artifacts = set() - - def mark_upstream_ios_of_input(template_input, marked_inputs, - marked_outputs): - # Stopping if the input has already been visited to save time and handle recursive calls - if template_input in marked_inputs: - return - marked_inputs.add(template_input) - - upstream_inputs = template_input_to_parent_dag_inputs.get( - template_input, []) - for upstream_input in upstream_inputs: - mark_upstream_ios_of_input(upstream_input, marked_inputs, - marked_outputs) - - upstream_outputs = template_input_to_parent_task_outputs.get( - template_input, []) - for upstream_output in upstream_outputs: - mark_upstream_ios_of_output(upstream_output, marked_inputs, - marked_outputs) - - def mark_upstream_ios_of_output(template_output, marked_inputs, - marked_outputs): - # Stopping if the output has already been visited to save time and handle recursive calls - if template_output in marked_outputs: - return - marked_outputs.add(template_output) - - upstream_outputs = dag_output_to_parent_template_outputs.get( - template_output, []) - for upstream_output in upstream_outputs: - mark_upstream_ios_of_output(upstream_output, marked_inputs, - marked_outputs) - - for input in inputs_directly_consumed_as_parameters: - mark_upstream_ios_of_input(input, inputs_consumed_as_parameters, - outputs_consumed_as_parameters) - for input in inputs_directly_consumed_as_artifacts: - mark_upstream_ios_of_input(input, inputs_consumed_as_artifacts, - outputs_consumed_as_artifacts) - for output in outputs_directly_consumed_as_parameters: - mark_upstream_ios_of_output(output, inputs_consumed_as_parameters, - outputs_consumed_as_parameters) - - # 4. Convert the inputs, outputs and arguments based on how they're consumed downstream. - - # Container templates already output all data as artifacts, so we do not need to convert their outputs to artifacts. (But they also output data as parameters and we need to fix that.) - - # Convert DAG argument passing from parameter to artifacts as needed - for template in dag_templates: - # Converting DAG inputs - inputs = template.get('inputs', {}) - input_parameters = inputs.get('parameters', []) - input_artifacts = inputs.setdefault('artifacts', []) # Should be empty - for input_parameter in input_parameters: - input_name = input_parameter['name'] - if (template['name'], input_name) in inputs_consumed_as_artifacts: - input_artifacts.append({ - 'name': input_name, - }) - - # Converting DAG outputs - outputs = template.get('outputs', {}) - output_parameters = outputs.get('parameters', []) - output_artifacts = outputs.setdefault('artifacts', - []) # Should be empty - for output_parameter in output_parameters: - output_name = output_parameter['name'] - if (template['name'], output_name) in outputs_consumed_as_artifacts: - parameter_reference_placeholder = output_parameter['valueFrom'][ - 'parameter'] - output_artifacts.append({ - 'name': - output_name, - 'from': - parameter_reference_placeholder.replace( - '.parameters.', '.artifacts.'), - }) - - # Converting DAG task arguments - tasks = template.get('dag', {}).get('tasks', []) - for task in tasks: - task_arguments = task.get('arguments', {}) - parameter_arguments = task_arguments.get('parameters', []) - artifact_arguments = task_arguments.setdefault('artifacts', []) - for parameter_argument in parameter_arguments: - input_name = parameter_argument['name'] - if (task['template'], - input_name) in inputs_consumed_as_artifacts: - argument_value = parameter_argument[ - 'value'] # argument parameters always use "value"; output parameters always use "valueFrom" (container/DAG/etc) - argument_placeholder_parts = deconstruct_single_placeholder( - argument_value) - # If the argument is consumed as artifact downstream: - # Pass DAG inputs and DAG/container task outputs as usual; - # Everything else (constant strings, loop variables, resource task outputs) is passed as raw artifact data. Argo properly replaces placeholders in it. - if argument_placeholder_parts and argument_placeholder_parts[ - 0] in [ - 'inputs', 'tasks' - ] and not (argument_placeholder_parts[0] == 'tasks' - and argument_placeholder_parts[1] - in resource_template_names): - artifact_arguments.append({ - 'name': - input_name, - 'from': - argument_value.replace('.parameters.', - '.artifacts.'), - }) - else: - artifact_arguments.append({ - 'name': input_name, - 'raw': { - 'data': argument_value, - }, - }) - - # Remove input parameters unless they're used downstream. This also removes unused container template inputs if any. - for template in container_templates + dag_templates: - inputs = template.get('inputs', {}) - inputs['parameters'] = [ - input_parameter for input_parameter in inputs.get('parameters', []) - if (template['name'], - input_parameter['name']) in inputs_consumed_as_parameters - ] - - # Remove output parameters unless they're used downstream - for template in container_templates + dag_templates: - outputs = template.get('outputs', {}) - outputs['parameters'] = [ - output_parameter - for output_parameter in outputs.get('parameters', []) - if (template['name'], - output_parameter['name']) in outputs_consumed_as_parameters - ] - - # Remove DAG parameter arguments unless they're used downstream - for template in dag_templates: - tasks = template.get('dag', {}).get('tasks', []) - for task in tasks: - task_arguments = task.get('arguments', {}) - task_arguments['parameters'] = [ - parameter_argument - for parameter_argument in task_arguments.get('parameters', []) - if (task['template'], - parameter_argument['name']) in inputs_consumed_as_parameters - ] - - # Fix Workflow parameter arguments that are consumed as artifacts downstream - # - workflow_spec = workflow['spec'] - entrypoint_template_name = workflow_spec['entrypoint'] - workflow_arguments = workflow_spec['arguments'] - parameter_arguments = workflow_arguments.get('parameters', []) - artifact_arguments = workflow_arguments.get('artifacts', - []) # Should be empty - for parameter_argument in parameter_arguments: - input_name = parameter_argument['name'] - if (entrypoint_template_name, - input_name) in inputs_consumed_as_artifacts: - artifact_arguments.append({ - 'name': input_name, - 'raw': { - 'data': '{{workflow.parameters.' + input_name + '}}', - }, - }) - if artifact_arguments: - workflow_arguments['artifacts'] = artifact_arguments - - clean_up_empty_workflow_structures(workflow) - return workflow - - -def clean_up_empty_workflow_structures(workflow: dict): - templates = workflow['spec']['templates'] - for template in templates: - inputs = template.setdefault('inputs', {}) - if not inputs.setdefault('parameters', []): - del inputs['parameters'] - if not inputs.setdefault('artifacts', []): - del inputs['artifacts'] - if not inputs: - del template['inputs'] - outputs = template.setdefault('outputs', {}) - if not outputs.setdefault('parameters', []): - del outputs['parameters'] - if not outputs.setdefault('artifacts', []): - del outputs['artifacts'] - if not outputs: - del template['outputs'] - if 'dag' in template: - for task in template['dag'].get('tasks', []): - arguments = task.setdefault('arguments', {}) - if not arguments.setdefault('parameters', []): - del arguments['parameters'] - if not arguments.setdefault('artifacts', []): - del arguments['artifacts'] - if not arguments: - del task['arguments'] - - -def extract_all_placeholders(template: dict) -> Set[str]: - template_str = json.dumps(template) - placeholders = set(re.findall('{{([-._a-zA-Z0-9]+)}}', template_str)) - return placeholders - - -def extract_input_parameter_name(s: str) -> Optional[str]: - match = re.fullmatch('{{inputs.parameters.([-_a-zA-Z0-9]+)}}', s) - if not match: - return None - (input_name,) = match.groups() - return input_name - - -def deconstruct_single_placeholder(s: str) -> List[str]: - if not re.fullmatch('{{[-._a-zA-Z0-9]+}}', s): - return None - return s.lstrip('{').rstrip('}').split('.') diff --git a/sdk/python/kfp/deprecated/compiler/_data_passing_using_volume.py b/sdk/python/kfp/deprecated/compiler/_data_passing_using_volume.py deleted file mode 100644 index 41224666df2..00000000000 --- a/sdk/python/kfp/deprecated/compiler/_data_passing_using_volume.py +++ /dev/null @@ -1,256 +0,0 @@ -#/bin/env python3 - -import copy -import os -import re -import warnings - - -def rewrite_data_passing_to_use_volumes( - workflow: dict, - volume: dict, - path_prefix: str = 'artifact_data/', -) -> dict: - """Converts Argo workflow that passes data using artifacts to workflow that - passes data using a single multi-write volume. - - Implementation: Changes file passing system from passing Argo artifacts to passing parameters holding volumeMount subPaths. - - Limitations: - * All artifact file names must be the same (e.g. "data"). E.g. "/tmp/outputs/my_out_1/data". - Otherwise consumer component that can receive data from different producers won't be able to specify input file path since files from different upstreams will have different names. - * Passing constant arguments to artifact inputs is not supported for now. Only references can be passed. - - Args: - workflow: Workflow (dict) that needs to be converted. - volume: Kubernetes spec (dict) of a volume that should be used for data passing. The volume should support multi-write (READ_WRITE_MANY) if the workflow has any parallel steps. Example: {'persistentVolumeClaim': {'claimName': 'my-pvc'}} - - Returns: - Converted workflow (dict). - """ - - # Limitation: All artifact file names must be the same (e.g. "data"). E.g. "/tmp/outputs/my_out_1/data". - # Otherwise consumer component that can receive data from different producers won't be able to specify input file path since files from different upstreams will have different names. - # Maybe this limitation can be lifted, but it is not trivial. Maybe with DSL compile it's easier since the graph it creates is usually a tree. But recursion creates a real graph. - # Limitation: Passing constant values to artifact inputs is not supported for now. - - # Idea: Passing volumeMount subPaths instead of artifcats - - workflow = copy.deepcopy(workflow) - templates = workflow['spec']['templates'] - - container_templates = [ - template for template in templates if 'container' in template - ] - dag_templates = [template for template in templates if 'dag' in template] - steps_templates = [ - template for template in templates if 'steps' in template - ] - - execution_data_dir = path_prefix + '{{workflow.uid}}_{{pod.name}}/' - - data_volume_name = 'data-storage' - volume['name'] = data_volume_name - - subpath_parameter_name_suffix = '-subpath' - - def convert_artifact_reference_to_parameter_reference( - reference: str) -> str: - parameter_reference = re.sub( - r'{{([^}]+)\.artifacts\.([^}]+)}}', - r'{{\1.parameters.\2' + subpath_parameter_name_suffix + - '}}', # re.escape(subpath_parameter_name_suffix) escapes too much. - reference, - ) - return parameter_reference - - # Adding the data storage volume to the workflow - workflow['spec'].setdefault('volumes', []).append(volume) - - all_artifact_file_names = set( - ) # All artifacts should have same file name (usually, "data"). This variable holds all different artifact names for verification. - - # Rewriting container templates - for template in templates: - if 'container' not in template and 'script' not in template: - continue - container_spec = template['container'] or template['steps'] - # Inputs - input_artifacts = template.get('inputs', {}).get('artifacts', []) - if input_artifacts: - input_parameters = template.setdefault('inputs', {}).setdefault( - 'parameters', []) - volume_mounts = container_spec.setdefault('volumeMounts', []) - for input_artifact in input_artifacts: - subpath_parameter_name = input_artifact[ - 'name'] + subpath_parameter_name_suffix # TODO: Maybe handle clashing names. - artifact_file_name = os.path.basename(input_artifact['path']) - all_artifact_file_names.add(artifact_file_name) - artifact_dir = os.path.dirname(input_artifact['path']) - volume_mounts.append({ - 'mountPath': - artifact_dir, - 'name': - data_volume_name, - 'subPath': - '{{inputs.parameters.' + subpath_parameter_name + '}}', - 'readOnly': - True, - }) - input_parameters.append({ - 'name': subpath_parameter_name, - }) - template.get('inputs', {}).pop('artifacts', None) - - # Outputs - output_artifacts = template.get('outputs', {}).get('artifacts', []) - if output_artifacts: - output_parameters = template.setdefault('outputs', {}).setdefault( - 'parameters', []) - del template.get('outputs', {})['artifacts'] - volume_mounts = container_spec.setdefault('volumeMounts', []) - for output_artifact in output_artifacts: - output_name = output_artifact['name'] - subpath_parameter_name = output_name + subpath_parameter_name_suffix # TODO: Maybe handle clashing names. - artifact_file_name = os.path.basename(output_artifact['path']) - all_artifact_file_names.add(artifact_file_name) - artifact_dir = os.path.dirname(output_artifact['path']) - output_subpath = execution_data_dir + output_name - volume_mounts.append({ - 'mountPath': artifact_dir, - 'name': data_volume_name, - 'subPath': - output_subpath, # TODO: Switch to subPathExpr when it's out of beta: https://kubernetes.io/docs/concepts/storage/volumes/#using-subpath-with-expanded-environment-variables - }) - output_parameters.append({ - 'name': subpath_parameter_name, - 'value': output_subpath, # Requires Argo 2.3.0+ - }) - whitelist = ['mlpipeline-ui-metadata', 'mlpipeline-metrics'] - output_artifacts = [ - artifact for artifact in output_artifacts - if artifact['name'] in whitelist - ] - if not output_artifacts: - template.get('outputs', {}).pop('artifacts', None) - else: - template.get('outputs', - {}).update({'artifacts': output_artifacts}) - - # Rewrite DAG templates - for template in templates: - if 'dag' not in template and 'steps' not in template: - continue - - # Inputs - input_artifacts = template.get('inputs', {}).get('artifacts', []) - if input_artifacts: - input_parameters = template.setdefault('inputs', {}).setdefault( - 'parameters', []) - volume_mounts = container_spec.setdefault('volumeMounts', []) - for input_artifact in input_artifacts: - subpath_parameter_name = input_artifact[ - 'name'] + subpath_parameter_name_suffix # TODO: Maybe handle clashing names. - input_parameters.append({ - 'name': subpath_parameter_name, - }) - template.get('inputs', {}).pop('artifacts', None) - - # Outputs - output_artifacts = template.get('outputs', {}).get('artifacts', []) - if output_artifacts: - output_parameters = template.setdefault('outputs', {}).setdefault( - 'parameters', []) - volume_mounts = container_spec.setdefault('volumeMounts', []) - for output_artifact in output_artifacts: - output_name = output_artifact['name'] - subpath_parameter_name = output_name + subpath_parameter_name_suffix # TODO: Maybe handle clashing names. - output_parameters.append({ - 'name': subpath_parameter_name, - 'valueFrom': { - 'parameter': - convert_artifact_reference_to_parameter_reference( - output_artifact['from']) - }, - }) - template.get('outputs', {}).pop('artifacts', None) - - # Arguments - for task in template.get('dag', {}).get('tasks', []) + [ - steps for group in template.get('steps', []) for steps in group - ]: - argument_artifacts = task.get('arguments', {}).get('artifacts', []) - if argument_artifacts: - argument_parameters = task.setdefault('arguments', - {}).setdefault( - 'parameters', []) - for argument_artifact in argument_artifacts: - if 'from' not in argument_artifact: - raise NotImplementedError( - 'Volume-based data passing rewriter does not support constant artifact arguments at this moment. Only references can be passed.' - ) - subpath_parameter_name = argument_artifact[ - 'name'] + subpath_parameter_name_suffix # TODO: Maybe handle clashing names. - argument_parameters.append({ - 'name': - subpath_parameter_name, - 'value': - convert_artifact_reference_to_parameter_reference( - argument_artifact['from']), - }) - task.get('arguments', {}).pop('artifacts', None) - - # There should not be any artifact references in any other part of DAG template (only parameter references) - - # Check that all artifacts have the same file names - if len(all_artifact_file_names) > 1: - warnings.warn( - 'Detected different artifact file names: [{}]. The workflow can fail at runtime. Please use the same file name (e.g. "data") for all artifacts.' - .format(', '.join(all_artifact_file_names))) - - # Fail if workflow has argument artifacts - workflow_argument_artifacts = workflow['spec'].get('arguments', - {}).get('artifacts', []) - if workflow_argument_artifacts: - raise NotImplementedError( - 'Volume-based data passing rewriter does not support constant artifact arguments at this moment. Only references can be passed.' - ) - - return workflow - - -if __name__ == '__main__': - """Converts Argo workflow that passes data using artifacts to workflow that - passes data using a single multi-write volume.""" - - import argparse - import io - import yaml - - parser = argparse.ArgumentParser( - prog='data_passing_using_volume', - epilog='''Example: data_passing_using_volume.py --input argo/examples/artifact-passing.yaml --output argo/examples/artifact-passing-volumes.yaml --volume "persistentVolumeClaim: {claimName: my-pvc}"''' - ) - parser.add_argument( - "--input", - type=argparse.FileType('r'), - required=True, - help='Path to workflow that needs to be converted.') - parser.add_argument( - "--output", - type=argparse.FileType('w'), - required=True, - help='Path where to write converted workflow.') - parser.add_argument( - "--volume", - type=str, - required=True, - help='''Kubernetes spec (YAML) of a volume that should be used for data passing. The volume should support multi-write (READ_WRITE_MANY) if the workflow has any parallel steps. Example: "persistentVolumeClaim: {claimName: my-pvc}".''' - ) - args = parser.parse_args() - - input_workflow = yaml.safe_load(args.input) - volume = yaml.safe_load(io.StringIO(args.volume)) - output_workflow = rewrite_data_passing_to_use_volumes( - input_workflow, volume) - yaml.dump(output_workflow, args.output) diff --git a/sdk/python/kfp/deprecated/compiler/_default_transformers.py b/sdk/python/kfp/deprecated/compiler/_default_transformers.py deleted file mode 100644 index 18d12b6755c..00000000000 --- a/sdk/python/kfp/deprecated/compiler/_default_transformers.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import warnings -from kubernetes import client as k8s_client -from typing import Callable, Dict, Optional -from kfp.deprecated.dsl._container_op import BaseOp, ContainerOp - - -def add_pod_env(op: BaseOp) -> BaseOp: - """Adds environment info if the Pod has the label `add-pod-env = true`. - """ - if isinstance( - op, ContainerOp - ) and op.pod_labels and 'add-pod-env' in op.pod_labels and op.pod_labels[ - 'add-pod-env'] == 'true': - return add_kfp_pod_env(op) - - -def add_kfp_pod_env(op: BaseOp) -> BaseOp: - """Adds KFP pod environment info to the specified ContainerOp.""" - if not isinstance(op, ContainerOp): - warnings.warn( - 'Trying to add default KFP environment variables to an Op that is ' - 'not a ContainerOp. Ignoring request.') - return op - - op.container.add_env_variable( - k8s_client.V1EnvVar( - name='KFP_POD_NAME', - value_from=k8s_client.V1EnvVarSource( - field_ref=k8s_client.V1ObjectFieldSelector( - field_path='metadata.name'))) - ).add_env_variable( - k8s_client.V1EnvVar( - name='KFP_POD_UID', - value_from=k8s_client.V1EnvVarSource( - field_ref=k8s_client.V1ObjectFieldSelector( - field_path='metadata.uid'))) - ).add_env_variable( - k8s_client.V1EnvVar( - name='KFP_NAMESPACE', - value_from=k8s_client.V1EnvVarSource( - field_ref=k8s_client.V1ObjectFieldSelector( - field_path='metadata.namespace'))) - ).add_env_variable( - k8s_client.V1EnvVar( - name='WORKFLOW_ID', - value_from=k8s_client.V1EnvVarSource( - field_ref=k8s_client.V1ObjectFieldSelector( - field_path="metadata.labels['workflows.argoproj.io/workflow']" - ))) - ).add_env_variable( - k8s_client.V1EnvVar( - name='KFP_RUN_ID', - value_from=k8s_client.V1EnvVarSource( - field_ref=k8s_client.V1ObjectFieldSelector( - field_path="metadata.labels['pipeline/runid']"))) - ).add_env_variable( - k8s_client.V1EnvVar( - name='ENABLE_CACHING', - value_from=k8s_client.V1EnvVarSource( - field_ref=k8s_client - .V1ObjectFieldSelector( - field_path="metadata.labels['pipelines.kubeflow.org/enable_caching']" - )))) - return op - - -def add_pod_labels(labels: Optional[Dict] = None) -> Callable: - """Adds provided pod labels to each pod.""" - - def _add_pod_labels(task): - for k, v in labels.items(): - # Only append but not update. - # This is needed to bypass TFX pipelines/components. - if k not in task.pod_labels: - task.add_pod_label(k, v) - return task - - return _add_pod_labels diff --git a/sdk/python/kfp/deprecated/compiler/_k8s_helper.py b/sdk/python/kfp/deprecated/compiler/_k8s_helper.py deleted file mode 100644 index a2f77a59382..00000000000 --- a/sdk/python/kfp/deprecated/compiler/_k8s_helper.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re - -from kfp.deprecated import dsl - - -def sanitize_k8s_name(name, allow_capital_underscore=False): - """From _make_kubernetes_name sanitize_k8s_name cleans and converts the - names in the workflow. - - Args: - name: original name, - allow_capital_underscore: whether to allow capital letter and underscore - in this name. - - Returns: - sanitized name. - """ - if allow_capital_underscore: - return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', - name)).lstrip('-').rstrip('-') - else: - return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', - name.lower())).lstrip('-').rstrip('-') - - -def convert_k8s_obj_to_json(k8s_obj): - """Builds a JSON K8s object. - - If obj is None, return None. - If obj is str, int, long, float, bool, return directly. - If obj is datetime.datetime, datetime.date - convert to string in iso8601 format. - If obj is list, sanitize each element in the list. - If obj is dict, return the dict. - If obj is swagger model, return the properties dict. - - Args: - obj: The data to serialize. - Returns: The serialized form of data. - """ - - from six import text_type, integer_types, iteritems - PRIMITIVE_TYPES = (float, bool, bytes, text_type) + integer_types - from datetime import date, datetime - if k8s_obj is None: - return None - elif isinstance(k8s_obj, PRIMITIVE_TYPES): - return k8s_obj - elif isinstance(k8s_obj, list): - return [convert_k8s_obj_to_json(sub_obj) for sub_obj in k8s_obj] - elif isinstance(k8s_obj, tuple): - return tuple(convert_k8s_obj_to_json(sub_obj) for sub_obj in k8s_obj) - elif isinstance(k8s_obj, (datetime, date)): - return k8s_obj.isoformat() - elif isinstance(k8s_obj, dsl.PipelineParam): - if isinstance(k8s_obj.value, str): - return k8s_obj.value - return '{{inputs.parameters.%s}}' % k8s_obj.full_name - - if isinstance(k8s_obj, dict): - obj_dict = k8s_obj - else: - # Convert model obj to dict except - # attributes `swagger_types`, `attribute_map` - # and attributes which value is not None. - # Convert attribute name to json key in - # model definition for request. - obj_dict = { - k8s_obj.attribute_map[attr]: getattr(k8s_obj, attr) - for attr in k8s_obj.attribute_map - if getattr(k8s_obj, attr) is not None - } - - return { - key: convert_k8s_obj_to_json(val) for key, val in iteritems(obj_dict) - } diff --git a/sdk/python/kfp/deprecated/compiler/_op_to_template.py b/sdk/python/kfp/deprecated/compiler/_op_to_template.py deleted file mode 100644 index ff64338c9e7..00000000000 --- a/sdk/python/kfp/deprecated/compiler/_op_to_template.py +++ /dev/null @@ -1,364 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import re -import warnings -import yaml -import copy -from collections import OrderedDict -from typing import Any, Dict, List, TypeVar - -from kfp.deprecated.compiler._k8s_helper import convert_k8s_obj_to_json -from kfp.deprecated import dsl -from kfp.deprecated.dsl._container_op import BaseOp - -# generics -T = TypeVar('T') - - -def _process_obj(obj: Any, map_to_tmpl_var: dict): - """Recursively sanitize and replace any PipelineParam (instances and - serialized strings) in the object with the corresponding template variables - (i.e. '{{inputs.parameters.}}'). - - Args: - obj: any obj that may have PipelineParam - map_to_tmpl_var: a dict that maps an unsanitized pipeline - params signature into a template var - """ - # serialized str might be unsanitized - if isinstance(obj, str): - # get signature - param_tuples = dsl.match_serialized_pipelineparam(obj) - if not param_tuples: - return obj - # replace all unsanitized signature with template var - for param_tuple in param_tuples: - obj = re.sub(param_tuple.pattern, - map_to_tmpl_var[param_tuple.pattern], obj) - - # list - if isinstance(obj, list): - return [_process_obj(item, map_to_tmpl_var) for item in obj] - - # tuple - if isinstance(obj, tuple): - return tuple((_process_obj(item, map_to_tmpl_var) for item in obj)) - - # dict - if isinstance(obj, dict): - return { - _process_obj(key, map_to_tmpl_var): - _process_obj(value, map_to_tmpl_var) for key, value in obj.items() - } - - # pipelineparam - if isinstance(obj, dsl.PipelineParam): - # if not found in unsanitized map, then likely to be sanitized - return map_to_tmpl_var.get( - str(obj), '{{inputs.parameters.%s}}' % obj.full_name) - - # k8s objects (generated from swaggercodegen) - if hasattr(obj, 'attribute_map') and isinstance(obj.attribute_map, dict): - # process everything inside recursively - for key in obj.attribute_map.keys(): - setattr(obj, key, _process_obj(getattr(obj, key), map_to_tmpl_var)) - # return json representation of the k8s obj - return convert_k8s_obj_to_json(obj) - - # do nothing - return obj - - -def _process_base_ops(op: BaseOp): - """Recursively go through the attrs listed in `attrs_with_pipelineparams` - and sanitize and replace pipeline params with template var string. - - Returns a processed `BaseOp`. - - NOTE this is an in-place update to `BaseOp`'s attributes (i.e. the ones - specified in `attrs_with_pipelineparams`, all `PipelineParam` are replaced - with the corresponding template variable strings). - - Args: - op {BaseOp}: class that inherits from BaseOp - - Returns: - BaseOp - """ - - # map param's (unsanitized pattern or serialized str pattern) -> input param var str - map_to_tmpl_var = {(param.pattern or str(param)): - '{{inputs.parameters.%s}}' % param.full_name - for param in op.inputs} - - # process all attr with pipelineParams except inputs and outputs parameters - for key in op.attrs_with_pipelineparams: - setattr(op, key, _process_obj(getattr(op, key), map_to_tmpl_var)) - - return op - - -def _parameters_to_json(params: List[dsl.PipelineParam]): - """Converts a list of PipelineParam into an argo `parameter` JSON obj.""" - _to_json = (lambda param: dict(name=param.full_name, value=param.value) - if param.value else dict(name=param.full_name)) - params = [_to_json(param) for param in params] - # Sort to make the results deterministic. - params.sort(key=lambda x: x['name']) - return params - - -def _inputs_to_json( - inputs_params: List[dsl.PipelineParam], - input_artifact_paths: Dict[str, str] = None, - artifact_arguments: Dict[str, str] = None, -) -> Dict[str, Dict]: - """Converts a list of PipelineParam into an argo `inputs` JSON obj.""" - parameters = _parameters_to_json(inputs_params) - - # Building the input artifacts section - artifacts = [] - for name, path in (input_artifact_paths or {}).items(): - artifact = {'name': name, 'path': path} - if name in artifact_arguments: # The arguments should be compiled as DAG task arguments, not template's default values, but in the current DSL-compiler implementation it's too hard to make that work when passing artifact references. - artifact['raw'] = {'data': str(artifact_arguments[name])} - artifacts.append(artifact) - artifacts.sort( - key=lambda x: x['name']) #Stabilizing the input artifact ordering - - inputs_dict = {} - if parameters: - inputs_dict['parameters'] = parameters - if artifacts: - inputs_dict['artifacts'] = artifacts - return inputs_dict - - -def _outputs_to_json(op: BaseOp, outputs: Dict[str, dsl.PipelineParam], - param_outputs: Dict[str, - str], output_artifacts: List[dict]): - """Creates an argo `outputs` JSON obj.""" - if isinstance(op, dsl.ResourceOp): - value_from_key = "jsonPath" - else: - value_from_key = "path" - output_parameters = [] - for param in set(outputs.values()): # set() dedupes output references - output_parameters.append({ - 'name': param.full_name, - 'valueFrom': { - value_from_key: param_outputs[param.name] - } - }) - output_parameters.sort(key=lambda x: x['name']) - ret = {} - if output_parameters: - ret['parameters'] = output_parameters - if output_artifacts: - ret['artifacts'] = output_artifacts - - return ret - - -# TODO: generate argo python classes from swagger and use convert_k8s_obj_to_json?? -def _op_to_template(op: BaseOp): - """Generate template given an operator inherited from BaseOp.""" - - # Display name - if op.display_name: - op.add_pod_annotation('pipelines.kubeflow.org/task_display_name', - op.display_name) - - # Caching option - op.add_pod_label('pipelines.kubeflow.org/enable_caching', - str(op.enable_caching).lower()) - - # NOTE in-place update to BaseOp - # replace all PipelineParams with template var strings - processed_op = _process_base_ops(op) - - if isinstance(op, dsl.ContainerOp): - output_artifact_paths = OrderedDict(op.output_artifact_paths) - # This should have been as easy as output_artifact_paths.update(op.file_outputs), but the _outputs_to_json function changes the output names and we must do the same here, so that the names are the same - output_artifact_paths.update( - sorted(((param.full_name, processed_op.file_outputs[param.name]) - for param in processed_op.outputs.values()), - key=lambda x: x[0])) - - output_artifacts = [{ - 'name': name, - 'path': path - } for name, path in output_artifact_paths.items()] - - # workflow template - template = { - 'name': processed_op.name, - 'container': convert_k8s_obj_to_json(processed_op.container) - } - elif isinstance(op, dsl.ResourceOp): - # no output artifacts - output_artifacts = [] - - # workflow template - processed_op.resource["manifest"] = yaml.dump( - convert_k8s_obj_to_json(processed_op.k8s_resource), - default_flow_style=False) - template = { - 'name': processed_op.name, - 'resource': convert_k8s_obj_to_json(processed_op.resource) - } - - # inputs - input_artifact_paths = processed_op.input_artifact_paths if isinstance( - processed_op, dsl.ContainerOp) else None - artifact_arguments = processed_op.artifact_arguments if isinstance( - processed_op, dsl.ContainerOp) else None - inputs = _inputs_to_json(processed_op.inputs, input_artifact_paths, - artifact_arguments) - if inputs: - template['inputs'] = inputs - - # outputs - if isinstance(op, dsl.ContainerOp): - param_outputs = processed_op.file_outputs - elif isinstance(op, dsl.ResourceOp): - param_outputs = processed_op.attribute_outputs - outputs_dict = _outputs_to_json(op, processed_op.outputs, param_outputs, - output_artifacts) - if outputs_dict: - template['outputs'] = outputs_dict - - # pod spec used for runtime container settings - podSpecPatch = {} - - # node selector - if processed_op.node_selector: - copy_node_selector = copy.deepcopy(processed_op.node_selector) - for key, value in processed_op.node_selector.items(): - if re.match('^{{inputs.parameters.*}}$', key) or re.match( - '^{{inputs.parameters.*}}$', value): - if not 'nodeSelector' in podSpecPatch: - podSpecPatch['nodeSelector'] = {} - podSpecPatch["nodeSelector"][key] = value - del copy_node_selector[ - key] # avoid to change the dict when iterating it - if processed_op.node_selector: - template['nodeSelector'] = copy_node_selector - - # tolerations - if processed_op.tolerations: - template['tolerations'] = processed_op.tolerations - - # affinity - if processed_op.affinity: - template['affinity'] = convert_k8s_obj_to_json(processed_op.affinity) - - # metadata - if processed_op.pod_annotations or processed_op.pod_labels: - template['metadata'] = {} - if processed_op.pod_annotations: - template['metadata']['annotations'] = processed_op.pod_annotations - if processed_op.pod_labels: - template['metadata']['labels'] = processed_op.pod_labels - # retries - if processed_op.num_retries or processed_op.retry_policy: - template['retryStrategy'] = {} - if processed_op.num_retries: - template['retryStrategy']['limit'] = processed_op.num_retries - if processed_op.retry_policy: - template['retryStrategy']['retryPolicy'] = processed_op.retry_policy - if not processed_op.num_retries: - warnings.warn('retry_policy is set, but num_retries is not') - backoff_dict = {} - if processed_op.backoff_duration: - backoff_dict['duration'] = processed_op.backoff_duration - if processed_op.backoff_factor: - backoff_dict['factor'] = processed_op.backoff_factor - if processed_op.backoff_max_duration: - backoff_dict['maxDuration'] = processed_op.backoff_max_duration - if backoff_dict: - template['retryStrategy']['backoff'] = backoff_dict - - # timeout - if processed_op.timeout: - template['activeDeadlineSeconds'] = processed_op.timeout - - # initContainers - if processed_op.init_containers: - template['initContainers'] = processed_op.init_containers - - # sidecars - if processed_op.sidecars: - template['sidecars'] = processed_op.sidecars - - # volumes - if processed_op.volumes: - template['volumes'] = [ - convert_k8s_obj_to_json(volume) for volume in processed_op.volumes - ] - template['volumes'].sort(key=lambda x: x['name']) - - # Runtime resource requests - if isinstance(op, dsl.ContainerOp) and ('resources' in op.container.keys()): - for setting, val in op.container['resources'].items(): - for resource, param in val.items(): - if (resource in ['cpu', 'memory', 'amd.com/gpu', 'nvidia.com/gpu'] or re.match('^{{inputs.parameters.*}}$', resource))\ - and re.match('^{{inputs.parameters.*}}$', str(param)): - if not 'containers' in podSpecPatch: - podSpecPatch['containers'] = [{ - 'name': 'main', - 'resources': {} - }] - if setting not in podSpecPatch['containers'][0][ - 'resources']: - podSpecPatch['containers'][0]['resources'][setting] = { - resource: param - } - else: - podSpecPatch['containers'][0]['resources'][setting][ - resource] = param - del template['container']['resources'][setting][resource] - if not template['container']['resources'][setting]: - del template['container']['resources'][setting] - - if isinstance(op, dsl.ContainerOp) and op._metadata and not op.is_v2: - template.setdefault('metadata', {}).setdefault( - 'annotations', - {})['pipelines.kubeflow.org/component_spec'] = json.dumps( - op._metadata.to_dict(), sort_keys=True) - - if hasattr(op, '_component_ref'): - template.setdefault('metadata', {}).setdefault( - 'annotations', - {})['pipelines.kubeflow.org/component_ref'] = json.dumps( - op._component_ref.to_dict(), sort_keys=True) - - if hasattr(op, '_parameter_arguments') and op._parameter_arguments: - template.setdefault('metadata', {}).setdefault( - 'annotations', - {})['pipelines.kubeflow.org/arguments.parameters'] = json.dumps( - op._parameter_arguments, sort_keys=True) - - if isinstance(op, dsl.ContainerOp) and op.execution_options: - if op.execution_options.caching_strategy.max_cache_staleness: - template.setdefault('metadata', {}).setdefault( - 'annotations', - {})['pipelines.kubeflow.org/max_cache_staleness'] = str( - op.execution_options.caching_strategy.max_cache_staleness) - - if podSpecPatch: - template['podSpecPatch'] = json.dumps(podSpecPatch) - return template diff --git a/sdk/python/kfp/deprecated/compiler/compiler.py b/sdk/python/kfp/deprecated/compiler/compiler.py deleted file mode 100644 index 55d35a8d8f8..00000000000 --- a/sdk/python/kfp/deprecated/compiler/compiler.py +++ /dev/null @@ -1,1271 +0,0 @@ -# Copyright 2018-2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import datetime -import json -from collections import defaultdict, OrderedDict -import inspect -import re -import tarfile -import uuid -import warnings -import zipfile -from typing import Callable, Set, List, Text, Dict, Tuple, Any, Union, Optional - -import kfp.deprecated as kfp -from kfp.deprecated.dsl import _for_loop -from kfp.deprecated.compiler import v2_compat - -from kfp.deprecated import dsl -from kfp.deprecated.compiler._k8s_helper import convert_k8s_obj_to_json, sanitize_k8s_name -from kfp.deprecated.compiler._op_to_template import _op_to_template, _process_obj -from kfp.deprecated.compiler._default_transformers import add_pod_env, add_pod_labels - -from kfp.deprecated.components.structures import InputSpec -from kfp.deprecated.components._yaml_utils import dump_yaml -from kfp.deprecated.dsl._metadata import _extract_pipeline_metadata -from kfp.deprecated.dsl._ops_group import OpsGroup -from kfp.deprecated.dsl._pipeline_param import extract_pipelineparams_from_any, PipelineParam - -_SDK_VERSION_LABEL = 'pipelines.kubeflow.org/kfp_sdk_version' -_SDK_ENV_LABEL = 'pipelines.kubeflow.org/pipeline-sdk-type' -_SDK_ENV_DEFAULT = 'kfp' - - -class Compiler(object): - """DSL Compiler that compiles pipeline functions into workflow yaml. - - Example: - How to use the compiler to construct workflow yaml:: - - @dsl.pipeline( - name='name', - description='description' - ) - def my_pipeline(a: int = 1, b: str = "default value"): - ... - - Compiler().compile(my_pipeline, 'path/to/workflow.yaml') - """ - - def __init__(self, - mode: dsl.PipelineExecutionMode = kfp.dsl.PipelineExecutionMode - .V1_LEGACY, - launcher_image: Optional[str] = None): - """Creates a KFP compiler for compiling pipeline functions for - execution. - - Args: - mode: The pipeline execution mode to use, defaults to kfp.dsl.PipelineExecutionMode.V1_LEGACY. - launcher_image: Configurable image for KFP launcher to use. Only applies - when `mode == dsl.PipelineExecutionMode.V2_COMPATIBLE`. Should only be - needed for tests or custom deployments right now. - """ - if mode == dsl.PipelineExecutionMode.V2_ENGINE: - raise ValueError('V2_ENGINE execution mode is not supported yet.') - - if mode == dsl.PipelineExecutionMode.V2_COMPATIBLE: - raise ValueError('V2_COMPATIBLE mode has been deprecated in KFP SDK' - ' 2.0. To use V2_COMPATIBLE mode, install KFP SDK' - ' 1.8.*.') - - self._mode = mode - self._launcher_image = launcher_image - self._pipeline_name_param: Optional[dsl.PipelineParam] = None - self._pipeline_root_param: Optional[dsl.PipelineParam] = None - - def _get_groups_for_ops(self, root_group): - """Helper function to get belonging groups for each op. - - Each pipeline has a root group. Each group has a list of operators (leaf) and groups. - This function traverse the tree and get all ancestor groups for all operators. - - Returns: - A dict. Key is the operator's name. Value is a list of ancestor groups including the - op itself. The list of a given operator is sorted in a way that the farthest - group is the first and operator itself is the last. - """ - - def _get_op_groups_helper(current_groups, ops_to_groups): - root_group = current_groups[-1] - for g in root_group.groups: - # Add recursive opsgroup in the ops_to_groups - # such that the i/o dependency can be propagated to the ancester opsgroups - if g.recursive_ref: - ops_to_groups[g.name] = [x.name for x in current_groups - ] + [g.name] - continue - current_groups.append(g) - _get_op_groups_helper(current_groups, ops_to_groups) - del current_groups[-1] - for op in root_group.ops: - ops_to_groups[op.name] = [x.name for x in current_groups - ] + [op.name] - - ops_to_groups = {} - current_groups = [root_group] - _get_op_groups_helper(current_groups, ops_to_groups) - return ops_to_groups - - #TODO: combine with the _get_groups_for_ops - def _get_groups_for_opsgroups(self, root_group): - """Helper function to get belonging groups for each opsgroup. - - Each pipeline has a root group. Each group has a list of operators (leaf) and groups. - This function traverse the tree and get all ancestor groups for all opsgroups. - - Returns: - A dict. Key is the opsgroup's name. Value is a list of ancestor groups including the - opsgroup itself. The list of a given opsgroup is sorted in a way that the farthest - group is the first and opsgroup itself is the last. - """ - - def _get_opsgroup_groups_helper(current_groups, opsgroups_to_groups): - root_group = current_groups[-1] - for g in root_group.groups: - # Add recursive opsgroup in the ops_to_groups - # such that the i/o dependency can be propagated to the ancester opsgroups - if g.recursive_ref: - continue - opsgroups_to_groups[g.name] = [x.name for x in current_groups - ] + [g.name] - current_groups.append(g) - _get_opsgroup_groups_helper(current_groups, opsgroups_to_groups) - del current_groups[-1] - - opsgroups_to_groups = {} - current_groups = [root_group] - _get_opsgroup_groups_helper(current_groups, opsgroups_to_groups) - return opsgroups_to_groups - - def _get_groups(self, root_group): - """Helper function to get all groups (not including ops) in a - pipeline.""" - - def _get_groups_helper(group): - groups = {group.name: group} - for g in group.groups: - # Skip the recursive opsgroup because no templates - # need to be generated for the recursive opsgroups. - if not g.recursive_ref: - groups.update(_get_groups_helper(g)) - return groups - - return _get_groups_helper(root_group) - - def _get_uncommon_ancestors(self, op_groups, opsgroup_groups, op1, op2): - """Helper function to get unique ancestors between two ops. - - For example, op1's ancestor groups are [root, G1, G2, G3, op1], - op2's ancestor groups are [root, G1, G4, op2], then it returns a - tuple ([G2, G3, op1], [G4, op2]). - """ - #TODO: extract a function for the following two code module - if op1.name in op_groups: - op1_groups = op_groups[op1.name] - elif op1.name in opsgroup_groups: - op1_groups = opsgroup_groups[op1.name] - else: - raise ValueError(op1.name + ' does not exist.') - - if op2.name in op_groups: - op2_groups = op_groups[op2.name] - elif op2.name in opsgroup_groups: - op2_groups = opsgroup_groups[op2.name] - else: - raise ValueError(op2.name + ' does not exist.') - - both_groups = [op1_groups, op2_groups] - common_groups_len = sum( - 1 for x in zip(*both_groups) if x == (x[0],) * len(x)) - group1 = op1_groups[common_groups_len:] - group2 = op2_groups[common_groups_len:] - return (group1, group2) - - def _get_condition_params_for_ops(self, root_group): - """Get parameters referenced in conditions of ops.""" - conditions = defaultdict(set) - - def _get_condition_params_for_ops_helper(group, - current_conditions_params): - new_current_conditions_params = current_conditions_params - if group.type == 'condition': - new_current_conditions_params = list(current_conditions_params) - if isinstance(group.condition.operand1, dsl.PipelineParam): - new_current_conditions_params.append( - group.condition.operand1) - if isinstance(group.condition.operand2, dsl.PipelineParam): - new_current_conditions_params.append( - group.condition.operand2) - for op in group.ops: - for param in new_current_conditions_params: - conditions[op.name].add(param) - for g in group.groups: - # If the subgroup is a recursive opsgroup, propagate the pipelineparams - # in the condition expression, similar to the ops. - if g.recursive_ref: - for param in new_current_conditions_params: - conditions[g.name].add(param) - else: - _get_condition_params_for_ops_helper( - g, new_current_conditions_params) - - _get_condition_params_for_ops_helper(root_group, []) - return conditions - - def _get_next_group_or_op(cls, to_visit: List, already_visited: Set): - """Get next group or op to visit.""" - if len(to_visit) == 0: - return None - next = to_visit.pop(0) - while next in already_visited: - next = to_visit.pop(0) - already_visited.add(next) - return next - - def _get_for_loop_ops(self, new_root) -> Dict[Text, dsl.ParallelFor]: - to_visit = self._get_all_subgroups_and_ops(new_root) - op_name_to_op = {} - already_visited = set() - - while len(to_visit): - next_op = self._get_next_group_or_op(to_visit, already_visited) - if next_op is None: - break - to_visit.extend(self._get_all_subgroups_and_ops(next_op)) - if isinstance(next_op, dsl.ParallelFor): - op_name_to_op[next_op.name] = next_op - - return op_name_to_op - - def _get_all_subgroups_and_ops(self, op): - """Get all ops and groups contained within this group.""" - subgroups = [] - if hasattr(op, 'ops'): - subgroups.extend(op.ops) - if hasattr(op, 'groups'): - subgroups.extend(op.groups) - return subgroups - - def _get_inputs_outputs( - self, - pipeline, - root_group, - op_groups, - opsgroup_groups, - condition_params, - op_name_to_for_loop_op: Dict[Text, dsl.ParallelFor], - ): - """Get inputs and outputs of each group and op. - - Returns: - A tuple (inputs, outputs). - inputs and outputs are dicts with key being the group/op names and values being list of - tuples (param_name, producing_op_name). producing_op_name is the name of the op that - produces the param. If the param is a pipeline param (no producer op), then - producing_op_name is None. - """ - inputs = defaultdict(set) - outputs = defaultdict(set) - - for op in pipeline.ops.values(): - # op's inputs and all params used in conditions for that op are both considered. - for param in op.inputs + list(condition_params[op.name]): - # if the value is already provided (immediate value), then no need to expose - # it as input for its parent groups. - if param.value: - continue - if param.op_name: - upstream_op = pipeline.ops[param.op_name] - upstream_groups, downstream_groups = \ - self._get_uncommon_ancestors(op_groups, opsgroup_groups, upstream_op, op) - for i, group_name in enumerate(downstream_groups): - if i == 0: - # If it is the first uncommon downstream group, then the input comes from - # the first uncommon upstream group. - inputs[group_name].add( - (param.full_name, upstream_groups[0])) - else: - # If not the first downstream group, then the input is passed down from - # its ancestor groups so the upstream group is None. - inputs[group_name].add((param.full_name, None)) - for i, group_name in enumerate(upstream_groups): - if i == len(upstream_groups) - 1: - # If last upstream group, it is an operator and output comes from container. - outputs[group_name].add((param.full_name, None)) - else: - # If not last upstream group, output value comes from one of its child. - outputs[group_name].add( - (param.full_name, upstream_groups[i + 1])) - else: - if not op.is_exit_handler: - for group_name in op_groups[op.name][::-1]: - # if group is for loop group and param is that loop's param, then the param - # is created by that for loop ops_group and it shouldn't be an input to - # any of its parent groups. - inputs[group_name].add((param.full_name, None)) - if group_name in op_name_to_for_loop_op: - # for example: - # loop_group.loop_args.name = 'loop-item-param-99ca152e' - # param.name = 'loop-item-param-99ca152e--a' - loop_group = op_name_to_for_loop_op[group_name] - if loop_group.loop_args.name in param.name: - break - - # Generate the input/output for recursive opsgroups - # It propagates the recursive opsgroups IO to their ancester opsgroups - def _get_inputs_outputs_recursive_opsgroup(group): - #TODO: refactor the following codes with the above - if group.recursive_ref: - params = [(param, False) for param in group.inputs] - params.extend([(param, True) - for param in list(condition_params[group.name])]) - for param, is_condition_param in params: - if param.value: - continue - full_name = param.full_name - if param.op_name: - upstream_op = pipeline.ops[param.op_name] - upstream_groups, downstream_groups = \ - self._get_uncommon_ancestors(op_groups, opsgroup_groups, upstream_op, group) - for i, g in enumerate(downstream_groups): - if i == 0: - inputs[g].add((full_name, upstream_groups[0])) - # There is no need to pass the condition param as argument to the downstream ops. - #TODO: this might also apply to ops. add a TODO here and think about it. - elif i == len(downstream_groups - ) - 1 and is_condition_param: - continue - else: - inputs[g].add((full_name, None)) - for i, g in enumerate(upstream_groups): - if i == len(upstream_groups) - 1: - outputs[g].add((full_name, None)) - else: - outputs[g].add( - (full_name, upstream_groups[i + 1])) - elif not is_condition_param: - for g in op_groups[group.name]: - inputs[g].add((full_name, None)) - for subgroup in group.groups: - _get_inputs_outputs_recursive_opsgroup(subgroup) - - _get_inputs_outputs_recursive_opsgroup(root_group) - - # Generate the input for SubGraph along with parallelfor - for sub_graph in opsgroup_groups: - if sub_graph in op_name_to_for_loop_op: - # The opsgroup list is sorted with the farthest group as the first and - # the opsgroup itself as the last. To get the latest opsgroup which is - # not the opsgroup itself -2 is used. - parent = opsgroup_groups[sub_graph][-2] - if parent and parent.startswith('subgraph'): - # propagate only op's pipeline param from subgraph to parallelfor - loop_op = op_name_to_for_loop_op[sub_graph] - pipeline_param = loop_op.loop_args.items_or_pipeline_param - if loop_op.items_is_pipeline_param and pipeline_param.op_name: - param_name = '%s-%s' % (sanitize_k8s_name( - pipeline_param.op_name), pipeline_param.name) - inputs[parent].add((param_name, pipeline_param.op_name)) - - return inputs, outputs - - def _get_dependencies(self, pipeline, root_group, op_groups, - opsgroups_groups, opsgroups, condition_params): - """Get dependent groups and ops for all ops and groups. - - Returns: - A dict. Key is group/op name, value is a list of dependent groups/ops. - The dependencies are calculated in the following way: if op2 depends on op1, - and their ancestors are [root, G1, G2, op1] and [root, G1, G3, G4, op2], - then G3 is dependent on G2. Basically dependency only exists in the first uncommon - ancesters in their ancesters chain. Only sibling groups/ops can have dependencies. - """ - dependencies = defaultdict(set) - for op in pipeline.ops.values(): - upstream_op_names = set() - for param in op.inputs + list(condition_params[op.name]): - if param.op_name: - upstream_op_names.add(param.op_name) - upstream_op_names |= set(op.dependent_names) - - for upstream_op_name in upstream_op_names: - # the dependent op could be either a BaseOp or an opsgroup - if upstream_op_name in pipeline.ops: - upstream_op = pipeline.ops[upstream_op_name] - elif upstream_op_name in opsgroups: - upstream_op = opsgroups[upstream_op_name] - else: - raise ValueError('compiler cannot find the ' + - upstream_op_name) - - upstream_groups, downstream_groups = self._get_uncommon_ancestors( - op_groups, opsgroups_groups, upstream_op, op) - dependencies[downstream_groups[0]].add(upstream_groups[0]) - - # Generate dependencies based on the recursive opsgroups - #TODO: refactor the following codes with the above - def _get_dependency_opsgroup(group, dependencies): - upstream_op_names = set( - [dependency.name for dependency in group.dependencies]) - if group.recursive_ref: - for param in group.inputs + list(condition_params[group.name]): - if param.op_name: - upstream_op_names.add(param.op_name) - - for op_name in upstream_op_names: - if op_name in pipeline.ops: - upstream_op = pipeline.ops[op_name] - elif op_name in opsgroups: - upstream_op = opsgroups[op_name] - else: - raise ValueError('compiler cannot find the ' + op_name) - upstream_groups, downstream_groups = \ - self._get_uncommon_ancestors(op_groups, opsgroups_groups, upstream_op, group) - dependencies[downstream_groups[0]].add(upstream_groups[0]) - - for subgroup in group.groups: - _get_dependency_opsgroup(subgroup, dependencies) - - _get_dependency_opsgroup(root_group, dependencies) - - return dependencies - - def _resolve_value_or_reference(self, value_or_reference, - potential_references): - """_resolve_value_or_reference resolves values and PipelineParams, - which could be task parameters or input parameters. - - Args: - value_or_reference: value or reference to be resolved. It could be basic python types or PipelineParam - potential_references(dict{str->str}): a dictionary of parameter names to task names - """ - if isinstance(value_or_reference, dsl.PipelineParam): - parameter_name = value_or_reference.full_name - task_names = [ - task_name for param_name, task_name in potential_references - if param_name == parameter_name - ] - if task_names: - task_name = task_names[0] - # When the task_name is None, the parameter comes directly from ancient ancesters - # instead of parents. Thus, it is resolved as the input parameter in the current group. - if task_name is None: - return '{{inputs.parameters.%s}}' % parameter_name - else: - return '{{tasks.%s.outputs.parameters.%s}}' % ( - task_name, parameter_name) - else: - return '{{inputs.parameters.%s}}' % parameter_name - else: - return str(value_or_reference) - - @staticmethod - def _resolve_task_pipeline_param(pipeline_param: PipelineParam, - group_type) -> str: - if pipeline_param.op_name is None: - return '{{workflow.parameters.%s}}' % pipeline_param.name - param_name = '%s-%s' % (sanitize_k8s_name( - pipeline_param.op_name), pipeline_param.name) - if group_type == 'subgraph': - return '{{inputs.parameters.%s}}' % (param_name) - return '{{tasks.%s.outputs.parameters.%s}}' % (sanitize_k8s_name( - pipeline_param.op_name), param_name) - - def _group_to_dag_template(self, group, inputs, outputs, dependencies): - """Generate template given an OpsGroup. - - inputs, outputs, dependencies are all helper dicts. - """ - template = {'name': group.name} - if group.parallelism != None: - template["parallelism"] = group.parallelism - - # Generate inputs section. - if inputs.get(group.name, None): - template_inputs = [{'name': x[0]} for x in inputs[group.name]] - template_inputs.sort(key=lambda x: x['name']) - template['inputs'] = {'parameters': template_inputs} - - # Generate outputs section. - if outputs.get(group.name, None): - template_outputs = [] - for param_name, dependent_name in outputs[group.name]: - template_outputs.append({ - 'name': param_name, - 'valueFrom': { - 'parameter': - '{{tasks.%s.outputs.parameters.%s}}' % - (dependent_name, param_name) - } - }) - template_outputs.sort(key=lambda x: x['name']) - template['outputs'] = {'parameters': template_outputs} - - # Generate tasks section. - tasks = [] - sub_groups = group.groups + group.ops - for sub_group in sub_groups: - is_recursive_subgroup = ( - isinstance(sub_group, OpsGroup) and sub_group.recursive_ref) - # Special handling for recursive subgroup: use the existing opsgroup name - if is_recursive_subgroup: - task = { - 'name': sub_group.recursive_ref.name, - 'template': sub_group.recursive_ref.name, - } - else: - task = { - 'name': sub_group.name, - 'template': sub_group.name, - } - if isinstance(sub_group, - dsl.OpsGroup) and sub_group.type == 'condition': - subgroup_inputs = inputs.get(sub_group.name, []) - condition = sub_group.condition - operand1_value = self._resolve_value_or_reference( - condition.operand1, subgroup_inputs) - operand2_value = self._resolve_value_or_reference( - condition.operand2, subgroup_inputs) - if condition.operator in ['==', '!=']: - operand1_value = '"' + operand1_value + '"' - operand2_value = '"' + operand2_value + '"' - task['when'] = '{} {} {}'.format(operand1_value, - condition.operator, - operand2_value) - - # Generate dependencies section for this task. - if dependencies.get(sub_group.name, None): - group_dependencies = list(dependencies[sub_group.name]) - group_dependencies.sort() - task['dependencies'] = group_dependencies - - # Generate arguments section for this task. - if inputs.get(sub_group.name, None): - task['arguments'] = { - 'parameters': - self.get_arguments_for_sub_group( - sub_group, is_recursive_subgroup, inputs) - } - - # additional task modifications for withItems and withParam - if isinstance(sub_group, dsl.ParallelFor): - if sub_group.items_is_pipeline_param: - # these loop args are a 'withParam' rather than 'withItems'. - # i.e., rather than a static list, they are either the output of another task or were input - # as global pipeline parameters - - pipeline_param = sub_group.loop_args.items_or_pipeline_param - withparam_value = self._resolve_task_pipeline_param( - pipeline_param, group.type) - if pipeline_param.op_name: - # these loop args are the output of another task - if 'dependencies' not in task or task[ - 'dependencies'] is None: - task['dependencies'] = [] - if sanitize_k8s_name( - pipeline_param.op_name - ) not in task[ - 'dependencies'] and group.type != 'subgraph': - task['dependencies'].append( - sanitize_k8s_name(pipeline_param.op_name)) - - task['withParam'] = withparam_value - else: - # Need to sanitize the dict keys for consistency. - loop_tasks = sub_group.loop_args.to_list_for_task_yaml() - nested_pipeline_params = extract_pipelineparams_from_any( - loop_tasks) - - # Set dependencies in case of nested pipeline_params - map_to_tmpl_var = { - str(p): - self._resolve_task_pipeline_param(p, group.type) - for p in nested_pipeline_params - } - for pipeline_param in nested_pipeline_params: - if pipeline_param.op_name: - # these pipeline_param are the output of another task - if 'dependencies' not in task or task[ - 'dependencies'] is None: - task['dependencies'] = [] - if sanitize_k8s_name(pipeline_param.op_name - ) not in task['dependencies']: - task['dependencies'].append( - sanitize_k8s_name(pipeline_param.op_name)) - - sanitized_tasks = [] - if isinstance(loop_tasks[0], dict): - for argument_set in loop_tasks: - c_dict = {} - for k, v in argument_set.items(): - c_dict[sanitize_k8s_name(k, True)] = v - sanitized_tasks.append(c_dict) - else: - sanitized_tasks = loop_tasks - # Replace pipeline param if map_to_tmpl_var not empty - task['withItems'] = _process_obj( - sanitized_tasks, - map_to_tmpl_var) if map_to_tmpl_var else sanitized_tasks - - # We will sort dependencies to have determinitc yaml and thus stable tests - if task.get('dependencies'): - task['dependencies'].sort() - - tasks.append(task) - tasks.sort(key=lambda x: x['name']) - template['dag'] = {'tasks': tasks} - return template - - def get_arguments_for_sub_group( - self, - sub_group: Union[OpsGroup, dsl._container_op.BaseOp], - is_recursive_subgroup: Optional[bool], - inputs: Dict[Text, Tuple[Text, Text]], - ): - arguments = [] - for param_name, dependent_name in inputs[sub_group.name]: - if is_recursive_subgroup: - for input_name, input in sub_group.arguments.items(): - if param_name == input.full_name: - break - referenced_input = sub_group.recursive_ref.arguments[input_name] - argument_name = referenced_input.full_name - else: - argument_name = param_name - - # Preparing argument. It can be pipeline input reference, task output reference or loop item (or loop item attribute - sanitized_loop_arg_full_name = '---' - if isinstance(sub_group, dsl.ParallelFor): - sanitized_loop_arg_full_name = sanitize_k8s_name( - sub_group.loop_args.full_name) - arg_ref_full_name = sanitize_k8s_name(param_name) - # We only care about the reference to the current loop item, not the outer loops - if isinstance(sub_group, - dsl.ParallelFor) and arg_ref_full_name.startswith( - sanitized_loop_arg_full_name): - if arg_ref_full_name == sanitized_loop_arg_full_name: - argument_value = '{{item}}' - elif _for_loop.LoopArgumentVariable.name_is_loop_arguments_variable( - param_name): - subvar_name = _for_loop.LoopArgumentVariable.get_subvar_name( - param_name) - argument_value = '{{item.%s}}' % subvar_name - else: - raise ValueError( - "Argument seems to reference the loop item, but not the item itself and not some attribute of the item. param_name: {}, " - .format(param_name)) - else: - if dependent_name: - argument_value = '{{tasks.%s.outputs.parameters.%s}}' % ( - dependent_name, param_name) - else: - argument_value = '{{inputs.parameters.%s}}' % param_name - - arguments.append({ - 'name': argument_name, - 'value': argument_value, - }) - - arguments.sort(key=lambda x: x['name']) - - return arguments - - def _create_dag_templates(self, - pipeline, - op_transformers=None, - op_to_templates_handler=None): - """Create all groups and ops templates in the pipeline. - - Args: - pipeline: Pipeline context object to get all the pipeline data from. - op_transformers: A list of functions that are applied to all ContainerOp instances that are being processed. - op_to_templates_handler: Handler which converts a base op into a list of argo templates. - """ - op_to_templates_handler = op_to_templates_handler or ( - lambda op: [_op_to_template(op)]) - root_group = pipeline.groups[0] - - # Call the transformation functions before determining the inputs/outputs, otherwise - # the user would not be able to use pipeline parameters in the container definition - # (for example as pod labels) - the generated template is invalid. - for op in pipeline.ops.values(): - for transformer in op_transformers or []: - transformer(op) - - # Generate core data structures to prepare for argo yaml generation - # op_name_to_parent_groups: op name -> list of ancestor groups including the current op - # opsgroups: a dictionary of ospgroup.name -> opsgroup - # inputs, outputs: group/op names -> list of tuples (full_param_name, producing_op_name) - # condition_params: recursive_group/op names -> list of pipelineparam - # dependencies: group/op name -> list of dependent groups/ops. - # Special Handling for the recursive opsgroup - # op_name_to_parent_groups also contains the recursive opsgroups - # condition_params from _get_condition_params_for_ops also contains the recursive opsgroups - # groups does not include the recursive opsgroups - opsgroups = self._get_groups(root_group) - op_name_to_parent_groups = self._get_groups_for_ops(root_group) - opgroup_name_to_parent_groups = self._get_groups_for_opsgroups( - root_group) - condition_params = self._get_condition_params_for_ops(root_group) - op_name_to_for_loop_op = self._get_for_loop_ops(root_group) - inputs, outputs = self._get_inputs_outputs( - pipeline, - root_group, - op_name_to_parent_groups, - opgroup_name_to_parent_groups, - condition_params, - op_name_to_for_loop_op, - ) - dependencies = self._get_dependencies( - pipeline, - root_group, - op_name_to_parent_groups, - opgroup_name_to_parent_groups, - opsgroups, - condition_params, - ) - - templates = [] - for opsgroup in opsgroups.keys(): - template = self._group_to_dag_template(opsgroups[opsgroup], inputs, - outputs, dependencies) - templates.append(template) - - for op in pipeline.ops.values(): - if hasattr(op, 'importer_spec'): - raise ValueError( - 'dsl.importer is not supported with v1 compiler.') - - if self._mode == dsl.PipelineExecutionMode.V2_COMPATIBLE: - v2_compat.update_op( - op, - pipeline_name=self._pipeline_name_param, - pipeline_root=self._pipeline_root_param, - launcher_image=self._launcher_image) - templates.extend(op_to_templates_handler(op)) - - if hasattr(op, 'custom_job_spec'): - warnings.warn( - 'CustomJob spec is not supported yet when running on KFP.' - ' The component will execute within the KFP cluster.') - - return templates - - def _create_pipeline_workflow(self, - parameter_defaults, - pipeline, - op_transformers=None, - pipeline_conf=None): - """Create workflow for the pipeline.""" - - # Input Parameters - input_params = [] - for name, value in parameter_defaults.items(): - param = {'name': name} - if value is not None: - param['value'] = value - input_params.append(param) - - # Making the pipeline group name unique to prevent name clashes with templates - pipeline_group = pipeline.groups[0] - temp_pipeline_group_name = uuid.uuid4().hex - pipeline_group.name = temp_pipeline_group_name - - # Templates - templates = self._create_dag_templates(pipeline, op_transformers) - - # Exit Handler - exit_handler = None - if pipeline.groups[0].groups: - first_group = pipeline.groups[0].groups[0] - if first_group.type == 'exit_handler': - exit_handler = first_group.exit_op - - # The whole pipeline workflow - # It must valid as a subdomain - pipeline_name = pipeline.name or 'pipeline' - - # Workaround for pipeline name clashing with container template names - # TODO: Make sure template names cannot clash at all (container, DAG, workflow) - template_map = { - template['name'].lower(): template for template in templates - } - from ..components._naming import _make_name_unique_by_adding_index - pipeline_template_name = _make_name_unique_by_adding_index( - pipeline_name, template_map, '-') - - # Restoring the name of the pipeline template - pipeline_template = template_map[temp_pipeline_group_name] - pipeline_template['name'] = pipeline_template_name - - templates.sort(key=lambda x: x['name']) - workflow = { - 'apiVersion': 'argoproj.io/v1alpha1', - 'kind': 'Workflow', - 'metadata': { - 'generateName': pipeline_template_name + '-' - }, - 'spec': { - 'entrypoint': pipeline_template_name, - 'templates': templates, - 'arguments': { - 'parameters': input_params - }, - 'serviceAccountName': 'pipeline-runner', - } - } - # set parallelism limits at pipeline level - if pipeline_conf.parallelism: - workflow['spec']['parallelism'] = pipeline_conf.parallelism - - # set ttl after workflow finishes - if pipeline_conf.ttl_seconds_after_finished >= 0: - workflow['spec']['ttlStrategy'] = { - 'secondsAfterCompletion': - pipeline_conf.ttl_seconds_after_finished - } - - if pipeline_conf._pod_disruption_budget_min_available: - pod_disruption_budget = { - "minAvailable": - pipeline_conf._pod_disruption_budget_min_available - } - workflow['spec']['podDisruptionBudget'] = pod_disruption_budget - - if len(pipeline_conf.image_pull_secrets) > 0: - image_pull_secrets = [] - for image_pull_secret in pipeline_conf.image_pull_secrets: - image_pull_secrets.append( - convert_k8s_obj_to_json(image_pull_secret)) - workflow['spec']['imagePullSecrets'] = image_pull_secrets - - if pipeline_conf.timeout: - workflow['spec']['activeDeadlineSeconds'] = pipeline_conf.timeout - - if exit_handler: - workflow['spec']['onExit'] = exit_handler.name - - # This can be overwritten by the task specific - # nodeselection, specified in the template. - if pipeline_conf.default_pod_node_selector: - workflow['spec'][ - 'nodeSelector'] = pipeline_conf.default_pod_node_selector - - if pipeline_conf.dns_config: - workflow['spec']['dnsConfig'] = convert_k8s_obj_to_json( - pipeline_conf.dns_config) - - if pipeline_conf.image_pull_policy != None: - if pipeline_conf.image_pull_policy in [ - "Always", "Never", "IfNotPresent" - ]: - for template in workflow["spec"]["templates"]: - container = template.get('container', None) - if container and "imagePullPolicy" not in container: - container[ - "imagePullPolicy"] = pipeline_conf.image_pull_policy - else: - raise ValueError( - 'Invalid imagePullPolicy. Must be one of `Always`, `Never`, `IfNotPresent`.' - ) - return workflow - - def _validate_exit_handler(self, pipeline): - """Makes sure there is only one global exit handler. - - Note this is a temporary workaround until argo supports local - exit handler. - """ - - def _validate_exit_handler_helper(group, exiting_op_names, - handler_exists): - if group.type == 'exit_handler': - if handler_exists or len(exiting_op_names) > 1: - raise ValueError( - 'Only one global exit_handler is allowed and all ops need to be included.' - ) - handler_exists = True - - if group.ops: - exiting_op_names.extend([x.name for x in group.ops]) - - for g in group.groups: - _validate_exit_handler_helper(g, exiting_op_names, - handler_exists) - - return _validate_exit_handler_helper(pipeline.groups[0], [], False) - - def _sanitize_and_inject_artifact(self, - pipeline: dsl.Pipeline, - pipeline_conf=None): - """Sanitize operator/param names and inject pipeline artifact - location.""" - - # Sanitize operator names and param names - sanitized_ops = {} - - for op in pipeline.ops.values(): - sanitized_name = sanitize_k8s_name(op.name) - op.name = sanitized_name - for param in op.outputs.values(): - param.name = sanitize_k8s_name(param.name, True) - if param.op_name: - param.op_name = sanitize_k8s_name(param.op_name) - if op.output is not None and not isinstance( - op.output, dsl._container_op._MultipleOutputsError): - op.output.name = sanitize_k8s_name(op.output.name, True) - op.output.op_name = sanitize_k8s_name(op.output.op_name) - if op.dependent_names: - op.dependent_names = [ - sanitize_k8s_name(name) for name in op.dependent_names - ] - if isinstance(op, dsl.ContainerOp) and op.file_outputs is not None: - sanitized_file_outputs = {} - for key in op.file_outputs.keys(): - sanitized_file_outputs[sanitize_k8s_name( - key, True)] = op.file_outputs[key] - op.file_outputs = sanitized_file_outputs - elif isinstance( - op, dsl.ResourceOp) and op.attribute_outputs is not None: - sanitized_attribute_outputs = {} - for key in op.attribute_outputs.keys(): - sanitized_attribute_outputs[sanitize_k8s_name(key, True)] = \ - op.attribute_outputs[key] - op.attribute_outputs = sanitized_attribute_outputs - if isinstance(op, dsl.ContainerOp): - if op.input_artifact_paths: - op.input_artifact_paths = { - sanitize_k8s_name(key, True): value - for key, value in op.input_artifact_paths.items() - } - if op.artifact_arguments: - op.artifact_arguments = { - sanitize_k8s_name(key, True): value - for key, value in op.artifact_arguments.items() - } - sanitized_ops[sanitized_name] = op - pipeline.ops = sanitized_ops - - def _create_workflow( - self, - pipeline_func: Callable, - pipeline_name: Optional[Text] = None, - pipeline_description: Optional[Text] = None, - params_list: Optional[List[dsl.PipelineParam]] = None, - pipeline_conf: Optional[dsl.PipelineConf] = None, - ) -> Dict[Text, Any]: - """Internal implementation of create_workflow.""" - params_list = params_list or [] - - # Create the arg list with no default values and call pipeline function. - # Assign type information to the PipelineParam - pipeline_meta = _extract_pipeline_metadata(pipeline_func) - pipeline_meta.name = pipeline_name or pipeline_meta.name - pipeline_meta.description = pipeline_description or pipeline_meta.description - pipeline_name = sanitize_k8s_name(pipeline_meta.name) - - # Need to first clear the default value of dsl.PipelineParams. Otherwise, it - # will be resolved immediately in place when being to each component. - default_param_values = OrderedDict() - - if self._pipeline_root_param: - params_list.append(self._pipeline_root_param) - if self._pipeline_name_param: - params_list.append(self._pipeline_name_param) - - for param in params_list: - default_param_values[param.name] = param.value - param.value = None - - args_list = [] - kwargs_dict = dict() - signature = inspect.signature(pipeline_func) - for arg_name, arg in signature.parameters.items(): - arg_type = None - for input in pipeline_meta.inputs or []: - if arg_name == input.name: - arg_type = input.type - break - param = dsl.PipelineParam( - sanitize_k8s_name(arg_name, True), param_type=arg_type) - if arg.kind == inspect.Parameter.KEYWORD_ONLY: - kwargs_dict[arg_name] = param - else: - args_list.append(param) - - with dsl.Pipeline(pipeline_name) as dsl_pipeline: - pipeline_func(*args_list, **kwargs_dict) - - pipeline_conf = pipeline_conf or dsl_pipeline.conf # Configuration passed to the compiler is overriding. Unfortunately, it's not trivial to detect whether the dsl_pipeline.conf was ever modified. - - self._validate_exit_handler(dsl_pipeline) - self._sanitize_and_inject_artifact(dsl_pipeline, pipeline_conf) - - # Fill in the default values by merging two param lists. - args_list_with_defaults = OrderedDict() - if pipeline_meta.inputs: - args_list_with_defaults = OrderedDict([ - (sanitize_k8s_name(input_spec.name, True), input_spec.default) - for input_spec in pipeline_meta.inputs - ]) - - if params_list: - # Or, if args are provided by params_list, fill in pipeline_meta. - for k, v in default_param_values.items(): - args_list_with_defaults[k] = v - - pipeline_meta.inputs = pipeline_meta.inputs or [] - for param in params_list: - pipeline_meta.inputs.append( - InputSpec( - name=param.name, - type=param.param_type, - default=default_param_values[param.name])) - - op_transformers = [add_pod_env] - pod_labels = { - _SDK_VERSION_LABEL: kfp.__version__, - _SDK_ENV_LABEL: _SDK_ENV_DEFAULT - } - op_transformers.append(add_pod_labels(pod_labels)) - op_transformers.extend(pipeline_conf.op_transformers) - - if self._mode == dsl.PipelineExecutionMode.V2_COMPATIBLE: - # Add self._pipeline_name_param and self._pipeline_root_param to ops inputs - # if they don't exist already. - for op in dsl_pipeline.ops.values(): - insert_pipeline_name_param = True - insert_pipeline_root_param = True - for param in op.inputs: - if param.name == self._pipeline_name_param.name: - insert_pipeline_name_param = False - elif param.name == self._pipeline_root_param.name: - insert_pipeline_root_param = False - - if insert_pipeline_name_param: - op.inputs.append(self._pipeline_name_param) - if insert_pipeline_root_param: - op.inputs.append(self._pipeline_root_param) - - workflow = self._create_pipeline_workflow( - args_list_with_defaults, - dsl_pipeline, - op_transformers, - pipeline_conf, - ) - - from ._data_passing_rewriter import fix_big_data_passing - workflow = fix_big_data_passing(workflow) - - if pipeline_conf and pipeline_conf.data_passing_method != None: - workflow = pipeline_conf.data_passing_method(workflow) - - metadata = workflow.setdefault('metadata', {}) - annotations = metadata.setdefault('annotations', {}) - labels = metadata.setdefault('labels', {}) - - annotations[_SDK_VERSION_LABEL] = kfp.__version__ - annotations[ - 'pipelines.kubeflow.org/pipeline_compilation_time'] = datetime.datetime.now( - ).isoformat() - annotations['pipelines.kubeflow.org/pipeline_spec'] = json.dumps( - pipeline_meta.to_dict(), sort_keys=True) - - if self._mode == dsl.PipelineExecutionMode.V2_COMPATIBLE: - annotations['pipelines.kubeflow.org/v2_pipeline'] = "true" - labels['pipelines.kubeflow.org/v2_pipeline'] = "true" - - # Labels might be logged better than annotations so adding some information here as well - labels[_SDK_VERSION_LABEL] = kfp.__version__ - - return workflow - - def compile(self, - pipeline_func, - package_path, - type_check: bool = True, - pipeline_conf: Optional[dsl.PipelineConf] = None): - """Compile the given pipeline function into workflow yaml. - - Args: - pipeline_func: Pipeline functions with @dsl.pipeline decorator. - package_path: The output workflow tar.gz file path. for example, - "~/a.tar.gz" - type_check: Whether to enable the type check or not, default: True. - pipeline_conf: PipelineConf instance. Can specify op transforms, image - pull secrets and other pipeline-level configuration options. Overrides - any configuration that may be set by the pipeline. - """ - pipeline_root_dir = getattr(pipeline_func, 'pipeline_root', None) - if (pipeline_root_dir is not None or - self._mode == dsl.PipelineExecutionMode.V2_COMPATIBLE): - self._pipeline_root_param = dsl.PipelineParam( - name=dsl.ROOT_PARAMETER_NAME, value=pipeline_root_dir or '') - - if self._mode == dsl.PipelineExecutionMode.V2_COMPATIBLE: - pipeline_name = getattr(pipeline_func, '_component_human_name', '') - if not pipeline_name: - raise ValueError( - '@dsl.pipeline decorator name field is required in v2 compatible mode' - ) - # pipeline names have one of the following formats: - # * pipeline/ - # * namespace//pipeline/ - # when compiling, we will only have pipeline/, but it will be overriden - # when uploading the pipeline to KFP API server. - self._pipeline_name_param = dsl.PipelineParam( - name='pipeline-name', value=f'pipeline/{pipeline_name}') - - import kfp.deprecated as kfp - type_check_old_value = kfp.TYPE_CHECK - compiling_for_v2_old_value = kfp.COMPILING_FOR_V2 - kfp.COMPILING_FOR_V2 = self._mode in [ - dsl.PipelineExecutionMode.V2_COMPATIBLE, - dsl.PipelineExecutionMode.V2_ENGINE, - ] - - try: - kfp.TYPE_CHECK = type_check - self._create_and_write_workflow( - pipeline_func=pipeline_func, - pipeline_conf=pipeline_conf, - package_path=package_path) - finally: - kfp.TYPE_CHECK = type_check_old_value - kfp.COMPILING_FOR_V2 = compiling_for_v2_old_value - - @staticmethod - def _write_workflow(workflow: Dict[Text, Any], package_path: Text = None): - """Dump pipeline workflow into yaml spec and write out in the format - specified by the user. - - Args: - workflow: Workflow spec of the pipline, dict. - package_path: file path to be written. If not specified, a yaml_text string will be returned. - """ - yaml_text = dump_yaml(workflow) - - if package_path is None: - return yaml_text - - if package_path.endswith('.tar.gz') or package_path.endswith('.tgz'): - from contextlib import closing - from io import BytesIO - with tarfile.open(package_path, "w:gz") as tar: - with closing(BytesIO(yaml_text.encode())) as yaml_file: - tarinfo = tarfile.TarInfo('pipeline.yaml') - tarinfo.size = len(yaml_file.getvalue()) - tar.addfile(tarinfo, fileobj=yaml_file) - elif package_path.endswith('.zip'): - with zipfile.ZipFile(package_path, "w") as zip: - zipinfo = zipfile.ZipInfo('pipeline.yaml') - zipinfo.compress_type = zipfile.ZIP_DEFLATED - zip.writestr(zipinfo, yaml_text) - elif package_path.endswith('.yaml') or package_path.endswith('.yml'): - with open(package_path, 'w') as yaml_file: - yaml_file.write(yaml_text) - else: - raise ValueError('The output path ' + package_path + - ' should ends with one of the following formats: ' - '[.tar.gz, .tgz, .zip, .yaml, .yml]') - - def _create_and_write_workflow(self, - pipeline_func: Callable, - pipeline_name: Text = None, - pipeline_description: Text = None, - params_list: List[dsl.PipelineParam] = None, - pipeline_conf: dsl.PipelineConf = None, - package_path: Text = None) -> None: - """Compile the given pipeline function and dump it to specified file - format.""" - workflow = self._create_workflow(pipeline_func, pipeline_name, - pipeline_description, params_list, - pipeline_conf) - self._write_workflow(workflow, package_path) - _validate_workflow(workflow) - - -def _validate_workflow(workflow: dict): - workflow = workflow.copy() - # Working around Argo lint issue - for argument in workflow['spec'].get('arguments', {}).get('parameters', []): - if 'value' not in argument: - argument['value'] = '' - - yaml_text = dump_yaml(workflow) - if '{{pipelineparam' in yaml_text: - raise RuntimeError( - '''Internal compiler error: Found unresolved PipelineParam. -Please create a new issue at https://github.com/kubeflow/pipelines/issues attaching the pipeline code and the pipeline package.''' - ) - - # Running Argo lint if available - import shutil - argo_path = shutil.which('argo') - if argo_path: - has_working_argo_lint = False - try: - has_working_argo_lint = _run_argo_lint(""" - apiVersion: argoproj.io/v1alpha1 - kind: Workflow - metadata: - generateName: hello-world- - spec: - entrypoint: whalesay - templates: - - name: whalesay - container: - image: docker/whalesay:latest""") - except: - warnings.warn( - "Cannot validate the compiled workflow. Found the argo program in PATH, but it's not usable. argo CLI v3.1.1+ should work." - ) - - if has_working_argo_lint: - _run_argo_lint(yaml_text) - - -def _run_argo_lint(yaml_text: str): - # Running Argo lint if available - import shutil - import subprocess - argo_path = shutil.which('argo') - if argo_path: - result = subprocess.run([ - argo_path, '--offline=true', '--kinds=workflows', 'lint', - '/dev/stdin' - ], - input=yaml_text.encode('utf-8'), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - if result.returncode: - if re.match( - pattern=r'.+failed to resolve {{tasks\..+\.outputs\.artifacts\..+}}.+', - string=result.stderr.decode('utf-8')): - raise RuntimeError( - 'Compiler has produced Argo-incompatible workflow due to ' - 'unresolvable input artifact(s). Please check whether inputPath has' - ' been connected to outputUri placeholder, which is not supported ' - 'yet. Otherwise, please create a new issue at ' - 'https://github.com/kubeflow/pipelines/issues attaching the ' - 'pipeline code and the pipeline package. Error: {}'.format( - result.stderr.decode('utf-8'))) - print(result) - raise RuntimeError( - '''Internal compiler error: Compiler has produced Argo-incompatible workflow. -Please create a new issue at https://github.com/kubeflow/pipelines/issues attaching the pipeline code and the pipeline package. -Error: {}'''.format(result.stdout.decode('utf-8'))) - - return True - return False diff --git a/sdk/python/kfp/deprecated/compiler/main.py b/sdk/python/kfp/deprecated/compiler/main.py deleted file mode 100644 index 1fc26984f61..00000000000 --- a/sdk/python/kfp/deprecated/compiler/main.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -from typing import Optional -from kfp.deprecated import dsl -import kfp.deprecated as kfp -import kfp.deprecated.compiler -import os -import sys - -_KF_PIPELINES_COMPILER_MODE_ENV = 'KF_PIPELINES_COMPILER_MODE' - - -def parse_arguments(): - """Parse command line arguments.""" - - parser = argparse.ArgumentParser() - parser.add_argument( - '--py', type=str, help='local absolute path to a py file.') - parser.add_argument( - '--function', - type=str, - help='The name of the function to compile if there are multiple.') - parser.add_argument( - '--namespace', type=str, help='The namespace for the pipeline function') - parser.add_argument( - '--output', - type=str, - required=True, - help='local path to the output workflow yaml file.') - parser.add_argument( - '--disable-type-check', - action='store_true', - help='disable the type check, default is enabled.') - parser.add_argument( - '--mode', - type=str, - help='compiler mode, defaults to V1, can also be V2_COMPATIBLE. You can override the default using env var KF_PIPELINES_COMPILER_MODE.' - ) - - args = parser.parse_args() - return args - - -def _compile_pipeline_function( - pipeline_funcs, - function_name, - output_path, - type_check, - mode: Optional[dsl.PipelineExecutionMode] = None, - pipeline_conf: Optional[dsl.PipelineConf] = None): - if len(pipeline_funcs) == 0: - raise ValueError( - 'A function with @dsl.pipeline decorator is required in the py file.' - ) - - if len(pipeline_funcs) > 1 and not function_name: - func_names = [x.__name__ for x in pipeline_funcs] - raise ValueError( - 'There are multiple pipelines: %s. Please specify --function.' % - func_names) - - if function_name: - pipeline_func = next( - (x for x in pipeline_funcs if x.__name__ == function_name), None) - if not pipeline_func: - raise ValueError('The function "%s" does not exist. ' - 'Did you forget @dsl.pipeline decoration?' % - function_name) - else: - pipeline_func = pipeline_funcs[0] - - kfp.deprecated.compiler.Compiler(mode=mode).compile(pipeline_func, - output_path, type_check, - pipeline_conf) - - -class PipelineCollectorContext(): - - def __enter__(self): - pipeline_funcs = [] - - def add_pipeline(func): - pipeline_funcs.append(func) - return func - - self.old_handler = dsl._pipeline._pipeline_decorator_handler - dsl._pipeline._pipeline_decorator_handler = add_pipeline - return pipeline_funcs - - def __exit__(self, *args): - dsl._pipeline._pipeline_decorator_handler = self.old_handler - - -def compile_pyfile(pyfile, - output_path, - function_name=None, - type_check=True, - mode: Optional[dsl.PipelineExecutionMode] = None, - pipeline_conf: Optional[dsl.PipelineConf] = None): - sys.path.insert(0, os.path.dirname(pyfile)) - try: - filename = os.path.basename(pyfile) - with PipelineCollectorContext() as pipeline_funcs: - __import__(os.path.splitext(filename)[0]) - _compile_pipeline_function(pipeline_funcs, function_name, output_path, - type_check, mode, pipeline_conf) - finally: - del sys.path[0] - - -def main(): - args = parse_arguments() - if args.py is None: - raise ValueError('The --py option must be specified.') - mode_str = args.mode - if not mode_str: - mode_str = os.environ.get(_KF_PIPELINES_COMPILER_MODE_ENV, 'V1') - mode = None - if mode_str == 'V1_LEGACY' or mode_str == 'V1': - mode = kfp.dsl.PipelineExecutionMode.V1_LEGACY - elif mode_str == 'V2_COMPATIBLE': - mode = kfp.dsl.PipelineExecutionMode.V2_COMPATIBLE - elif mode_str == 'V2_ENGINE': - mode = kfp.dsl.PipelineExecutionMode.V2_ENGINE - else: - raise ValueError( - f'Got unexpected --mode option "{mode_str}", it must be one of V1, V2_COMPATIBLE or V2_ENGINE' - ) - compile_pyfile( - args.py, - args.output, - args.function, - not args.disable_type_check, - mode, - ) diff --git a/sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline.yaml b/sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline.yaml deleted file mode 100644 index b85db0a0a3a..00000000000 --- a/sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline.yaml +++ /dev/null @@ -1,254 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: my-test-pipeline- - annotations: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 - pipelines.kubeflow.org/pipeline_compilation_time: '2021-10-26T15:02:07.868312' - pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "gs://output-directory/v2-artifacts", - "name": "pipeline-root"}, {"default": "pipeline/my-test-pipeline", "name": "pipeline-name"}], - "name": "my-test-pipeline"}' - pipelines.kubeflow.org/v2_pipeline: "true" - labels: - pipelines.kubeflow.org/v2_pipeline: "true" - pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 -spec: - entrypoint: my-test-pipeline - templates: - - name: my-test-pipeline - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - dag: - tasks: - - name: preprocess - template: preprocess - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - - name: train - template: train - dependencies: [preprocess] - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - - {name: preprocess-output_parameter_one, value: '{{tasks.preprocess.outputs.parameters.preprocess-output_parameter_one}}'} - artifacts: - - {name: preprocess-output_dataset_one, from: '{{tasks.preprocess.outputs.artifacts.preprocess-output_dataset_one}}'} - - name: preprocess - container: - args: - - sh - - -c - - |2 - - if ! [ -x "$(command -v pip)" ]; then - python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip - fi - - PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && "$0" "$@" - - sh - - -ec - - | - program_path=$(mktemp -d) - printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - |2+ - - import kfp - from kfp import dsl - from kfp.dsl import * - from typing import * - - def preprocess(uri: str, some_int: int, output_parameter_one: OutputPath(int), - output_dataset_one: OutputPath('Dataset')): - """Dummy Preprocess Step.""" - with open(output_dataset_one, 'w') as f: - f.write('Output dataset') - with open(output_parameter_one, 'w') as f: - f.write("{}".format(1234)) - - - --executor_input - - '{{$}}' - - --function_to_execute - - preprocess - command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST), - --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO), - --container_image, $(KFP_V2_IMAGE), --task_name, preprocess, --pipeline_name, - '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource, - workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name, - $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}', - --enable_caching, $(ENABLE_CACHING), --, some_int=12, uri=uri-to-import, --] - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: {fieldPath: metadata.name} - - name: KFP_POD_UID - valueFrom: - fieldRef: {fieldPath: metadata.uid} - - name: KFP_NAMESPACE - valueFrom: - fieldRef: {fieldPath: metadata.namespace} - - name: WORKFLOW_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'} - - name: KFP_RUN_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'} - - name: ENABLE_CACHING - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'python:3.9'} - - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"some_int": {"type": - "NUMBER_INTEGER"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters": - {"output_parameter_one": {"type": "NUMBER_INTEGER", "path": "/tmp/outputs/output_parameter_one/data"}}, - "outputArtifacts": {"output_dataset_one": {"schemaTitle": "system.Dataset", - "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'} - envFrom: - - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: python:3.9 - volumeMounts: - - {mountPath: /kfp-launcher, name: kfp-launcher} - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - outputs: - parameters: - - name: preprocess-output_parameter_one - valueFrom: {path: /tmp/outputs/output_parameter_one/data} - artifacts: - - {name: preprocess-output_dataset_one, path: /tmp/outputs/output_dataset_one/data} - - {name: preprocess-output_parameter_one, path: /tmp/outputs/output_parameter_one/data} - metadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/component_ref: '{}' - pipelines.kubeflow.org/arguments.parameters: '{"some_int": "12", "uri": "uri-to-import"}' - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/enable_caching: "true" - initContainers: - - command: [launcher, --copy, /kfp-launcher/launch] - image: gcr.io/ml-pipeline/kfp-launcher:1.8.6 - name: kfp-launcher - mirrorVolumeMounts: true - volumes: - - {name: kfp-launcher} - - name: train - container: - args: - - sh - - -c - - |2 - - if ! [ -x "$(command -v pip)" ]; then - python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip - fi - - PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && "$0" "$@" - - sh - - -ec - - | - program_path=$(mktemp -d) - printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - |2+ - - import kfp - from kfp import dsl - from kfp.dsl import * - from typing import * - - def train(dataset: InputPath('Dataset'), - model: OutputPath('Model'), - num_steps: int = 100): - """Dummy Training Step.""" - - with open(dataset, 'r') as input_file: - input_string = input_file.read() - with open(model, 'w') as output_file: - for i in range(num_steps): - output_file.write("Step {}\n{}\n=====\n".format( - i, input_string)) - - - --executor_input - - '{{$}}' - - --function_to_execute - - train - command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST), - --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO), - --container_image, $(KFP_V2_IMAGE), --task_name, train, --pipeline_name, '{{inputs.parameters.pipeline-name}}', - --run_id, $(KFP_RUN_ID), --run_resource, workflows.argoproj.io/$(WORKFLOW_ID), - --namespace, $(KFP_NAMESPACE), --pod_name, $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), - --pipeline_root, '{{inputs.parameters.pipeline-root}}', --enable_caching, - $(ENABLE_CACHING), --, 'num_steps={{inputs.parameters.preprocess-output_parameter_one}}', - --] - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: {fieldPath: metadata.name} - - name: KFP_POD_UID - valueFrom: - fieldRef: {fieldPath: metadata.uid} - - name: KFP_NAMESPACE - valueFrom: - fieldRef: {fieldPath: metadata.namespace} - - name: WORKFLOW_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'} - - name: KFP_RUN_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'} - - name: ENABLE_CACHING - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'python:3.9'} - - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type": - "NUMBER_INTEGER"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data", - "schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion": - "0.0.1"}}, "outputParameters": {}, "outputArtifacts": {"model": {"schemaTitle": - "system.Model", "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": - "/tmp/outputs/model/data"}}}'} - envFrom: - - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: python:3.9 - volumeMounts: - - {mountPath: /kfp-launcher, name: kfp-launcher} - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - - {name: preprocess-output_parameter_one} - artifacts: - - {name: preprocess-output_dataset_one, path: /tmp/inputs/dataset/data} - outputs: - artifacts: - - {name: train-model, path: /tmp/outputs/model/data} - metadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/component_ref: '{}' - pipelines.kubeflow.org/arguments.parameters: '{"num_steps": "{{inputs.parameters.preprocess-output_parameter_one}}"}' - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/enable_caching: "true" - initContainers: - - command: [launcher, --copy, /kfp-launcher/launch] - image: gcr.io/ml-pipeline/kfp-launcher:1.8.6 - name: kfp-launcher - mirrorVolumeMounts: true - volumes: - - {name: kfp-launcher} - arguments: - parameters: - - {name: pipeline-root, value: 'gs://output-directory/v2-artifacts'} - - {name: pipeline-name, value: pipeline/my-test-pipeline} - serviceAccountName: pipeline-runner diff --git a/sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml b/sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml deleted file mode 100644 index 94195b292a3..00000000000 --- a/sdk/python/kfp/deprecated/compiler/testdata/v2_compatible_two_step_pipeline_with_custom_launcher.yaml +++ /dev/null @@ -1,254 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: my-test-pipeline-with-custom-launcher- - annotations: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 - pipelines.kubeflow.org/pipeline_compilation_time: '2021-10-26T15:02:07.414964' - pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "gs://output-directory/v2-artifacts", - "name": "pipeline-root"}, {"default": "pipeline/my-test-pipeline-with-custom-launcher", - "name": "pipeline-name"}], "name": "my-test-pipeline-with-custom-launcher"}' - pipelines.kubeflow.org/v2_pipeline: "true" - labels: - pipelines.kubeflow.org/v2_pipeline: "true" - pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 -spec: - entrypoint: my-test-pipeline-with-custom-launcher - templates: - - name: my-test-pipeline-with-custom-launcher - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - dag: - tasks: - - name: preprocess - template: preprocess - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - - name: train - template: train - dependencies: [preprocess] - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - - {name: preprocess-output_parameter_one, value: '{{tasks.preprocess.outputs.parameters.preprocess-output_parameter_one}}'} - artifacts: - - {name: preprocess-output_dataset_one, from: '{{tasks.preprocess.outputs.artifacts.preprocess-output_dataset_one}}'} - - name: preprocess - container: - args: - - sh - - -c - - |2 - - if ! [ -x "$(command -v pip)" ]; then - python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip - fi - - PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && "$0" "$@" - - sh - - -ec - - | - program_path=$(mktemp -d) - printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - |2+ - - import kfp - from kfp import dsl - from kfp.dsl import * - from typing import * - - def preprocess(uri: str, some_int: int, output_parameter_one: OutputPath(int), - output_dataset_one: OutputPath('Dataset')): - """Dummy Preprocess Step.""" - with open(output_dataset_one, 'w') as f: - f.write('Output dataset') - with open(output_parameter_one, 'w') as f: - f.write("{}".format(1234)) - - - --executor_input - - '{{$}}' - - --function_to_execute - - preprocess - command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST), - --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO), - --container_image, $(KFP_V2_IMAGE), --task_name, preprocess, --pipeline_name, - '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource, - workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name, - $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}', - --enable_caching, $(ENABLE_CACHING), --, some_int=12, uri=uri-to-import, --] - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: {fieldPath: metadata.name} - - name: KFP_POD_UID - valueFrom: - fieldRef: {fieldPath: metadata.uid} - - name: KFP_NAMESPACE - valueFrom: - fieldRef: {fieldPath: metadata.namespace} - - name: WORKFLOW_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'} - - name: KFP_RUN_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'} - - name: ENABLE_CACHING - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'python:3.9'} - - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"some_int": {"type": - "NUMBER_INTEGER"}, "uri": {"type": "STRING"}}, "inputArtifacts": {}, "outputParameters": - {"output_parameter_one": {"type": "NUMBER_INTEGER", "path": "/tmp/outputs/output_parameter_one/data"}}, - "outputArtifacts": {"output_dataset_one": {"schemaTitle": "system.Dataset", - "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_dataset_one/data"}}}'} - envFrom: - - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: python:3.9 - volumeMounts: - - {mountPath: /kfp-launcher, name: kfp-launcher} - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - outputs: - parameters: - - name: preprocess-output_parameter_one - valueFrom: {path: /tmp/outputs/output_parameter_one/data} - artifacts: - - {name: preprocess-output_dataset_one, path: /tmp/outputs/output_dataset_one/data} - - {name: preprocess-output_parameter_one, path: /tmp/outputs/output_parameter_one/data} - metadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/component_ref: '{}' - pipelines.kubeflow.org/arguments.parameters: '{"some_int": "12", "uri": "uri-to-import"}' - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/enable_caching: "true" - initContainers: - - command: [launcher, --copy, /kfp-launcher/launch] - image: my-custom-image - name: kfp-launcher - mirrorVolumeMounts: true - volumes: - - {name: kfp-launcher} - - name: train - container: - args: - - sh - - -c - - |2 - - if ! [ -x "$(command -v pip)" ]; then - python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip - fi - - PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location 'kfp==1.8.6' && "$0" "$@" - - sh - - -ec - - | - program_path=$(mktemp -d) - printf "%s" "$0" > "$program_path/ephemeral_component.py" - python3 -m kfp.components.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - |2+ - - import kfp - from kfp import dsl - from kfp.dsl import * - from typing import * - - def train(dataset: InputPath('Dataset'), - model: OutputPath('Model'), - num_steps: int = 100): - """Dummy Training Step.""" - - with open(dataset, 'r') as input_file: - input_string = input_file.read() - with open(model, 'w') as output_file: - for i in range(num_steps): - output_file.write("Step {}\n{}\n=====\n".format( - i, input_string)) - - - --executor_input - - '{{$}}' - - --function_to_execute - - train - command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST), - --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO), - --container_image, $(KFP_V2_IMAGE), --task_name, train, --pipeline_name, '{{inputs.parameters.pipeline-name}}', - --run_id, $(KFP_RUN_ID), --run_resource, workflows.argoproj.io/$(WORKFLOW_ID), - --namespace, $(KFP_NAMESPACE), --pod_name, $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), - --pipeline_root, '{{inputs.parameters.pipeline-root}}', --enable_caching, - $(ENABLE_CACHING), --, 'num_steps={{inputs.parameters.preprocess-output_parameter_one}}', - --] - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: {fieldPath: metadata.name} - - name: KFP_POD_UID - valueFrom: - fieldRef: {fieldPath: metadata.uid} - - name: KFP_NAMESPACE - valueFrom: - fieldRef: {fieldPath: metadata.namespace} - - name: WORKFLOW_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'} - - name: KFP_RUN_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'} - - name: ENABLE_CACHING - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'python:3.9'} - - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"num_steps": {"type": - "NUMBER_INTEGER"}}, "inputArtifacts": {"dataset": {"metadataPath": "/tmp/inputs/dataset/data", - "schemaTitle": "system.Dataset", "instanceSchema": "", "schemaVersion": - "0.0.1"}}, "outputParameters": {}, "outputArtifacts": {"model": {"schemaTitle": - "system.Model", "instanceSchema": "", "schemaVersion": "0.0.1", "metadataPath": - "/tmp/outputs/model/data"}}}'} - envFrom: - - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: python:3.9 - volumeMounts: - - {mountPath: /kfp-launcher, name: kfp-launcher} - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - - {name: preprocess-output_parameter_one} - artifacts: - - {name: preprocess-output_dataset_one, path: /tmp/inputs/dataset/data} - outputs: - artifacts: - - {name: train-model, path: /tmp/outputs/model/data} - metadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/component_ref: '{}' - pipelines.kubeflow.org/arguments.parameters: '{"num_steps": "{{inputs.parameters.preprocess-output_parameter_one}}"}' - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.6 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/enable_caching: "true" - initContainers: - - command: [launcher, --copy, /kfp-launcher/launch] - image: my-custom-image - name: kfp-launcher - mirrorVolumeMounts: true - volumes: - - {name: kfp-launcher} - arguments: - parameters: - - {name: pipeline-root, value: 'gs://output-directory/v2-artifacts'} - - {name: pipeline-name, value: pipeline/my-test-pipeline-with-custom-launcher} - serviceAccountName: pipeline-runner diff --git a/sdk/python/kfp/deprecated/compiler/v2_compat.py b/sdk/python/kfp/deprecated/compiler/v2_compat.py deleted file mode 100644 index 03af5d7fae9..00000000000 --- a/sdk/python/kfp/deprecated/compiler/v2_compat.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utility functions for enabling v2-compatible pipelines in v1.""" -import collections -import json -from typing import Optional - -from kfp.deprecated import dsl -from kfp.deprecated.compiler import _default_transformers -from kfp.pipeline_spec import pipeline_spec_pb2 -from kubernetes import client as k8s_client - -_DEFAULT_LAUNCHER_IMAGE = "gcr.io/ml-pipeline/kfp-launcher:1.8.7" - - -def update_op(op: dsl.ContainerOp, - pipeline_name: dsl.PipelineParam, - pipeline_root: dsl.PipelineParam, - launcher_image: Optional[str] = None) -> None: - """Updates the passed in Op for running in v2-compatible mode. - - Args: - op: The Op to update. - pipeline_spec: The PipelineSpec for the pipeline under which `op` - runs. - pipeline_root: The root output directory for pipeline artifacts. - launcher_image: An optional launcher image. Useful for tests. - """ - op.is_v2 = True - # Inject the launcher binary and overwrite the entrypoint. - image_name = launcher_image or _DEFAULT_LAUNCHER_IMAGE - launcher_container = dsl.UserContainer( - name="kfp-launcher", - image=image_name, - command=["launcher", "--copy", "/kfp-launcher/launch"], - mirror_volume_mounts=True) - - op.add_init_container(launcher_container) - op.add_volume(k8s_client.V1Volume(name='kfp-launcher')) - op.add_volume_mount( - k8s_client.V1VolumeMount( - name='kfp-launcher', mount_path='/kfp-launcher')) - - # op.command + op.args will have the following sections: - # 1. args passed to kfp-launcher - # 2. a separator "--" - # 3. parameters in format "key1=value1", "key2=value2", ... - # 4. a separator "--" as end of arguments passed to launcher - # 5. (start of op.args) arguments of the original user program command + args - # - # example: - # - command: - # - /kfp-launcher/launch - # - '--mlmd_server_address' - # - $(METADATA_GRPC_SERVICE_HOST) - # - '--mlmd_server_port' - # - $(METADATA_GRPC_SERVICE_PORT) - # - ... # more launcher params - # - '--pipeline_task_id' - # - $(KFP_POD_NAME) - # - '--pipeline_root' - # - '' - # - '--' # start of parameter values - # - first=first - # - second=second - # - '--' # start of user command and args - # args: - # - sh - # - '-ec' - # - | - # program_path=$(mktemp) - # printf "%s" "$0" > "$program_path" - # python3 -u "$program_path" "$@" - # - > - # import json - # import xxx - # ... - op.command = [ - "/kfp-launcher/launch", - "--mlmd_server_address", - "$(METADATA_GRPC_SERVICE_HOST)", - "--mlmd_server_port", - "$(METADATA_GRPC_SERVICE_PORT)", - "--runtime_info_json", - "$(KFP_V2_RUNTIME_INFO)", - "--container_image", - "$(KFP_V2_IMAGE)", - "--task_name", - op.name, - "--pipeline_name", - pipeline_name, - "--run_id", - "$(KFP_RUN_ID)", - "--run_resource", - "workflows.argoproj.io/$(WORKFLOW_ID)", - "--namespace", - "$(KFP_NAMESPACE)", - "--pod_name", - "$(KFP_POD_NAME)", - "--pod_uid", - "$(KFP_POD_UID)", - "--pipeline_root", - pipeline_root, - "--enable_caching", - "$(ENABLE_CACHING)", - ] - - # Mount necessary environment variables. - op.apply(_default_transformers.add_kfp_pod_env) - op.container.add_env_variable( - k8s_client.V1EnvVar(name="KFP_V2_IMAGE", value=op.container.image)) - - config_map_ref = k8s_client.V1ConfigMapEnvSource( - name='metadata-grpc-configmap', optional=True) - op.container.add_env_from( - k8s_client.V1EnvFromSource(config_map_ref=config_map_ref)) - - op.arguments = list(op.container_spec.command) + list( - op.container_spec.args) - - runtime_info = { - "inputParameters": collections.OrderedDict(), - "inputArtifacts": collections.OrderedDict(), - "outputParameters": collections.OrderedDict(), - "outputArtifacts": collections.OrderedDict(), - } - - op.command += ["--"] - component_spec = op.component_spec - for parameter, spec in sorted( - component_spec.input_definitions.parameters.items()): - parameter_type = pipeline_spec_pb2.ParameterType.ParameterTypeEnum.Name( - spec.parameter_type) - parameter_info = {"type": parameter_type} - - parameter_value = op._parameter_arguments[parameter] - op.command += [f"{parameter}={parameter_value}"] - - runtime_info["inputParameters"][parameter] = parameter_info - op.command += ["--"] - - for artifact_name, spec in sorted( - component_spec.input_definitions.artifacts.items()): - artifact_info = { - "metadataPath": op.input_artifact_paths[artifact_name], - "schemaTitle": spec.artifact_type.schema_title, - "instanceSchema": spec.artifact_type.instance_schema, - "schemaVersion": spec.artifact_type.schema_version, - } - runtime_info["inputArtifacts"][artifact_name] = artifact_info - - for parameter, spec in sorted( - component_spec.output_definitions.parameters.items()): - parameter_info = { - "type": - pipeline_spec_pb2.ParameterType.ParameterTypeEnum.Name( - spec.parameter_type), - "path": - op.file_outputs[parameter], - } - runtime_info["outputParameters"][parameter] = parameter_info - - for artifact_name, spec in sorted( - component_spec.output_definitions.artifacts.items()): - # TODO: Assert instance_schema. - artifact_info = { - # Type used to register output artifacts. - "schemaTitle": spec.artifact_type.schema_title, - "instanceSchema": spec.artifact_type.instance_schema, - "schemaVersion": spec.artifact_type.schema_version, - # File used to write out the registered artifact ID. - "metadataPath": op.file_outputs[artifact_name], - } - runtime_info["outputArtifacts"][artifact_name] = artifact_info - - op.container.add_env_variable( - k8s_client.V1EnvVar( - name="KFP_V2_RUNTIME_INFO", value=json.dumps(runtime_info))) - - op.pod_annotations['pipelines.kubeflow.org/v2_component'] = "true" - op.pod_labels['pipelines.kubeflow.org/v2_component'] = "true" diff --git a/sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py b/sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py deleted file mode 100644 index 482f7d7422c..00000000000 --- a/sdk/python/kfp/deprecated/compiler/v2_compatible_compiler_test.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for v2-compatible compiled pipelines.""" - -import json -import os -import re -import tempfile -import unittest -from typing import Callable, Dict - -import yaml -from kfp.deprecated import compiler -from kfp.deprecated import dsl as v1dsl -from kfp import dsl -from kfp.dsl import Artifact, InputPath, OutputPath, component - - -@component -def preprocess(uri: str, some_int: int, output_parameter_one: OutputPath(int), - output_dataset_one: OutputPath('Dataset')): - """Dummy Preprocess Step.""" - with open(output_dataset_one, 'w') as f: - f.write('Output dataset') - with open(output_parameter_one, 'w') as f: - f.write("{}".format(1234)) - - -@component -def train(dataset: InputPath('Dataset'), - model: OutputPath('Model'), - num_steps: int = 100): - """Dummy Training Step.""" - - with open(dataset, 'r') as input_file: - input_string = input_file.read() - with open(model, 'w') as output_file: - for i in range(num_steps): - output_file.write("Step {}\n{}\n=====\n".format( - i, input_string)) - - -@unittest.skip('v2 compatible mode is being deprecated in SDK v2.0') -class TestV2CompatibleModeCompiler(unittest.TestCase): - - def setUp(self) -> None: - self.maxDiff = None - return super().setUp() - - def _ignore_kfp_version_in_template(self, template): - """Ignores kfp sdk versioning in container spec.""" - - def _assert_compiled_pipeline_equals_golden(self, - kfp_compiler: compiler.Compiler, - pipeline_func: Callable, - golden_yaml_filename: str): - compiled_file = os.path.join(tempfile.mkdtemp(), 'workflow.yaml') - kfp_compiler.compile(pipeline_func, package_path=compiled_file) - - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - golden_file = os.path.join(test_data_dir, golden_yaml_filename) - - def _load_compiled_template(filename: str) -> Dict: - with open(filename, 'r') as f: - workflow = yaml.safe_load(f) - - del workflow['metadata'] - for template in workflow['spec']['templates']: - template.pop('metadata', None) - - if 'initContainers' not in template: - continue - # Strip off the launcher image label before comparison - for initContainer in template['initContainers']: - initContainer['image'] = initContainer['image'].split( - ':')[0] - - if 'container' in template: - template['container'] = json.loads( - re.sub(r"'kfp==(\d+).(\d+).(\d+)'", 'kfp', - json.dumps(template['container']))) - - return workflow - - golden = _load_compiled_template(golden_file) - compiled = _load_compiled_template(compiled_file) - - # Devs can run the following command to update golden files: - # UPDATE_GOLDENS=True python3 -m unittest kfp/compiler/v2_compatible_compiler_test.py - # If UPDATE_GOLDENS=True, and the diff is - # different, update the golden file and reload it. - update_goldens = os.environ.get('UPDATE_GOLDENS', False) - if golden != compiled and update_goldens: - kfp_compiler.compile(pipeline_func, package_path=golden_file) - golden = _load_compiled_template(golden_file) - - self.assertDictEqual(golden, compiled) - - def test_two_step_pipeline(self): - - @dsl.pipeline( - pipeline_root='gs://output-directory/v2-artifacts', - name='my-test-pipeline') - def v2_compatible_two_step_pipeline(): - preprocess_task = preprocess(uri='uri-to-import', some_int=12) - train_task = train( - num_steps=preprocess_task.outputs['output_parameter_one'], - dataset=preprocess_task.outputs['output_dataset_one']) - - kfp_compiler = compiler.Compiler( - mode=v1dsl.PipelineExecutionMode.V2_COMPATIBLE) - self._assert_compiled_pipeline_equals_golden( - kfp_compiler, v2_compatible_two_step_pipeline, - 'v2_compatible_two_step_pipeline.yaml') - - def test_custom_launcher(self): - - @dsl.pipeline( - pipeline_root='gs://output-directory/v2-artifacts', - name='my-test-pipeline-with-custom-launcher') - def v2_compatible_two_step_pipeline(): - preprocess_task = preprocess(uri='uri-to-import', some_int=12) - train_task = train( - num_steps=preprocess_task.outputs['output_parameter_one'], - dataset=preprocess_task.outputs['output_dataset_one']) - - kfp_compiler = compiler.Compiler( - mode=v1dsl.PipelineExecutionMode.V2_COMPATIBLE, - launcher_image='my-custom-image') - self._assert_compiled_pipeline_equals_golden( - kfp_compiler, v2_compatible_two_step_pipeline, - 'v2_compatible_two_step_pipeline_with_custom_launcher.yaml') - - def test_constructing_container_op_directly_should_error(self): - - @dsl.pipeline(name='test-pipeline') - def my_pipeline(): - v1dsl.ContainerOp( - name='comp1', - image='gcr.io/dummy', - command=['python', 'main.py']) - - with self.assertRaisesRegex( - RuntimeError, - 'Constructing ContainerOp instances directly is deprecated and not ' - r'supported when compiling to v2 \(using v2 compiler or v1 compiler ' - r'with V2_COMPATIBLE or V2_ENGINE mode\)\.'): - compiler.Compiler( - mode=v1dsl.PipelineExecutionMode.V2_COMPATIBLE).compile( - pipeline_func=my_pipeline, package_path='result.json') - - def test_use_importer_should_error(self): - - @dsl.pipeline(name='test-pipeline') - def my_pipeline(): - dsl.importer(artifact_uri='dummy', artifact_class=Artifact) - - with self.assertRaisesRegex( - ValueError, - 'dsl.importer is not supported with v1 compiler.', - ): - compiler.Compiler( - mode=v1dsl.PipelineExecutionMode.V2_COMPATIBLE).compile( - pipeline_func=my_pipeline, package_path='result.json') - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/components/__init__.py b/sdk/python/kfp/deprecated/components/__init__.py deleted file mode 100644 index f56765ca6b3..00000000000 --- a/sdk/python/kfp/deprecated/components/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ._airflow_op import * -from ._components import * -from ._python_op import * -from ._python_to_graph_component import * -from ._component_store import * diff --git a/sdk/python/kfp/deprecated/components/_airflow_op.py b/sdk/python/kfp/deprecated/components/_airflow_op.py deleted file mode 100644 index 752e23f9062..00000000000 --- a/sdk/python/kfp/deprecated/components/_airflow_op.py +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__all__ = [ - 'create_component_from_airflow_op', -] - -from typing import List - -from ._python_op import _func_to_component_spec, _create_task_factory_from_component_spec - -_default_airflow_base_image = 'apache/airflow:master-python3.6-ci' #TODO: Update a production release image once they become available: https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-10+Multi-layered+and+multi-stage+official+Airflow+CI+image#AIP-10Multi-layeredandmulti-stageofficialAirflowCIimage-ProposedsetupoftheDockerHubandTravisCI . See https://issues.apache.org/jira/browse/AIRFLOW-5093 - - -def create_component_from_airflow_op( - op_class: type, - base_image: str = _default_airflow_base_image, - variable_output_names: List[str] = None, - xcom_output_names: List[str] = None, - modules_to_capture: List[str] = None): - """Creates component function from an Airflow operator class. The inputs of - the component are the same as the operator constructor parameters. By - default the component has the following outputs: "Result", "Variables" and - "XComs". "Variables" and "XComs" are serialized JSON maps of all variables - and xcoms produced by the operator during the execution. Use the - variable_output_names and xcom_output_names parameters to output individual - variables/xcoms as separate outputs. - - Args: - op_class: Reference to the Airflow operator class (e.g. EmailOperator or BashOperator) to convert to componenent. - base_image: Optional. The container image to use for the component. Default is apache/airflow. The container image must have the same python version as the environment used to run create_component_from_airflow_op. The image should have python 3.5+ with airflow package installed. - variable_output_names: Optional. A list of Airflow "variables" produced by the operator that should be returned as separate outputs. - xcom_output_names: Optional. A list of Airflow "XComs" produced by the operator that should be returned as separate outputs. - modules_to_capture: Optional. A list of names of additional modules that the operator depends on. By default only the module containing the operator class is captured. If the operator class uses the code from another module, the name of that module can be specified in this list. - """ - component_spec = _create_component_spec_from_airflow_op( - op_class=op_class, - base_image=base_image, - variables_to_output=variable_output_names, - xcoms_to_output=xcom_output_names, - modules_to_capture=modules_to_capture, - ) - task_factory = _create_task_factory_from_component_spec(component_spec) - return task_factory - - -def _create_component_spec_from_airflow_op( - op_class: type, - base_image: str = _default_airflow_base_image, - result_output_name: str = 'Result', - variables_dict_output_name: str = 'Variables', - xcoms_dict_output_name: str = 'XComs', - variables_to_output: List[str] = None, - xcoms_to_output: List[str] = None, - modules_to_capture: List[str] = None, -): - variables_output_names = variables_to_output or [] - xcoms_output_names = xcoms_to_output or [] - modules_to_capture = modules_to_capture or [op_class.__module__] - modules_to_capture.append(_run_airflow_op.__module__) - - output_names = [] - if result_output_name is not None: - output_names.append(result_output_name) - if variables_dict_output_name is not None: - output_names.append(variables_dict_output_name) - if xcoms_dict_output_name is not None: - output_names.append(xcoms_dict_output_name) - output_names.extend(variables_output_names) - output_names.extend(xcoms_output_names) - - from collections import namedtuple - returnType = namedtuple('AirflowOpOutputs', output_names) - - def _run_airflow_op_closure(*op_args, **op_kwargs) -> returnType: - (result, variables, xcoms) = _run_airflow_op(op_class, *op_args, - **op_kwargs) - - output_values = {} - - import json - if result_output_name is not None: - output_values[result_output_name] = str(result) - if variables_dict_output_name is not None: - output_values[variables_dict_output_name] = json.dumps(variables) - if xcoms_dict_output_name is not None: - output_values[xcoms_dict_output_name] = json.dumps(xcoms) - for name in variables_output_names: - output_values[name] = variables[name] - for name in xcoms_output_names: - output_values[name] = xcoms[name] - - return returnType(**output_values) - - # Hacking the function signature so that correct component interface is generated - import inspect - parameters = inspect.signature(op_class).parameters.values() - #Filtering out `*args` and `**kwargs` parameters that some operators have - parameters = [ - param for param in parameters - if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD - ] - sig = inspect.Signature( - parameters=parameters, - return_annotation=returnType, - ) - _run_airflow_op_closure.__signature__ = sig - _run_airflow_op_closure.__name__ = op_class.__name__ - - return _func_to_component_spec( - _run_airflow_op_closure, - base_image=base_image, - use_code_pickling=True, - modules_to_capture=modules_to_capture) - - -def _run_airflow_op(Op, *op_args, **op_kwargs): - from airflow.utils import db - db.initdb() - - from datetime import datetime - from airflow import DAG, settings - from airflow.models import TaskInstance, Variable, XCom - - dag = DAG(dag_id='anydag', start_date=datetime.now()) - task = Op(*op_args, **op_kwargs, dag=dag, task_id='anytask') - ti = TaskInstance(task=task, execution_date=datetime.now()) - result = task.execute(ti.get_template_context()) - - variables = { - var.id: var.val for var in settings.Session().query(Variable).all() - } - xcoms = {msg.key: msg.value for msg in settings.Session().query(XCom).all()} - return (result, variables, xcoms) diff --git a/sdk/python/kfp/deprecated/components/_component_store.py b/sdk/python/kfp/deprecated/components/_component_store.py deleted file mode 100644 index 747b2730b8b..00000000000 --- a/sdk/python/kfp/deprecated/components/_component_store.py +++ /dev/null @@ -1,395 +0,0 @@ -__all__ = [ - 'ComponentStore', -] - -from pathlib import Path -import copy -import hashlib -import json -import logging -import requests -import tempfile -from typing import Callable, Iterable -from uritemplate import URITemplate -from . import _components as comp -from .structures import ComponentReference -from ._key_value_store import KeyValueStore - -_COMPONENT_FILENAME = 'component.yaml' - - -class ComponentStore: - """Component store. - - Enables external components to be loaded by name and digest/tag. - - Attributes: - - local_search_paths: A list of local directories to include in the search. - url_seach_prefixes: A list of URL prefixes to include in the search. - uri_search_template: A URI template for components, which may include {name}, {digest} and {tag} variables. - """ - - def __init__(self, - local_search_paths=None, - url_search_prefixes=None, - auth=None, - uri_search_template=None): - """Instantiates a ComponentStore including the specified locations. - - Args: - - local_search_paths: A list of local directories to include in the search. - url_seach_prefixes: A list of URL prefixes to include in the search. - auth: Auth object for the requests library. See https://requests.readthedocs.io/en/master/user/authentication/ - uri_search_template: A URI template for components, which may include {name}, {digest} and {tag} variables. - """ - self.local_search_paths = local_search_paths or ['.'] - if uri_search_template: - self.uri_search_template = URITemplate(uri_search_template) - self.url_search_prefixes = url_search_prefixes or [] - self._auth = auth - - self._component_file_name = 'component.yaml' - self._digests_subpath = 'versions/sha256' - self._tags_subpath = 'versions/tags' - - cache_base_dir = Path(tempfile.gettempdir()) / '.kfp_components' - self._git_blob_hash_to_data_db = KeyValueStore( - cache_dir=cache_base_dir / 'git_blob_hash_to_data') - self._url_to_info_db = KeyValueStore(cache_dir=cache_base_dir / - 'url_to_info') - - def load_component_from_url(self, url): - """Loads a component from a URL. - - Args: - url: The url of the component specification. - - Returns: - A factory function with a strongly-typed signature. - """ - return comp.load_component_from_url(url=url, auth=self._auth) - - def load_component_from_file(self, path): - """Loads a component from a path. - - Args: - path: The path of the component specification. - - Returns: - A factory function with a strongly-typed signature. - """ - return comp.load_component_from_file(path) - - def load_component(self, name, digest=None, tag=None): - """Loads component local file or URL and creates a task factory - function. - - Search locations: - - * :code:`//component.yaml` - * :code:`//component.yaml` - - If the digest is specified, then the search locations are: - - * :code:`//versions/sha256/` - * :code:`//versions/sha256/` - - If the tag is specified, then the search locations are: - - * :code:`//versions/tags/` - * :code:`//versions/tags/` - - Args: - name: Component name used to search and load the component artifact containing the component definition. - Component name usually has the following form: group/subgroup/component - digest: Strict component version. SHA256 hash digest of the component artifact file. Can be used to load a specific component version so that the pipeline is reproducible. - tag: Version tag. Can be used to load component version from a specific branch. The version of the component referenced by a tag can change in future. - - Returns: - A factory function with a strongly-typed signature. - Once called with the required arguments, the factory constructs a pipeline task instance (ContainerOp). - """ - #This function should be called load_task_factory since it returns a factory function. - #The real load_component function should produce an object with component properties (e.g. name, description, inputs/outputs). - #TODO: Change this function to return component spec object but it should be callable to construct tasks. - component_ref = ComponentReference(name=name, digest=digest, tag=tag) - component_ref = self._load_component_spec_in_component_ref( - component_ref) - return comp._create_task_factory_from_component_spec( - component_spec=component_ref.spec, - component_ref=component_ref, - ) - - def _load_component_spec_in_component_ref( - self, - component_ref: ComponentReference, - ) -> ComponentReference: - """Takes component_ref, finds the component spec and returns - component_ref with .spec set to the component spec. - - See ComponentStore.load_component for the details of the search - logic. - """ - if component_ref.spec: - return component_ref - - component_ref = copy.copy(component_ref) - if component_ref.url: - component_ref.spec = comp._load_component_spec_from_url( - url=component_ref.url, auth=self._auth) - return component_ref - - name = component_ref.name - if not name: - raise TypeError("name is required") - if name.startswith('/') or name.endswith('/'): - raise ValueError( - 'Component name should not start or end with slash: "{}"' - .format(name)) - - digest = component_ref.digest - tag = component_ref.tag - - tried_locations = [] - - if digest is not None and tag is not None: - raise ValueError('Cannot specify both tag and digest') - - if digest is not None: - path_suffix = name + '/' + self._digests_subpath + '/' + digest - elif tag is not None: - path_suffix = name + '/' + self._tags_subpath + '/' + tag - #TODO: Handle symlinks in GIT URLs - else: - path_suffix = name + '/' + self._component_file_name - - #Trying local search paths - for local_search_path in self.local_search_paths: - component_path = Path(local_search_path, path_suffix) - tried_locations.append(str(component_path)) - if component_path.is_file(): - # TODO: Verify that the content matches the digest (if specified). - component_ref._local_path = str(component_path) - component_ref.spec = comp._load_component_spec_from_file( - str(component_path)) - return component_ref - - #Trying URI template - if self.uri_search_template: - url = self.uri_search_template.expand( - name=name, digest=digest, tag=tag) - tried_locations.append(url) - if self._load_component_spec_from_url(component_ref, url): - return component_ref - - #Trying URL prefixes - for url_search_prefix in self.url_search_prefixes: - url = url_search_prefix + path_suffix - tried_locations.append(url) - if self._load_component_spec_from_url(component_ref, url): - return component_ref - - raise RuntimeError( - 'Component {} was not found. Tried the following locations:\n{}' - .format(name, '\n'.join(tried_locations))) - - def _load_component_spec_from_url(self, component_ref, url) -> bool: - """Loads component spec from a URL. - - On success, the url and spec attributes of the component_ref arg will be populated. - - Args: - component_ref: the component whose spec to load. - url: the location from which to obtain the component spec. - - Returns: - True if the component was retrieved and non-empty; otherwise False. - """ - - try: - response = requests.get( - url, auth=self._auth - ) #Does not throw exceptions on bad status, but throws on dead domains and malformed URLs. Should we log those cases? - response.raise_for_status() - except: - return False - - if response.content: - # TODO: Verify that the content matches the digest (if specified). - component_ref.url = url - component_ref.spec = comp._load_component_spec_from_yaml_or_zip_bytes( - response.content) - return True - - return False - - def _load_component_from_ref(self, - component_ref: ComponentReference) -> Callable: - component_ref = self._load_component_spec_in_component_ref( - component_ref) - return comp._create_task_factory_from_component_spec( - component_spec=component_ref.spec, component_ref=component_ref) - - def search(self, name: str): - """Searches for components by name in the configured component store. - - Prints the component name and URL for components that match the given name. - Only components on GitHub are currently supported. - - Example:: - - kfp.components.ComponentStore.default_store.search('xgboost') - - # Returns results: - # Xgboost train https://raw.githubusercontent.com/.../components/XGBoost/Train/component.yaml - # Xgboost predict https://raw.githubusercontent.com/.../components/XGBoost/Predict/component.yaml - """ - self._refresh_component_cache() - for url in self._url_to_info_db.keys(): - component_info = json.loads( - self._url_to_info_db.try_get_value_bytes(url)) - component_name = component_info['name'] - if name.casefold() in component_name.casefold(): - print('\t'.join([ - component_name, - url, - ])) - - def list(self): - self.search('') - - def _refresh_component_cache(self): - for url_search_prefix in self.url_search_prefixes: - if url_search_prefix.startswith( - 'https://raw.githubusercontent.com/'): - logging.info('Searching for components in "{}"'.format( - url_search_prefix)) - for candidate in _list_candidate_component_uris_from_github_repo( - url_search_prefix, auth=self._auth): - component_url = candidate['url'] - if self._url_to_info_db.exists(component_url): - continue - - logging.debug( - 'Found new component URL: "{}"'.format(component_url)) - - blob_hash = candidate['git_blob_hash'] - if not self._git_blob_hash_to_data_db.exists(blob_hash): - logging.debug( - 'Downloading component spec from "{}"'.format( - component_url)) - response = _get_request_session().get( - component_url, auth=self._auth) - response.raise_for_status() - component_data = response.content - - # Verifying the hash - received_data_hash = _calculate_git_blob_hash( - component_data) - if received_data_hash.lower() != blob_hash.lower(): - raise RuntimeError( - 'The downloaded component ({}) has incorrect hash: "{}" != "{}"' - .format( - component_url, - received_data_hash, - blob_hash, - )) - - # Verifying that the component is loadable - try: - component_spec = comp._load_component_spec_from_component_text( - component_data) - except: - continue - self._git_blob_hash_to_data_db.store_value_bytes( - blob_hash, component_data) - else: - component_data = self._git_blob_hash_to_data_db.try_get_value_bytes( - blob_hash) - component_spec = comp._load_component_spec_from_component_text( - component_data) - - component_name = component_spec.name - self._url_to_info_db.store_value_text( - component_url, - json.dumps( - dict( - name=component_name, - url=component_url, - git_blob_hash=blob_hash, - digest=_calculate_component_digest( - component_data), - ))) - - -def _get_request_session(max_retries: int = 3): - session = requests.Session() - - retry_strategy = requests.packages.urllib3.util.retry.Retry( - total=max_retries, - backoff_factor=0.1, - status_forcelist=[413, 429, 500, 502, 503, 504], - method_whitelist=frozenset(['GET', 'POST']), - ) - - session.mount('https://', - requests.adapters.HTTPAdapter(max_retries=retry_strategy)) - session.mount('http://', - requests.adapters.HTTPAdapter(max_retries=retry_strategy)) - - return session - - -def _calculate_git_blob_hash(data: bytes) -> str: - return hashlib.sha1(b'blob ' + str(len(data)).encode('utf-8') + b'\x00' + - data).hexdigest() - - -def _calculate_component_digest(data: bytes) -> str: - return hashlib.sha256(data.replace(b'\r\n', b'\n')).hexdigest() - - -def _list_candidate_component_uris_from_github_repo(url_search_prefix: str, - auth=None) -> Iterable[str]: - (schema, _, host, org, repo, ref, - path_prefix) = url_search_prefix.split('/', 6) - for page in range(1, 999): - search_url = ( - 'https://api.github.com/search/code?q=filename:{}+repo:{}/{}&page={}&per_page=1000' - ).format(_COMPONENT_FILENAME, org, repo, page) - response = _get_request_session().get(search_url, auth=auth) - response.raise_for_status() - result = response.json() - items = result['items'] - if not items: - break - for item in items: - html_url = item['html_url'] - # Constructing direct content URL - # There is an API (/repos/:owner/:repo/git/blobs/:file_sha) for - # getting the blob content, but it requires decoding the content. - raw_url = html_url.replace( - 'https://github.com/', - 'https://raw.githubusercontent.com/').replace('/blob/', '/', 1) - if not raw_url.endswith(_COMPONENT_FILENAME): - # GitHub matches component_test.yaml when searching for filename:"component.yaml" - continue - result_item = dict( - url=raw_url, - path=item['path'], - git_blob_hash=item['sha'], - ) - yield result_item - - -ComponentStore.default_store = ComponentStore( - local_search_paths=[ - '.', - ], - uri_search_template='https://raw.githubusercontent.com/kubeflow/pipelines/{tag}/components/{name}/component.yaml', - url_search_prefixes=[ - 'https://raw.githubusercontent.com/kubeflow/pipelines/master/components/' - ], -) diff --git a/sdk/python/kfp/deprecated/components/_components.py b/sdk/python/kfp/deprecated/components/_components.py deleted file mode 100644 index 8fbfecbcd44..00000000000 --- a/sdk/python/kfp/deprecated/components/_components.py +++ /dev/null @@ -1,676 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__all__ = [ - 'load_component', - 'load_component_from_text', - 'load_component_from_url', - 'load_component_from_file', -] - -import copy -from collections import OrderedDict -import pathlib -from typing import Any, Callable, List, Mapping, NamedTuple, Sequence, Union -import warnings - -from ._naming import _sanitize_file_name, _sanitize_python_function_name, generate_unique_name_conversion_table -from ._yaml_utils import load_yaml -from .structures import * -from ._data_passing import serialize_value, get_canonical_type_for_type_name - -_default_component_name = 'Component' - - -def load_component(filename=None, url=None, text=None, component_spec=None): - """Loads component from text, file or URL and creates a task factory - function. - - Only one argument should be specified. - - Args: - filename: Path of local file containing the component definition. - url: The URL of the component file data. - text: A string containing the component file data. - component_spec: A ComponentSpec containing the component definition. - - Returns: - A factory function with a strongly-typed signature. - Once called with the required arguments, the factory constructs a pipeline task instance (ContainerOp). - """ - #This function should be called load_task_factory since it returns a factory function. - #The real load_component function should produce an object with component properties (e.g. name, description, inputs/outputs). - #TODO: Change this function to return component spec object but it should be callable to construct tasks. - non_null_args_count = len( - [name for name, value in locals().items() if value != None]) - if non_null_args_count != 1: - raise ValueError('Need to specify exactly one source') - if filename: - return load_component_from_file(filename) - elif url: - return load_component_from_url(url) - elif text: - return load_component_from_text(text) - elif component_spec: - return load_component_from_spec(component_spec) - else: - raise ValueError('Need to specify a source') - - -def load_component_from_url(url: str, auth=None): - """Loads component from URL and creates a task factory function. - - Args: - url: The URL of the component file data - auth: Auth object for the requests library. See https://requests.readthedocs.io/en/master/user/authentication/ - - Returns: - A factory function with a strongly-typed signature. - Once called with the required arguments, the factory constructs a pipeline task instance (ContainerOp). - """ - component_spec = _load_component_spec_from_url(url, auth) - url = _fix_component_uri(url) - component_ref = ComponentReference(url=url) - return _create_task_factory_from_component_spec( - component_spec=component_spec, - component_filename=url, - component_ref=component_ref, - ) - - -def load_component_from_file(filename): - """Loads component from file and creates a task factory function. - - Args: - filename: Path of local file containing the component definition. - - Returns: - A factory function with a strongly-typed signature. - Once called with the required arguments, the factory constructs a pipeline task instance (ContainerOp). - """ - component_spec = _load_component_spec_from_file(path=filename) - return _create_task_factory_from_component_spec( - component_spec=component_spec, - component_filename=filename, - ) - - -def load_component_from_text(text): - """Loads component from text and creates a task factory function. - - Args: - text: A string containing the component file data. - - Returns: - A factory function with a strongly-typed signature. - Once called with the required arguments, the factory constructs a pipeline task instance (ContainerOp). - """ - if text is None: - raise TypeError - component_spec = _load_component_spec_from_component_text(text) - return _create_task_factory_from_component_spec( - component_spec=component_spec) - - -def load_component_from_spec(component_spec): - """Loads component from a ComponentSpec and creates a task factory - function. - - Args: - component_spec: A ComponentSpec containing the component definition. - - Returns: - A factory function with a strongly-typed signature. - Once called with the required arguments, the factory constructs a pipeline task instance (ContainerOp). - """ - if component_spec is None: - raise TypeError - return _create_task_factory_from_component_spec( - component_spec=component_spec) - - -def _fix_component_uri(uri: str) -> str: - #Handling Google Cloud Storage URIs - if uri.startswith('gs://'): - #Replacing the gs:// URI with https:// URI (works for public objects) - uri = 'https://storage.googleapis.com/' + uri[len('gs://'):] - return uri - - -def _load_component_spec_from_file(path) -> ComponentSpec: - with open(path, 'rb') as component_stream: - return _load_component_spec_from_yaml_or_zip_bytes( - component_stream.read()) - - -def _load_component_spec_from_url(url: str, auth=None): - if url is None: - raise TypeError - - url = _fix_component_uri(url) - - import requests - resp = requests.get(url, auth=auth) - resp.raise_for_status() - return _load_component_spec_from_yaml_or_zip_bytes(resp.content) - - -_COMPONENT_FILE_NAME_IN_ARCHIVE = 'component.yaml' - - -def _load_component_spec_from_yaml_or_zip_bytes(data: bytes): - """Loads component spec from binary data. - - The data can be a YAML file or a zip file with a component.yaml file - inside. - """ - import zipfile - import io - stream = io.BytesIO(data) - if zipfile.is_zipfile(stream): - stream.seek(0) - with zipfile.ZipFile(stream) as zip_obj: - data = zip_obj.read(_COMPONENT_FILE_NAME_IN_ARCHIVE) - return _load_component_spec_from_component_text(data) - - -def _load_component_spec_from_component_text(text) -> ComponentSpec: - component_dict = load_yaml(text) - component_spec = ComponentSpec.from_dict(component_dict) - - if isinstance(component_spec.implementation, ContainerImplementation) and ( - component_spec.implementation.container.command is None): - warnings.warn( - 'Container component must specify command to be compatible with KFP ' - 'v2 compatible mode and emissary executor, which will be the default' - ' executor for KFP v2.' - 'https://www.kubeflow.org/docs/components/pipelines/installation/choose-executor/', - category=FutureWarning, - ) - - # Calculating hash digest for the component - import hashlib - data = text if isinstance(text, bytes) else text.encode('utf-8') - data = data.replace(b'\r\n', b'\n') # Normalizing line endings - digest = hashlib.sha256(data).hexdigest() - component_spec._digest = digest - - return component_spec - - -_inputs_dir = '/tmp/inputs' -_outputs_dir = '/tmp/outputs' -_single_io_file_name = 'data' - - -def _generate_input_file_name(port_name): - return str( - pathlib.PurePosixPath(_inputs_dir, _sanitize_file_name(port_name), - _single_io_file_name)) - - -def _generate_output_file_name(port_name): - return str( - pathlib.PurePosixPath(_outputs_dir, _sanitize_file_name(port_name), - _single_io_file_name)) - - -def _react_to_incompatible_reference_type( - input_type, - argument_type, - input_name: str, -): - """Raises error for the case when the argument type is incompatible with - the input type.""" - message = 'Argument with type "{}" was passed to the input "{}" that has type "{}".'.format( - argument_type, input_name, input_type) - raise TypeError(message) - - -def _create_task_spec_from_component_and_arguments( - component_spec: ComponentSpec, - arguments: Mapping[str, Any], - component_ref: ComponentReference = None, - **kwargs) -> TaskSpec: - """Constructs a TaskSpec object from component reference and arguments. - - The function also checks the arguments types and serializes them. - """ - if component_ref is None: - component_ref = ComponentReference(spec=component_spec) - else: - component_ref = copy.copy(component_ref) - component_ref.spec = component_spec - - # Not checking for missing or extra arguments since the dynamic factory function checks that - task_arguments = {} - for input_name, argument_value in arguments.items(): - input_type = component_spec._inputs_dict[input_name].type - - if isinstance(argument_value, (GraphInputArgument, TaskOutputArgument)): - # argument_value is a reference - if isinstance(argument_value, GraphInputArgument): - reference_type = argument_value.graph_input.type - elif isinstance(argument_value, TaskOutputArgument): - reference_type = argument_value.task_output.type - else: - reference_type = None - - if reference_type and input_type and reference_type != input_type: - _react_to_incompatible_reference_type(input_type, - reference_type, - input_name) - - task_arguments[input_name] = argument_value - else: - # argument_value is a constant value - serialized_argument_value = serialize_value(argument_value, - input_type) - task_arguments[input_name] = serialized_argument_value - - task = TaskSpec( - component_ref=component_ref, - arguments=task_arguments, - ) - task._init_outputs() - - return task - - -_default_container_task_constructor = _create_task_spec_from_component_and_arguments - -# Holds the function that constructs a task object based on ComponentSpec, arguments and ComponentReference. -# Framework authors can override this constructor function to construct different framework-specific task-like objects. -# The task object should have the task.outputs dictionary with keys corresponding to the ComponentSpec outputs. -# The default constructor creates and instance of the TaskSpec class. -_container_task_constructor = _default_container_task_constructor - -_always_expand_graph_components = False - - -def _create_task_object_from_component_and_arguments( - component_spec: ComponentSpec, - arguments: Mapping[str, Any], - component_ref: ComponentReference = None, - **kwargs): - """Creates a task object from component and argument. - - Unlike _container_task_constructor, handles the graph components as - well. - """ - if (isinstance(component_spec.implementation, GraphImplementation) and ( - # When the container task constructor is not overriden, we just construct TaskSpec for both container and graph tasks. - # If the container task constructor is overriden, we should expand the graph components so that the override is called for all sub-tasks. - _container_task_constructor != _default_container_task_constructor - or _always_expand_graph_components)): - return _resolve_graph_task( - component_spec=component_spec, - arguments=arguments, - component_ref=component_ref, - **kwargs, - ) - - task = _container_task_constructor( - component_spec=component_spec, - arguments=arguments, - component_ref=component_ref, - **kwargs, - ) - - return task - - -class _DefaultValue: - - def __init__(self, value): - self.value = value - - def __repr__(self): - return repr(self.value) - - -#TODO: Refactor the function to make it shorter -def _create_task_factory_from_component_spec( - component_spec: ComponentSpec, - component_filename=None, - component_ref: ComponentReference = None): - name = component_spec.name or _default_component_name - - func_docstring_lines = [] - if component_spec.name: - func_docstring_lines.append(component_spec.name) - if component_spec.description: - func_docstring_lines.append(component_spec.description) - - inputs_list = component_spec.inputs or [] #List[InputSpec] - input_names = [input.name for input in inputs_list] - - #Creating the name translation tables : Original <-> Pythonic - input_name_to_pythonic = generate_unique_name_conversion_table( - input_names, _sanitize_python_function_name) - pythonic_name_to_input_name = { - v: k for k, v in input_name_to_pythonic.items() - } - - if component_ref is None: - component_ref = ComponentReference( - spec=component_spec, url=component_filename) - else: - component_ref.spec = component_spec - - digest = getattr(component_spec, '_digest', None) - # TODO: Calculate the digest if missing - if digest: - # TODO: Report possible digest conflicts - component_ref.digest = digest - - def create_task_object_from_component_and_pythonic_arguments( - pythonic_arguments): - arguments = { - pythonic_name_to_input_name[argument_name]: argument_value - for argument_name, argument_value in pythonic_arguments.items() - if not isinstance( - argument_value, _DefaultValue - ) # Skipping passing arguments for optional values that have not been overridden. - } - return _create_task_object_from_component_and_arguments( - component_spec=component_spec, - arguments=arguments, - component_ref=component_ref, - ) - - import inspect - from . import _dynamic - - #Reordering the inputs since in Python optional parameters must come after required parameters - reordered_input_list = [ - input for input in inputs_list - if input.default is None and not input.optional - ] + [ - input for input in inputs_list - if not (input.default is None and not input.optional) - ] - - def component_default_to_func_default(component_default: str, - is_optional: bool): - if is_optional: - return _DefaultValue(component_default) - if component_default is not None: - return component_default - return inspect.Parameter.empty - - input_parameters = [ - _dynamic.KwParameter( - input_name_to_pythonic[port.name], - annotation=(get_canonical_type_for_type_name(str(port.type)) or str( - port.type) if port.type else inspect.Parameter.empty), - default=component_default_to_func_default(port.default, - port.optional), - ) for port in reordered_input_list - ] - factory_function_parameters = input_parameters #Outputs are no longer part of the task factory function signature. The paths are always generated by the system. - - task_factory = _dynamic.create_function_from_parameters( - create_task_object_from_component_and_pythonic_arguments, - factory_function_parameters, - documentation='\n'.join(func_docstring_lines), - func_name=name, - func_filename=component_filename if - (component_filename and - (component_filename.endswith('.yaml') or - component_filename.endswith('.yml'))) else None, - ) - task_factory.component_spec = component_spec - return task_factory - - -_ResolvedCommandLineAndPaths = NamedTuple( - '_ResolvedCommandLineAndPaths', - [ - ('command', Sequence[str]), - ('args', Sequence[str]), - ('input_paths', Mapping[str, str]), - ('output_paths', Mapping[str, str]), - ('inputs_consumed_by_value', Mapping[str, str]), - ], -) - - -def _resolve_command_line_and_paths( - component_spec: ComponentSpec, - arguments: Mapping[str, str], - input_path_generator: Callable[[str], str] = _generate_input_file_name, - output_path_generator: Callable[[str], str] = _generate_output_file_name, - argument_serializer: Callable[[str], str] = serialize_value, - placeholder_resolver: Callable[[Any, ComponentSpec, Mapping[str, str]], - str] = None, -) -> _ResolvedCommandLineAndPaths: - """Resolves the command line argument placeholders. - - Also produces the maps of the generated inpuit/output paths. - """ - argument_values = arguments - - if not isinstance(component_spec.implementation, ContainerImplementation): - raise TypeError( - 'Only container components have command line to resolve') - - inputs_dict = { - input_spec.name: input_spec - for input_spec in component_spec.inputs or [] - } - container_spec = component_spec.implementation.container - - output_paths = OrderedDict( - ) #Preserving the order to make the kubernetes output names deterministic - unconfigurable_output_paths = container_spec.file_outputs or {} - for output in component_spec.outputs or []: - if output.name in unconfigurable_output_paths: - output_paths[output.name] = unconfigurable_output_paths[output.name] - - input_paths = OrderedDict() - inputs_consumed_by_value = {} - - def expand_command_part(arg) -> Union[str, List[str], None]: - if arg is None: - return None - if placeholder_resolver: - resolved_arg = placeholder_resolver( - arg=arg, - component_spec=component_spec, - arguments=arguments, - ) - if resolved_arg is not None: - return resolved_arg - if isinstance(arg, (str, int, float, bool)): - return str(arg) - if isinstance(arg, InputValuePlaceholder): - input_name = arg.input_name - input_spec = inputs_dict[input_name] - input_value = argument_values.get(input_name, None) - if input_value is not None: - serialized_argument = argument_serializer( - input_value, input_spec.type) - inputs_consumed_by_value[input_name] = serialized_argument - return serialized_argument - else: - if input_spec.optional: - return None - else: - raise ValueError( - 'No value provided for input {}'.format(input_name)) - - if isinstance(arg, InputPathPlaceholder): - input_name = arg.input_name - input_value = argument_values.get(input_name, None) - if input_value is not None: - input_path = input_path_generator(input_name) - input_paths[input_name] = input_path - return input_path - else: - input_spec = inputs_dict[input_name] - if input_spec.optional: - #Even when we support default values there is no need to check for a default here. - #In current execution flow (called by python task factory), the missing argument would be replaced with the default value by python itself. - return None - else: - raise ValueError( - 'No value provided for input {}'.format(input_name)) - - elif isinstance(arg, OutputPathPlaceholder): - output_name = arg.output_name - output_filename = output_path_generator(output_name) - if arg.output_name in output_paths: - if output_paths[output_name] != output_filename: - raise ValueError( - 'Conflicting output files specified for port {}: {} and {}' - .format(output_name, output_paths[output_name], - output_filename)) - else: - output_paths[output_name] = output_filename - - return output_filename - elif isinstance(arg, ConcatPlaceholder): - expanded_argument_strings = expand_argument_list(arg.items) - return ''.join(expanded_argument_strings) - - elif isinstance(arg, IfPlaceholder): - arg = arg.if_structure - condition_result = expand_command_part(arg.condition) - from distutils.util import strtobool - condition_result_bool = condition_result and strtobool( - condition_result - ) #Python gotcha: bool('False') == True; Need to use strtobool; Also need to handle None and [] - result_node = arg.then_value if condition_result_bool else arg.else_value - if result_node is None: - return [] - if isinstance(result_node, list): - expanded_result = expand_argument_list(result_node) - else: - expanded_result = expand_command_part(result_node) - return expanded_result - - elif isinstance(arg, IsPresentPlaceholder): - argument_is_present = argument_values.get(arg.input_name, - None) is not None - return str(argument_is_present) - else: - raise TypeError('Unrecognized argument type: {}'.format(arg)) - - def expand_argument_list(argument_list): - expanded_list = [] - if argument_list is not None: - for part in argument_list: - expanded_part = expand_command_part(part) - if expanded_part is not None: - if isinstance(expanded_part, list): - expanded_list.extend(expanded_part) - else: - expanded_list.append(str(expanded_part)) - return expanded_list - - expanded_command = expand_argument_list(container_spec.command) - expanded_args = expand_argument_list(container_spec.args) - - return _ResolvedCommandLineAndPaths( - command=expanded_command, - args=expanded_args, - input_paths=input_paths, - output_paths=output_paths, - inputs_consumed_by_value=inputs_consumed_by_value, - ) - - -_ResolvedGraphTask = NamedTuple( - '_ResolvedGraphTask', - [ - ('component_spec', ComponentSpec), - ('component_ref', ComponentReference), - ('outputs', Mapping[str, Any]), - ('task_arguments', Mapping[str, Any]), - ], -) - - -def _resolve_graph_task( - component_spec: ComponentSpec, - arguments: Mapping[str, Any], - component_ref: ComponentReference = None, -) -> TaskSpec: - from ..components import ComponentStore - component_store = ComponentStore.default_store - - graph = component_spec.implementation.graph - - graph_input_arguments = { - input.name: input.default - for input in component_spec.inputs or [] - if input.default is not None - } - graph_input_arguments.update(arguments) - - outputs_of_tasks = {} - - def resolve_argument(argument): - if isinstance(argument, (str, int, float, bool)): - return argument - elif isinstance(argument, GraphInputArgument): - return graph_input_arguments[argument.graph_input.input_name] - elif isinstance(argument, TaskOutputArgument): - upstream_task_output_ref = argument.task_output - upstream_task_outputs = outputs_of_tasks[ - upstream_task_output_ref.task_id] - upstream_task_output = upstream_task_outputs[ - upstream_task_output_ref.output_name] - return upstream_task_output - else: - raise TypeError( - 'Argument for input has unexpected type "{}".'.format( - type(argument))) - - for task_id, task_spec in graph._toposorted_tasks.items( - ): # Cannot use graph.tasks here since they might be listed not in dependency order. Especially on python <3.6 where the dicts do not preserve ordering - resolved_task_component_ref = component_store._load_component_spec_in_component_ref( - task_spec.component_ref) - # TODO: Handle the case when optional graph component input is passed to optional task component input - task_arguments = { - input_name: resolve_argument(argument) - for input_name, argument in task_spec.arguments.items() - } - task_component_spec = resolved_task_component_ref.spec - - task_obj = _create_task_object_from_component_and_arguments( - component_spec=task_component_spec, - arguments=task_arguments, - component_ref=task_spec.component_ref, - ) - task_outputs_with_original_names = { - output.name: task_obj.outputs[output.name] - for output in task_component_spec.outputs or [] - } - outputs_of_tasks[task_id] = task_outputs_with_original_names - - resolved_graph_outputs = OrderedDict([ - (output_name, resolve_argument(argument)) - for output_name, argument in graph.output_values.items() - ]) - - # For resolved graph component tasks task.outputs point to the actual tasks that originally produced the output that is later returned from the graph - graph_task = _ResolvedGraphTask( - component_ref=component_ref, - component_spec=component_spec, - outputs=resolved_graph_outputs, - task_arguments=arguments, - ) - return graph_task diff --git a/sdk/python/kfp/deprecated/components/_data_passing.py b/sdk/python/kfp/deprecated/components/_data_passing.py deleted file mode 100644 index 285cfc05148..00000000000 --- a/sdk/python/kfp/deprecated/components/_data_passing.py +++ /dev/null @@ -1,252 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__all__ = [ - 'get_canonical_type_name_for_type', - 'get_canonical_type_for_type_name', - 'get_deserializer_code_for_type_name', - 'get_serializer_func_for_type_name', -] - -import inspect -from typing import Any, Callable, NamedTuple, Optional, Sequence, Type -import warnings - -from kfp.deprecated.components import type_annotation_utils - -Converter = NamedTuple('Converter', [ - ('types', Sequence[Type]), - ('type_names', Sequence[str]), - ('serializer', Callable[[Any], str]), - ('deserializer_code', str), - ('definitions', str), -]) - - -def _serialize_str(str_value: str) -> str: - if not isinstance(str_value, str): - raise TypeError('Value "{}" has type "{}" instead of str.'.format( - str(str_value), str(type(str_value)))) - return str_value - - -def _serialize_int(int_value: int) -> str: - if isinstance(int_value, str): - return int_value - if not isinstance(int_value, int): - raise TypeError('Value "{}" has type "{}" instead of int.'.format( - str(int_value), str(type(int_value)))) - return str(int_value) - - -def _serialize_float(float_value: float) -> str: - if isinstance(float_value, str): - return float_value - if not isinstance(float_value, (float, int)): - raise TypeError('Value "{}" has type "{}" instead of float.'.format( - str(float_value), str(type(float_value)))) - return str(float_value) - - -def _serialize_bool(bool_value: bool) -> str: - if isinstance(bool_value, str): - return bool_value - if not isinstance(bool_value, bool): - raise TypeError('Value "{}" has type "{}" instead of bool.'.format( - str(bool_value), str(type(bool_value)))) - return str(bool_value) - - -def _deserialize_bool(s) -> bool: - from distutils.util import strtobool - return strtobool(s) == 1 - - -_bool_deserializer_definitions = inspect.getsource(_deserialize_bool) -_bool_deserializer_code = _deserialize_bool.__name__ - - -def _serialize_json(obj) -> str: - if isinstance(obj, str): - return obj - import json - - def default_serializer(obj): - if hasattr(obj, 'to_struct'): - return obj.to_struct() - else: - raise TypeError( - "Object of type '%s' is not JSON serializable and does not have .to_struct() method." - % obj.__class__.__name__) - - return json.dumps(obj, default=default_serializer, sort_keys=True) - - -def _serialize_base64_pickle(obj) -> str: - if isinstance(obj, str): - return obj - import base64 - import pickle - return base64.b64encode(pickle.dumps(obj)).decode('ascii') - - -def _deserialize_base64_pickle(s): - import base64 - import pickle - return pickle.loads(base64.b64decode(s)) - - -_deserialize_base64_pickle_definitions = inspect.getsource( - _deserialize_base64_pickle) -_deserialize_base64_pickle_code = _deserialize_base64_pickle.__name__ - -_converters = [ - Converter([str], ['String', 'str'], _serialize_str, 'str', None), - Converter([int], ['Integer', 'int'], _serialize_int, 'int', None), - Converter([float], ['Float', 'float'], _serialize_float, 'float', None), - Converter([bool], ['Boolean', 'Bool', 'bool'], _serialize_bool, - _bool_deserializer_code, _bool_deserializer_definitions), - Converter( - [list], ['JsonArray', 'List', 'list'], _serialize_json, 'json.loads', - 'import json' - ), # ! JSON map keys are always strings. Python converts all keys to strings without warnings - Converter( - [dict], ['JsonObject', 'Dictionary', 'Dict', 'dict'], _serialize_json, - 'json.loads', 'import json' - ), # ! JSON map keys are always strings. Python converts all keys to strings without warnings - Converter([], ['Json'], _serialize_json, 'json.loads', 'import json'), - Converter([], ['Base64Pickle'], _serialize_base64_pickle, - _deserialize_base64_pickle_code, - _deserialize_base64_pickle_definitions), -] - -type_to_type_name = { - typ: converter.type_names[0] for converter in _converters - for typ in converter.types -} -type_name_to_type = { - type_name: converter.types[0] for converter in _converters - for type_name in converter.type_names - if converter.types -} -type_to_deserializer = { - typ: (converter.deserializer_code, converter.definitions) - for converter in _converters for typ in converter.types -} -type_name_to_deserializer = { - type_name: (converter.deserializer_code, converter.definitions) - for converter in _converters for type_name in converter.type_names -} -type_name_to_serializer = { - type_name: converter.serializer for converter in _converters - for type_name in converter.type_names -} - - -def get_canonical_type_name_for_type(typ: Type) -> str: - """Find the canonical type name for a given type. - - Args: - typ: The type to search for. - - Returns: - The canonical name of the type found. - """ - try: - return type_to_type_name.get(typ, None) - except: - return None - - -def get_canonical_type_for_type_name(type_name: str) -> Optional[Type]: - """Find the canonical type for a given type name. - - Args: - type_name: The type name to search for. - - Returns: - The canonical type found. - """ - try: - return type_name_to_type.get(type_name, None) - except: - return None - - -def get_deserializer_code_for_type_name(type_name: str) -> Optional[str]: - """Find the deserializer code for the given type name. - - Args: - type_name: The type name to search for. - - Returns: - The deserializer code needed to deserialize the type. - """ - try: - return type_name_to_deserializer.get( - type_annotation_utils.get_short_type_name(type_name), None) - except: - return None - - -def get_serializer_func_for_type_name(type_name: str) -> Optional[Callable]: - """Find the serializer code for the given type name. - - Args: - type_name: The type name to search for. - - Returns: - The serializer func needed to serialize the type. - """ - try: - return type_name_to_serializer.get( - type_annotation_utils.get_short_type_name(type_name), None) - except: - return None - - -def serialize_value(value, type_name: str) -> str: - """serialize_value converts the passed value to string based on the - serializer associated with the passed type_name.""" - if isinstance(value, str): - return value # The value is supposedly already serialized - - if type_name is None: - type_name = type_to_type_name.get(type(value), type(value).__name__) - warnings.warn( - 'Missing type name was inferred as "{}" based on the value "{}".' - .format(type_name, str(value))) - - serializer = type_name_to_serializer.get( - type_annotation_utils.get_short_type_name(type_name)) - if serializer: - try: - serialized_value = serializer(value) - if not isinstance(serialized_value, str): - raise TypeError( - 'Serializer {} returned result of type "{}" instead of string.' - .format(serializer, type(serialized_value))) - return serialized_value - except Exception as e: - raise ValueError( - 'Failed to serialize the value "{}" of type "{}" to type "{}". Exception: {}' - .format( - str(value), - str(type(value).__name__), - str(type_name), - str(e), - )) - - raise TypeError('There are no registered serializers for type "{}".'.format( - str(type_name),)) diff --git a/sdk/python/kfp/deprecated/components/_dynamic.py b/sdk/python/kfp/deprecated/components/_dynamic.py deleted file mode 100644 index 3ab93da0230..00000000000 --- a/sdk/python/kfp/deprecated/components/_dynamic.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import types -from typing import Any, Callable, Mapping, Sequence -from inspect import Parameter, Signature - - -class KwParameter(Parameter): - - def __init__(self, - name, - default=Parameter.empty, - annotation=Parameter.empty): - super().__init__( - name, - Parameter.POSITIONAL_OR_KEYWORD, - default=default, - annotation=annotation) - - -def create_function_from_parameter_names(func: Callable[[Mapping[str, Any]], - Any], - parameter_names: Sequence[str], - documentation=None, - func_name=None, - func_filename=None): - return create_function_from_parameters( - func, [KwParameter(name) for name in parameter_names], documentation, - func_name, func_filename) - - -def create_function_from_parameters(func: Callable[[Mapping[str, Any]], Any], - parameters: Sequence[Parameter], - documentation=None, - func_name=None, - func_filename=None): - new_signature = Signature(parameters) # Checks the parameter consistency - - def pass_locals(): - return dict_func(locals()) # noqa: F821 TODO - - code = pass_locals.__code__ - mod_co_argcount = len(parameters) - mod_co_nlocals = len(parameters) - mod_co_varnames = tuple(param.name for param in parameters) - mod_co_name = func_name or code.co_name - if func_filename: - mod_co_filename = func_filename - mod_co_firstlineno = 1 - else: - mod_co_filename = code.co_filename - mod_co_firstlineno = code.co_firstlineno - - if sys.version_info >= (3, 8): - modified_code = code.replace( - co_argcount=mod_co_argcount, - co_nlocals=mod_co_nlocals, - co_varnames=mod_co_varnames, - co_filename=mod_co_filename, - co_name=mod_co_name, - co_firstlineno=mod_co_firstlineno, - ) - else: - modified_code = types.CodeType( - mod_co_argcount, code.co_kwonlyargcount, mod_co_nlocals, - code.co_stacksize, code.co_flags, code.co_code, code.co_consts, - code.co_names, mod_co_varnames, mod_co_filename, mod_co_name, - mod_co_firstlineno, code.co_lnotab) - - default_arg_values = tuple( - p.default for p in parameters if p.default != Parameter.empty - ) #!argdefs "starts from the right"/"is right-aligned" - modified_func = types.FunctionType( - modified_code, { - 'dict_func': func, - 'locals': locals - }, - name=func_name, - argdefs=default_arg_values) - modified_func.__doc__ = documentation - modified_func.__signature__ = new_signature - - return modified_func diff --git a/sdk/python/kfp/deprecated/components/_key_value_store.py b/sdk/python/kfp/deprecated/components/_key_value_store.py deleted file mode 100644 index ede146c6af3..00000000000 --- a/sdk/python/kfp/deprecated/components/_key_value_store.py +++ /dev/null @@ -1,68 +0,0 @@ -import hashlib -from pathlib import Path - - -class KeyValueStore: - KEY_FILE_SUFFIX = '.key' - VALUE_FILE_SUFFIX = '.value' - - def __init__( - self, - cache_dir: str, - ): - cache_dir = Path(cache_dir) - hash_func = ( - lambda text: hashlib.sha256(text.encode('utf-8')).hexdigest()) - self.cache_dir = cache_dir - self.hash_func = hash_func - - def store_value_text(self, key: str, text: str) -> str: - return self.store_value_bytes(key, text.encode('utf-8')) - - def store_value_bytes(self, key: str, data: bytes) -> str: - cache_id = self.hash_func(key) - self.cache_dir.mkdir(parents=True, exist_ok=True) - cache_key_file_path = self.cache_dir / ( - cache_id + KeyValueStore.KEY_FILE_SUFFIX) - cache_value_file_path = self.cache_dir / ( - cache_id + KeyValueStore.VALUE_FILE_SUFFIX) - if cache_key_file_path.exists(): - old_key = cache_key_file_path.read_text() - if key != old_key: - raise RuntimeError( - 'Cache is corrupted: File "{}" contains existing key ' - '"{}" != new key "{}"'.format(cache_key_file_path, old_key, - key)) - if cache_value_file_path.exists(): - old_data = cache_value_file_path.read_bytes() - if data != old_data: - # TODO: Add options to raise error when overwriting the value. - pass - cache_value_file_path.write_bytes(data) - cache_key_file_path.write_text(key) - return cache_id - - def try_get_value_text(self, key: str) -> str: - result = self.try_get_value_bytes(key) - if result is None: - return None - return result.decode('utf-8') - - def try_get_value_bytes(self, key: str) -> bytes: - cache_id = self.hash_func(key) - cache_value_file_path = self.cache_dir / ( - cache_id + KeyValueStore.VALUE_FILE_SUFFIX) - if cache_value_file_path.exists(): - return cache_value_file_path.read_bytes() - return None - - def exists(self, key: str) -> bool: - cache_id = self.hash_func(key) - cache_key_file_path = self.cache_dir / ( - cache_id + KeyValueStore.KEY_FILE_SUFFIX) - return cache_key_file_path.exists() - - def keys(self): - for cache_key_file_path in self.cache_dir.glob( - '*' + KeyValueStore.KEY_FILE_SUFFIX): - yield Path(cache_key_file_path).read_text() diff --git a/sdk/python/kfp/deprecated/components/_naming.py b/sdk/python/kfp/deprecated/components/_naming.py deleted file mode 100644 index 1d119b2221d..00000000000 --- a/sdk/python/kfp/deprecated/components/_naming.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__all__ = [ - '_normalize_identifier_name', - '_sanitize_kubernetes_resource_name', - '_sanitize_python_function_name', - '_sanitize_file_name', - '_convert_to_human_name', - '_generate_unique_suffix', - '_make_name_unique_by_adding_index', - '_convert_name_and_make_it_unique_by_adding_number', - 'generate_unique_name_conversion_table', -] - -import sys -from typing import Callable, Sequence, Mapping - - -def _normalize_identifier_name(name): - import re - normalized_name = name.lower() - normalized_name = re.sub(r'[\W_]', ' ', - normalized_name) #No non-word characters - normalized_name = re.sub( - ' +', ' ', - normalized_name).strip() #No double spaces, leading or trailing spaces - if re.match(r'\d', normalized_name): - normalized_name = 'n' + normalized_name #No leading digits - return normalized_name - - -def _sanitize_kubernetes_resource_name(name): - return _normalize_identifier_name(name).replace(' ', '-') - - -def _sanitize_python_function_name(name): - return _normalize_identifier_name(name).replace(' ', '_') - - -def _sanitize_file_name(name): - import re - return re.sub('[^-_.0-9a-zA-Z]+', '_', name) - - -def _convert_to_human_name(name: str): - """Converts underscore or dash delimited name to space-delimited name that - starts with a capital letter. - - Does not handle "camelCase" names. - """ - return name.replace('_', ' ').replace('-', ' ').strip().capitalize() - - -def _generate_unique_suffix(data): - import time - import hashlib - string_data = str((data, time.time())) - return hashlib.sha256(string_data.encode()).hexdigest()[0:8] - - -def _make_name_unique_by_adding_index(name: str, collection, delimiter: str): - unique_name = name - if unique_name in collection: - for i in range(2, sys.maxsize**10): - unique_name = name + delimiter + str(i) - if unique_name not in collection: - break - return unique_name - - -def _convert_name_and_make_it_unique_by_adding_number( - name: str, used_converted_names, conversion_func: Callable[[str], str]): - converted_name = conversion_func(name) - if converted_name in used_converted_names: - for i in range( - 2, sys.maxsize** - 10): #Starting indices from 2: "Something", "Something_2", ... - converted_name = conversion_func(name + ' ' + str(i)) - if converted_name not in used_converted_names: - break - return converted_name - - -def generate_unique_name_conversion_table( - names: Sequence[str], - conversion_func: Callable[[str], str]) -> Mapping[str, str]: - """Given a list of names and conversion_func, this function generates a map - from original names to converted names that are made unique by adding - numbers.""" - forward_map = {} - reverse_map = {} - - # Names that do not change when applying the conversion_func should remain unchanged in the table - names_that_need_conversion = [] - for name in names: - if conversion_func(name) == name: - forward_map[name] = name - reverse_map[name] = name - else: - names_that_need_conversion.append(name) - - for name in names_that_need_conversion: - if name in forward_map: - raise ValueError('Original name {} is not unique.'.format(name)) - converted_name = _convert_name_and_make_it_unique_by_adding_number( - name, reverse_map, conversion_func) - forward_map[name] = converted_name - reverse_map[converted_name] = name - return forward_map diff --git a/sdk/python/kfp/deprecated/components/_python_op.py b/sdk/python/kfp/deprecated/components/_python_op.py deleted file mode 100644 index d24924e0cac..00000000000 --- a/sdk/python/kfp/deprecated/components/_python_op.py +++ /dev/null @@ -1,1123 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__all__ = [ - 'create_component_from_func', - 'create_component_from_func_v2', - 'func_to_container_op', - 'func_to_component_text', - 'default_base_image_or_builder', - 'InputArtifact', - 'InputPath', - 'InputTextFile', - 'InputBinaryFile', - 'OutputArtifact', - 'OutputPath', - 'OutputTextFile', - 'OutputBinaryFile', -] - -from ._yaml_utils import dump_yaml -from ._components import _create_task_factory_from_component_spec -from ._data_passing import serialize_value, get_deserializer_code_for_type_name, get_serializer_func_for_type_name, get_canonical_type_name_for_type -from ._naming import _make_name_unique_by_adding_index -from .structures import * -from . import _structures as structures -from kfp.deprecated.components import type_annotation_utils - -import inspect -import itertools -from pathlib import Path -import textwrap -from typing import Callable, List, Mapping, Optional, TypeVar -import warnings - -import docstring_parser - -T = TypeVar('T') - -# InputPath(list) or InputPath('JsonObject') - - -class InputPath: - """When creating component from function, :class:`.InputPath` should be - used as function parameter annotation to tell the system to pass the *data - file path* to the function instead of passing the actual data.""" - - def __init__(self, type=None): - self.type = type - - -class InputTextFile: - """When creating component from function, :class:`.InputTextFile` should be - used as function parameter annotation to tell the system to pass the *text - data stream* object (`io.TextIOWrapper`) to the function instead of passing - the actual data.""" - - def __init__(self, type=None): - self.type = type - - -class InputBinaryFile: - """When creating component from function, :class:`.InputBinaryFile` should - be used as function parameter annotation to tell the system to pass the. - - *binary data stream* object (`io.BytesIO`) to the function instead of - passing the actual data. - """ - - def __init__(self, type=None): - self.type = type - - -class InputArtifact: - """InputArtifact function parameter annotation. - - When creating a component from a Python function, indicates to the - system that function parameter with this annotation should be passed - as a RuntimeArtifact. - """ - - def __init__(self, type: Optional[str] = None): - self.type = type - - -class OutputPath: - """When creating component from function, :class:`.OutputPath` should be - used as function parameter annotation to tell the system that the function - wants to output data by writing it into a file with the given path instead - of returning the data from the function.""" - - def __init__(self, type=None): - self.type = type - - -class OutputTextFile: - """When creating component from function, :class:`.OutputTextFile` should - be used as function parameter annotation to tell the system that the - function wants to output data by writing it into a given text file stream - (`io.TextIOWrapper`) instead of returning the data from the function.""" - - def __init__(self, type=None): - self.type = type - - -class OutputBinaryFile: - """When creating component from function, :class:`.OutputBinaryFile` should - be used as function parameter annotation to tell the system that the - function wants to output data by writing it into a given binary file stream - (:code:`io.BytesIO`) instead of returning the data from the function.""" - - def __init__(self, type=None): - self.type = type - - -class OutputArtifact: - """OutputArtifact function parameter annotation. - - When creating component from function. OutputArtifact indicates that - the associated input parameter should be treated as an MLMD - artifact, whose underlying content, together with metadata will be - updated by this component - """ - - def __init__(self, type: Optional[str] = None): - self.type = type - - -def _make_parent_dirs_and_return_path(file_path: str): - import os - os.makedirs(os.path.dirname(file_path), exist_ok=True) - return file_path - - -def _parent_dirs_maker_that_returns_open_file(mode: str, encoding: str = None): - - def make_parent_dirs_and_return_path(file_path: str): - import os - os.makedirs(os.path.dirname(file_path), exist_ok=True) - return open(file_path, mode=mode, encoding=encoding) - - return make_parent_dirs_and_return_path - - -default_base_image_or_builder = 'python:3.9' - - -def _python_function_name_to_component_name(name): - import re - name_with_spaces = re.sub(' +', ' ', name.replace('_', ' ')).strip(' ') - return name_with_spaces[0].upper() + name_with_spaces[1:] - - -def _capture_function_code_using_cloudpickle( - func, modules_to_capture: List[str] = None) -> str: - import base64 - import sys - import cloudpickle - import pickle - - if modules_to_capture is None: - modules_to_capture = [func.__module__] - - # Hack to force cloudpickle to capture the whole function instead of just referencing the code file. See https://github.com/cloudpipe/cloudpickle/blob/74d69d759185edaeeac7bdcb7015cfc0c652f204/cloudpickle/cloudpickle.py#L490 - old_modules = {} - old_sig = getattr(func, '__signature__', None) - try: # Try is needed to restore the state if something goes wrong - for module_name in modules_to_capture: - if module_name in sys.modules: - old_modules[module_name] = sys.modules.pop(module_name) - # Hack to prevent cloudpickle from trying to pickle generic types that might be present in the signature. See https://github.com/cloudpipe/cloudpickle/issues/196 - # Currently the __signature__ is only set by Airflow components as a means to spoof/pass the function signature to _func_to_component_spec - if hasattr(func, '__signature__'): - del func.__signature__ - func_pickle = base64.b64encode( - cloudpickle.dumps(func, pickle.DEFAULT_PROTOCOL)) - finally: - sys.modules.update(old_modules) - if old_sig: - func.__signature__ = old_sig - - function_loading_code = '''\ -import sys -try: - import cloudpickle as _cloudpickle -except ImportError: - import subprocess - try: - print("cloudpickle is not installed. Installing it globally", file=sys.stderr) - subprocess.run([sys.executable, "-m", "pip", "install", "cloudpickle==1.1.1", "--quiet"], env={"PIP_DISABLE_PIP_VERSION_CHECK": "1"}, check=True) - print("Installed cloudpickle globally", file=sys.stderr) - except: - print("Failed to install cloudpickle globally. Installing for the current user.", file=sys.stderr) - subprocess.run([sys.executable, "-m", "pip", "install", "cloudpickle==1.1.1", "--user", "--quiet"], env={"PIP_DISABLE_PIP_VERSION_CHECK": "1"}, check=True) - print("Installed cloudpickle for the current user", file=sys.stderr) - # Enable loading from user-installed package directory. Python does not add it to sys.path if it was empty at start. Running pip does not refresh `sys.path`. - import site - sys.path.append(site.getusersitepackages()) - import cloudpickle as _cloudpickle - print("cloudpickle loaded successfully after installing.", file=sys.stderr) -''' + ''' -pickler_python_version = {pickler_python_version} -current_python_version = tuple(sys.version_info) -if ( - current_python_version[0] != pickler_python_version[0] or - current_python_version[1] < pickler_python_version[1] or - current_python_version[0] == 3 and ((pickler_python_version[1] < 6) != (current_python_version[1] < 6)) - ): - raise RuntimeError("Incompatible python versions: " + str(current_python_version) + " instead of " + str(pickler_python_version)) - -if current_python_version != pickler_python_version: - print("Warning!: Different python versions. The code may crash! Current environment python version: " + str(current_python_version) + ". Component code python version: " + str(pickler_python_version), file=sys.stderr) - -import base64 -import pickle - -{func_name} = pickle.loads(base64.b64decode({func_pickle})) -'''.format( - func_name=func.__name__, - func_pickle=repr(func_pickle), - pickler_python_version=repr(tuple(sys.version_info)), - ) - - return function_loading_code - - -def strip_type_hints(source_code: str) -> str: - try: - return _strip_type_hints_using_lib2to3(source_code) - except Exception as ex: - print('Error when stripping type annotations: ' + str(ex)) - - try: - return _strip_type_hints_using_strip_hints(source_code) - except Exception as ex: - print('Error when stripping type annotations: ' + str(ex)) - - return source_code - - -def _strip_type_hints_using_strip_hints(source_code: str) -> str: - from strip_hints import strip_string_to_string - - # Workaround for https://github.com/abarker/strip-hints/issues/4 , https://bugs.python.org/issue35107 - # I could not repro it though - if source_code[-1] != '\n': - source_code += '\n' - - return strip_string_to_string(source_code, to_empty=True) - - -def _strip_type_hints_using_lib2to3(source_code: str) -> str: - """Strips type annotations from the function definitions in the provided - source code.""" - - # Using the standard lib2to3 library to strip type annotations. - # Switch to another library like strip-hints if issues are found. - from lib2to3 import fixer_base, refactor - - class StripAnnotations(fixer_base.BaseFix): - PATTERN = r''' - typed_func_parameter=tname - | - typed_func_return_value=funcdef< any+ '->' any+ > - ''' - - def transform(self, node, results): - if 'typed_func_parameter' in results: - # Delete the annotation part of the function parameter declaration - del node.children[1:] - elif 'typed_func_return_value' in results: - # Delete the return annotation part of the function declaration - del node.children[-4:-2] - return node - - class Refactor(refactor.RefactoringTool): - - def __init__(self, fixers): - self._fixers = [cls(None, None) for cls in fixers] - super().__init__(None, {'print_function': True}) - - def get_fixers(self): - return self._fixers, [] - - stripped_code = str( - Refactor([StripAnnotations]).refactor_string(source_code, '')) - return stripped_code - - -def _get_function_source_definition(func: Callable) -> str: - func_code = inspect.getsource(func) - - # Function might be defined in some indented scope (e.g. in another - # function). We need to handle this and properly dedent the function source - # code - func_code = textwrap.dedent(func_code) - func_code_lines = func_code.split('\n') - - # Removing possible decorators (can be multiline) until the function - # definition is found - func_code_lines = itertools.dropwhile(lambda x: not x.startswith('def'), - func_code_lines) - - if not func_code_lines: - raise ValueError( - 'Failed to dedent and clean up the source of function "{}". ' - 'It is probably not properly indented.'.format(func.__name__)) - - return '\n'.join(func_code_lines) - - -def _capture_function_code_using_source_copy(func: Callable) -> str: - func_code = _get_function_source_definition(func) - - # Stripping type annotations to prevent import errors. - # The most common cases are InputPath/OutputPath and typing.NamedTuple annotations - return strip_type_hints(func_code) - - -def _extract_component_interface(func: Callable) -> ComponentSpec: - single_output_name_const = 'Output' - - signature = inspect.signature(func) - parameters = list(signature.parameters.values()) - - parsed_docstring = docstring_parser.parse(inspect.getdoc(func)) - doc_dict = {p.arg_name: p.description for p in parsed_docstring.params} - - inputs = [] - outputs = [] - - def annotation_to_type_struct(annotation): - if not annotation or annotation == inspect.Parameter.empty: - return None - if hasattr(annotation, 'to_dict'): - annotation = annotation.to_dict() - if isinstance(annotation, dict): - return annotation - if isinstance(annotation, type): - type_struct = get_canonical_type_name_for_type(annotation) - if type_struct: - return type_struct - type_name = str(annotation.__name__) - elif hasattr( - annotation, '__forward_arg__' - ): # Handling typing.ForwardRef('Type_name') (the name was _ForwardRef in python 3.5-3.6) - type_name = str(annotation.__forward_arg__) - else: - type_name = str(annotation) - - # It's also possible to get the converter by type name - type_struct = get_canonical_type_name_for_type(type_name) - if type_struct: - return type_struct - return type_name - - input_names = set() - output_names = set() - for parameter in parameters: - parameter_type = type_annotation_utils.maybe_strip_optional_from_annotation( - parameter.annotation) - passing_style = None - io_name = parameter.name - - if isinstance( - parameter_type, - (InputArtifact, InputPath, InputTextFile, InputBinaryFile, - OutputArtifact, OutputPath, OutputTextFile, OutputBinaryFile)): - - # Removing the "_path" and "_file" suffixes from the input/output names as the argument passed to the component needs to be the data itself, not local file path. - # Problem: When accepting file inputs (outputs), the function inside the component receives file paths (or file streams), so it's natural to call the function parameter "something_file_path" (e.g. model_file_path or number_file_path). - # But from the outside perspective, there are no files or paths - the actual data objects (or references to them) are passed in. - # It looks very strange when argument passing code looks like this: `component(number_file_path=42)`. This looks like an error since 42 is not a path. It's not even a string. - # It's much more natural to strip the names of file inputs and outputs of "_file" or "_path" suffixes. Then the argument passing code will look natural: "component(number=42)". - if isinstance( - parameter_type, - (InputPath, OutputPath)) and io_name.endswith('_path'): - io_name = io_name[0:-len('_path')] - if io_name.endswith('_file'): - io_name = io_name[0:-len('_file')] - - passing_style = type(parameter_type) - parameter_type = parameter_type.type - if parameter.default is not inspect.Parameter.empty and not ( - passing_style == InputPath and parameter.default is None): - raise ValueError( - 'Path inputs only support default values of None. Default values for outputs are not supported.' - ) - - type_struct = annotation_to_type_struct(parameter_type) - - if passing_style in [ - OutputArtifact, OutputPath, OutputTextFile, OutputBinaryFile - ]: - io_name = _make_name_unique_by_adding_index(io_name, output_names, - '_') - output_names.add(io_name) - output_spec = OutputSpec( - name=io_name, - type=type_struct, - description=doc_dict.get(parameter.name)) - output_spec._passing_style = passing_style - output_spec._parameter_name = parameter.name - outputs.append(output_spec) - else: - io_name = _make_name_unique_by_adding_index(io_name, input_names, - '_') - input_names.add(io_name) - input_spec = InputSpec( - name=io_name, - type=type_struct, - description=doc_dict.get(parameter.name)) - if parameter.default is not inspect.Parameter.empty: - input_spec.optional = True - if parameter.default is not None: - outer_type_name = list(type_struct.keys())[0] if isinstance( - type_struct, dict) else type_struct - try: - input_spec.default = serialize_value( - parameter.default, outer_type_name) - except Exception as ex: - warnings.warn( - 'Could not serialize the default value of the parameter "{}". {}' - .format(parameter.name, ex)) - input_spec._passing_style = passing_style - input_spec._parameter_name = parameter.name - inputs.append(input_spec) - - #Analyzing the return type annotations. - return_ann = signature.return_annotation - if hasattr(return_ann, '_fields'): #NamedTuple - # Getting field type annotations. - # __annotations__ does not exist in python 3.5 and earlier - # _field_types does not exist in python 3.9 and later - field_annotations = getattr(return_ann, - '__annotations__', None) or getattr( - return_ann, '_field_types', None) - for field_name in return_ann._fields: - type_struct = None - if field_annotations: - type_struct = annotation_to_type_struct( - field_annotations.get(field_name, None)) - - output_name = _make_name_unique_by_adding_index( - field_name, output_names, '_') - output_names.add(output_name) - output_spec = OutputSpec( - name=output_name, - type=type_struct, - ) - output_spec._passing_style = None - output_spec._return_tuple_field_name = field_name - outputs.append(output_spec) - # Deprecated dict-based way of declaring multiple outputs. Was only used by the @component decorator - elif isinstance(return_ann, dict): - warnings.warn( - "The ability to specify multiple outputs using the dict syntax has been deprecated." - "It will be removed soon after release 0.1.32." - "Please use typing.NamedTuple to declare multiple outputs.") - for output_name, output_type_annotation in return_ann.items(): - output_type_struct = annotation_to_type_struct( - output_type_annotation) - output_spec = OutputSpec( - name=output_name, - type=output_type_struct, - ) - outputs.append(output_spec) - elif signature.return_annotation is not None and signature.return_annotation != inspect.Parameter.empty: - output_name = _make_name_unique_by_adding_index( - single_output_name_const, output_names, '_' - ) # Fixes exotic, but possible collision: `def func(output_path: OutputPath()) -> str: ...` - output_names.add(output_name) - type_struct = annotation_to_type_struct(signature.return_annotation) - output_spec = OutputSpec( - name=output_name, - type=type_struct, - ) - output_spec._passing_style = None - outputs.append(output_spec) - - # Component name and description are derived from the function's name and docstring. - # The name can be overridden by setting setting func.__name__ attribute (of the legacy func._component_human_name attribute). - # The description can be overridden by setting the func.__doc__ attribute (or the legacy func._component_description attribute). - component_name = getattr(func, '_component_human_name', - None) or _python_function_name_to_component_name( - func.__name__) - description = getattr(func, '_component_description', - None) or parsed_docstring.short_description - if description: - description = description.strip() - - component_spec = ComponentSpec( - name=component_name, - description=description, - inputs=inputs if inputs else None, - outputs=outputs if outputs else None, - ) - return component_spec - - -def _func_to_component_spec(func, - extra_code='', - base_image: str = None, - packages_to_install: List[str] = None, - modules_to_capture: List[str] = None, - use_code_pickling=False) -> ComponentSpec: - """Takes a self-contained python function and converts it to component. - - Args: - func: Required. The function to be converted - base_image: Optional. Docker image to be used as a base image for the python component. Must have python 3.5+ installed. Default is python:3.7 - Note: The image can also be specified by decorating the function with the @python_component decorator. If different base images are explicitly specified in both places, an error is raised. - extra_code: Optional. Python source code that gets placed before the function code. Can be used as workaround to define types used in function signature. - packages_to_install: Optional. List of [versioned] python packages to pip install before executing the user function. - modules_to_capture: Optional. List of module names that will be captured (instead of just referencing) during the dependency scan. By default the :code:`func.__module__` is captured. - use_code_pickling: Specifies whether the function code should be captured using pickling as opposed to source code manipulation. Pickling has better support for capturing dependencies, but is sensitive to version mismatch between python in component creation environment and runtime image. - - Returns: - A :py:class:`kfp.components.structures.ComponentSpec` instance. - """ - decorator_base_image = getattr(func, '_component_base_image', None) - if decorator_base_image is not None: - if base_image is not None and decorator_base_image != base_image: - raise ValueError( - 'base_image ({}) conflicts with the decorator-specified base image metadata ({})' - .format(base_image, decorator_base_image)) - else: - base_image = decorator_base_image - else: - if base_image is None: - base_image = default_base_image_or_builder - if isinstance(base_image, Callable): - base_image = base_image() - - packages_to_install = packages_to_install or [] - - component_spec = _extract_component_interface(func) - - component_inputs = component_spec.inputs or [] - component_outputs = component_spec.outputs or [] - - arguments = [] - arguments.extend( - InputValuePlaceholder(input.name) for input in component_inputs) - arguments.extend( - OutputPathPlaceholder(output.name) for output in component_outputs) - - if use_code_pickling: - func_code = _capture_function_code_using_cloudpickle( - func, modules_to_capture) - # pip startup is quite slow. TODO: Remove the special cloudpickle installation code in favor of the the following line once a way to speed up pip startup is discovered. - #packages_to_install.append('cloudpickle==1.1.1') - else: - func_code = _capture_function_code_using_source_copy(func) - - definitions = set() - - def get_deserializer_and_register_definitions(type_name): - deserializer_code = get_deserializer_code_for_type_name(type_name) - if deserializer_code: - (deserializer_code_str, definition_str) = deserializer_code - if definition_str: - definitions.add(definition_str) - return deserializer_code_str - return 'str' - - pre_func_definitions = set() - - def get_argparse_type_for_input_file(passing_style): - # Bypass InputArtifact and OutputArtifact. - if passing_style in (None, InputArtifact, OutputArtifact): - return None - - if passing_style is InputPath: - return 'str' - elif passing_style is InputTextFile: - return "argparse.FileType('rt')" - elif passing_style is InputBinaryFile: - return "argparse.FileType('rb')" - # For Output* we cannot use the build-in argparse.FileType objects since they do not create parent directories. - elif passing_style is OutputPath: - # ~= return 'str' - pre_func_definitions.add( - inspect.getsource(_make_parent_dirs_and_return_path)) - return _make_parent_dirs_and_return_path.__name__ - elif passing_style is OutputTextFile: - # ~= return "argparse.FileType('wt')" - pre_func_definitions.add( - inspect.getsource(_parent_dirs_maker_that_returns_open_file)) - return _parent_dirs_maker_that_returns_open_file.__name__ + "('wt')" - elif passing_style is OutputBinaryFile: - # ~= return "argparse.FileType('wb')" - pre_func_definitions.add( - inspect.getsource(_parent_dirs_maker_that_returns_open_file)) - return _parent_dirs_maker_that_returns_open_file.__name__ + "('wb')" - raise NotImplementedError('Unexpected data passing style: "{}".'.format( - str(passing_style))) - - def get_serializer_and_register_definitions(type_name) -> str: - serializer_func = get_serializer_func_for_type_name(type_name) - if serializer_func: - # If serializer is not part of the standard python library, then include its code in the generated program - if hasattr(serializer_func, - '__module__') and not _module_is_builtin_or_standard( - serializer_func.__module__): - import inspect - serializer_code_str = inspect.getsource(serializer_func) - definitions.add(serializer_code_str) - return serializer_func.__name__ - return 'str' - - arg_parse_code_lines = [ - 'import argparse', - '_parser = argparse.ArgumentParser(prog={prog_repr}, description={description_repr})' - .format( - prog_repr=repr(component_spec.name or ''), - description_repr=repr(component_spec.description or ''), - ), - ] - outputs_passed_through_func_return_tuple = [ - output for output in component_outputs if output._passing_style is None - ] - file_outputs_passed_using_func_parameters = [ - output for output in component_outputs - if output._passing_style is not None - ] - arguments = [] - for input in component_inputs + file_outputs_passed_using_func_parameters: - param_flag = "--" + input.name.replace("_", "-") - is_required = isinstance(input, OutputSpec) or not input.optional - line = '_parser.add_argument("{param_flag}", dest="{param_var}", type={param_type}, required={is_required}, default=argparse.SUPPRESS)'.format( - param_flag=param_flag, - param_var=input. - _parameter_name, # Not input.name, since the inputs could have been renamed - param_type=get_argparse_type_for_input_file(input._passing_style) or - get_deserializer_and_register_definitions(input.type), - is_required=str(is_required), - ) - arg_parse_code_lines.append(line) - - if input._passing_style in [ - InputPath, InputTextFile, InputBinaryFile, InputArtifact - ]: - arguments_for_input = [param_flag, InputPathPlaceholder(input.name)] - elif input._passing_style in [ - OutputPath, OutputTextFile, OutputBinaryFile, OutputArtifact - ]: - arguments_for_input = [ - param_flag, OutputPathPlaceholder(input.name) - ] - else: - arguments_for_input = [ - param_flag, InputValuePlaceholder(input.name) - ] - - if is_required: - arguments.extend(arguments_for_input) - else: - arguments.append( - IfPlaceholder( - IfPlaceholderStructure( - condition=IsPresentPlaceholder(input.name), - then_value=arguments_for_input, - ))) - - if outputs_passed_through_func_return_tuple: - param_flag = "----output-paths" - output_param_var = "_output_paths" - line = '_parser.add_argument("{param_flag}", dest="{param_var}", type=str, nargs={nargs})'.format( - param_flag=param_flag, - param_var=output_param_var, - nargs=len(outputs_passed_through_func_return_tuple), - ) - arg_parse_code_lines.append(line) - arguments.append(param_flag) - arguments.extend( - OutputPathPlaceholder(output.name) - for output in outputs_passed_through_func_return_tuple) - - output_serialization_expression_strings = [] - for output in outputs_passed_through_func_return_tuple: - serializer_call_str = get_serializer_and_register_definitions( - output.type) - output_serialization_expression_strings.append(serializer_call_str) - - pre_func_code = '\n'.join(list(pre_func_definitions)) - - arg_parse_code_lines = sorted(list(definitions)) + arg_parse_code_lines - - arg_parse_code_lines.append('_parsed_args = vars(_parser.parse_args())',) - if outputs_passed_through_func_return_tuple: - arg_parse_code_lines.append( - '_output_files = _parsed_args.pop("_output_paths", [])',) - - # Putting singular return values in a list to be "zipped" with the serializers and output paths - outputs_to_list_code = '' - return_ann = inspect.signature(func).return_annotation - if ( # The return type is singular, not sequence - return_ann is not None and return_ann != inspect.Parameter.empty and - not isinstance(return_ann, dict) and - not hasattr(return_ann, '_fields') # namedtuple - ): - outputs_to_list_code = '_outputs = [_outputs]' - - output_serialization_code = ''.join( - ' {},\n'.format(s) for s in output_serialization_expression_strings) - - full_output_handling_code = ''' - -{outputs_to_list_code} - -_output_serializers = [ -{output_serialization_code} -] - -import os -for idx, output_file in enumerate(_output_files): - try: - os.makedirs(os.path.dirname(output_file)) - except OSError: - pass - with open(output_file, 'w') as f: - f.write(_output_serializers[idx](_outputs[idx])) -'''.format( - output_serialization_code=output_serialization_code, - outputs_to_list_code=outputs_to_list_code, - ) - - full_source = \ -'''\ -{pre_func_code} - -{extra_code} - -{func_code} - -{arg_parse_code} - -_outputs = {func_name}(**_parsed_args) -'''.format( - func_name=func.__name__, - func_code=func_code, - pre_func_code=pre_func_code, - extra_code=extra_code, - arg_parse_code='\n'.join(arg_parse_code_lines), - ) - - if outputs_passed_through_func_return_tuple: - full_source += full_output_handling_code - - #Removing consecutive blank lines - import re - full_source = re.sub('\n\n\n+', '\n\n', full_source).strip('\n') + '\n' - - package_preinstallation_command = [] - if packages_to_install: - package_install_command_line = 'PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location {}'.format( - ' '.join([repr(str(package)) for package in packages_to_install])) - package_preinstallation_command = [ - 'sh', '-c', - '({pip_install} || {pip_install} --user) && "$0" "$@"'.format( - pip_install=package_install_command_line) - ] - - component_spec.implementation = ContainerImplementation( - container=ContainerSpec( - image=base_image, - command=package_preinstallation_command + [ - 'sh', - '-ec', - # Writing the program code to a file. - # This is needed for Python to show stack traces and for `inspect.getsource` to work (used by PyTorch JIT and this module for example). - textwrap.dedent('''\ - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - '''), - full_source, - ], - args=arguments, - )) - - return component_spec - - -def _func_to_component_dict(func, - extra_code='', - base_image: str = None, - packages_to_install: List[str] = None, - modules_to_capture: List[str] = None, - use_code_pickling=False): - return _func_to_component_spec( - func=func, - extra_code=extra_code, - base_image=base_image, - packages_to_install=packages_to_install, - modules_to_capture=modules_to_capture, - use_code_pickling=use_code_pickling, - ).to_dict() - - -def func_to_component_text(func, - extra_code='', - base_image: str = None, - packages_to_install: List[str] = None, - modules_to_capture: List[str] = None, - use_code_pickling=False): - '''Converts a Python function to a component definition and returns its textual representation. - - Function docstring is used as component description. Argument and return annotations are used as component input/output types. - - To declare a function with multiple return values, use the NamedTuple return annotation syntax:: - - from typing import NamedTuple - def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('sum', float), ('product', float)]): - """Returns sum and product of two arguments""" - return (a + b, a * b) - - Args: - func: The python function to convert - base_image: Optional. Specify a custom Docker container image to use in the component. For lightweight components, the image needs to have python 3.5+. Default is python:3.7 - extra_code: Optional. Extra code to add before the function code. Can be used as workaround to define types used in function signature. - packages_to_install: Optional. List of [versioned] python packages to pip install before executing the user function. - modules_to_capture: Optional. List of module names that will be captured (instead of just referencing) during the dependency scan. By default the :code:`func.__module__` is captured. The actual algorithm: Starting with the initial function, start traversing dependencies. If the dependency.__module__ is in the modules_to_capture list then it's captured and it's dependencies are traversed. Otherwise the dependency is only referenced instead of capturing and its dependencies are not traversed. - use_code_pickling: Specifies whether the function code should be captured using pickling as opposed to source code manipulation. Pickling has better support for capturing dependencies, but is sensitive to version mismatch between python in component creation environment and runtime image. - - Returns: - Textual representation of a component definition - ''' - component_dict = _func_to_component_dict( - func=func, - extra_code=extra_code, - base_image=base_image, - packages_to_install=packages_to_install, - modules_to_capture=modules_to_capture, - use_code_pickling=use_code_pickling, - ) - return dump_yaml(component_dict) - - -def func_to_component_file(func, - output_component_file, - base_image: str = None, - extra_code='', - packages_to_install: List[str] = None, - modules_to_capture: List[str] = None, - use_code_pickling=False) -> None: - '''Converts a Python function to a component definition and writes it to a file. - - Function docstring is used as component description. Argument and return annotations are used as component input/output types. - - To declare a function with multiple return values, use the NamedTuple return annotation syntax:: - - from typing import NamedTuple - def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('sum', float), ('product', float)]): - """Returns sum and product of two arguments""" - return (a + b, a * b) - - Args: - func: The python function to convert - output_component_file: Write a component definition to a local file. Can be used for sharing. - base_image: Optional. Specify a custom Docker container image to use in the component. For lightweight components, the image needs to have python 3.5+. Default is tensorflow/tensorflow:1.13.2-py3 - extra_code: Optional. Extra code to add before the function code. Can be used as workaround to define types used in function signature. - packages_to_install: Optional. List of [versioned] python packages to pip install before executing the user function. - modules_to_capture: Optional. List of module names that will be captured (instead of just referencing) during the dependency scan. By default the :code:`func.__module__` is captured. The actual algorithm: Starting with the initial function, start traversing dependencies. If the :code:`dependency.__module__` is in the :code:`modules_to_capture` list then it's captured and it's dependencies are traversed. Otherwise the dependency is only referenced instead of capturing and its dependencies are not traversed. - use_code_pickling: Specifies whether the function code should be captured using pickling as opposed to source code manipulation. Pickling has better support for capturing dependencies, but is sensitive to version mismatch between python in component creation environment and runtime image. - ''' - - component_yaml = func_to_component_text( - func=func, - extra_code=extra_code, - base_image=base_image, - packages_to_install=packages_to_install, - modules_to_capture=modules_to_capture, - use_code_pickling=use_code_pickling, - ) - - Path(output_component_file).write_text(component_yaml) - - -def func_to_container_op( - func: Callable, - output_component_file: Optional[str] = None, - base_image: Optional[str] = None, - extra_code: Optional[str] = '', - packages_to_install: List[str] = None, - modules_to_capture: List[str] = None, - use_code_pickling: bool = False, - annotations: Optional[Mapping[str, str]] = None, -): - '''Converts a Python function to a component and returns a task - (:class:`kfp.dsl.ContainerOp`) factory. - - Function docstring is used as component description. Argument and return annotations are used as component input/output types. - - To declare a function with multiple return values, use the :code:`NamedTuple` return annotation syntax:: - - from typing import NamedTuple - def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('DummyName', [('sum', float), ('product', float)]): - """Returns sum and product of two arguments""" - return (a + b, a * b) - - Args: - func: The python function to convert - base_image: Optional. Specify a custom Docker container image to use in the component. For lightweight components, the image needs to have python 3.5+. Default is tensorflow/tensorflow:1.13.2-py3 - output_component_file: Optional. Write a component definition to a local file. Can be used for sharing. - extra_code: Optional. Extra code to add before the function code. Can be used as workaround to define types used in function signature. - packages_to_install: Optional. List of [versioned] python packages to pip install before executing the user function. - modules_to_capture: Optional. List of module names that will be captured (instead of just referencing) during the dependency scan. By default the :code:`func.__module__` is captured. The actual algorithm: Starting with the initial function, start traversing dependencies. If the :code:`dependency.__module__` is in the :code:`modules_to_capture` list then it's captured and it's dependencies are traversed. Otherwise the dependency is only referenced instead of capturing and its dependencies are not traversed. - use_code_pickling: Specifies whether the function code should be captured using pickling as opposed to source code manipulation. Pickling has better support for capturing dependencies, but is sensitive to version mismatch between python in component creation environment and runtime image. - annotations: Optional. Allows adding arbitrary key-value data to the component specification. - - Returns: - A factory function with a strongly-typed signature taken from the python function. - Once called with the required arguments, the factory constructs a pipeline task instance (:class:`kfp.dsl.ContainerOp`) that can run the original function in a container. - ''' - - component_spec = _func_to_component_spec( - func=func, - extra_code=extra_code, - base_image=base_image, - packages_to_install=packages_to_install, - modules_to_capture=modules_to_capture, - use_code_pickling=use_code_pickling, - ) - if annotations: - component_spec.metadata = structures.MetadataSpec( - annotations=annotations,) - - output_component_file = output_component_file or getattr( - func, '_component_target_component_file', None) - if output_component_file: - component_spec.save(output_component_file) - #TODO: assert ComponentSpec.from_dict(load_yaml(output_component_file)) == component_spec - - return _create_task_factory_from_component_spec(component_spec) - - -def create_component_from_func_v2(func: Callable, - base_image: Optional[str] = None, - packages_to_install: List[str] = None, - output_component_file: Optional[str] = None, - install_kfp_package: bool = True, - kfp_package_path: Optional[str] = None): - """Converts a Python function to a v2 lightweight component. - - A lightweight component is a self-contained Python function that includes - all necessary imports and dependencies. - - Args: - func: The python function to create a component from. The function - should have type annotations for all its arguments, indicating how - it is intended to be used (e.g. as an input/output Artifact object, - a plain parameter, or a path to a file). - base_image: The image to use when executing |func|. It should - contain a default Python interpreter that is compatible with KFP. - packages_to_install: A list of optional packages to install before - executing |func|. - install_kfp_package: Specifies if we should add a KFP Python package to - |packages_to_install|. Lightweight Python functions always require - an installation of KFP in |base_image| to work. If you specify - a |base_image| that already contains KFP, you can set this to False. - kfp_package_path: Specifies the location from which to install KFP. By - default, this will try to install from PyPi using the same version - as that used when this component was created. KFP developers can - choose to override this to point to a Github pull request or - other pip-compatible location when testing changes to lightweight - Python functions. - - Returns: - A component task factory that can be used in pipeline definitions. - """ - warnings.warn( - 'create_component_from_func_v2() has been deprecated and will be' - ' removed in KFP v1.9. Please use' - ' @kfp.dsl.component() instead.', - category=FutureWarning, - ) - from kfp.components import component_factory - return component_factory.create_component_from_func( - func=func, - base_image=base_image, - packages_to_install=packages_to_install, - install_kfp_package=install_kfp_package, - kfp_package_path=kfp_package_path) - - -def create_component_from_func( - func: Callable, - output_component_file: Optional[str] = None, - base_image: Optional[str] = None, - packages_to_install: List[str] = None, - annotations: Optional[Mapping[str, str]] = None, -): - '''Converts a Python function to a component and returns a task factory - (a function that accepts arguments and returns a task object). - - Args: - func: The python function to convert - base_image: Optional. Specify a custom Docker container image to use in the component. For lightweight components, the image needs to have python 3.5+. Default is the python image corresponding to the current python environment. - output_component_file: Optional. Write a component definition to a local file. The produced component file can be loaded back by calling :code:`load_component_from_file` or :code:`load_component_from_uri`. - packages_to_install: Optional. List of [versioned] python packages to pip install before executing the user function. - annotations: Optional. Allows adding arbitrary key-value data to the component specification. - - Returns: - A factory function with a strongly-typed signature taken from the python function. - Once called with the required arguments, the factory constructs a task instance that can run the original function in a container. - - Examples: - The function name and docstring are used as component name and description. Argument and return annotations are used as component input/output types:: - - def add(a: float, b: float) -> float: - """Returns sum of two arguments""" - return a + b - - # add_op is a task factory function that creates a task object when given arguments - add_op = create_component_from_func( - func=add, - base_image='python:3.9', # Optional - output_component_file='add.component.yaml', # Optional - packages_to_install=['pandas==0.24'], # Optional - ) - - # The component spec can be accessed through the .component_spec attribute: - add_op.component_spec.save('add.component.yaml') - - # The component function can be called with arguments to create a task: - add_task = add_op(1, 3) - - # The resulting task has output references, corresponding to the component outputs. - # When the function only has a single anonymous return value, the output name is "Output": - sum_output_ref = add_task.outputs['Output'] - - # These task output references can be passed to other component functions, constructing a computation graph: - task2 = add_op(sum_output_ref, 5) - - - :code:`create_component_from_func` function can also be used as decorator:: - - @create_component_from_func - def add_op(a: float, b: float) -> float: - """Returns sum of two arguments""" - return a + b - - To declare a function with multiple return values, use the :code:`NamedTuple` return annotation syntax:: - - from typing import NamedTuple - - def add_multiply_two_numbers(a: float, b: float) -> NamedTuple('Outputs', [('sum', float), ('product', float)]): - """Returns sum and product of two arguments""" - return (a + b, a * b) - - add_multiply_op = create_component_from_func(add_multiply_two_numbers) - - # The component function can be called with arguments to create a task: - add_multiply_task = add_multiply_op(1, 3) - - # The resulting task has output references, corresponding to the component outputs: - sum_output_ref = add_multiply_task.outputs['sum'] - - # These task output references can be passed to other component functions, constructing a computation graph: - task2 = add_multiply_op(sum_output_ref, 5) - - Bigger data should be read from files and written to files. - Use the :py:class:`kfp.components.InputPath` parameter annotation to tell the system that the function wants to consume the corresponding input data as a file. The system will download the data, write it to a local file and then pass the **path** of that file to the function. - Use the :py:class:`kfp.components.OutputPath` parameter annotation to tell the system that the function wants to produce the corresponding output data as a file. The system will prepare and pass the **path** of a file where the function should write the output data. After the function exits, the system will upload the data to the storage system so that it can be passed to downstream components. - - You can specify the type of the consumed/produced data by specifying the type argument to :py:class:`kfp.components.InputPath` and :py:class:`kfp.components.OutputPath`. The type can be a python type or an arbitrary type name string. :code:`OutputPath('CatBoostModel')` means that the function states that the data it has written to a file has type :code:`CatBoostModel`. :code:`InputPath('CatBoostModel')` means that the function states that it expect the data it reads from a file to have type 'CatBoostModel'. When the pipeline author connects inputs to outputs the system checks whether the types match. - Every kind of data can be consumed as a file input. Conversely, bigger data should not be consumed by value as all value inputs pass through the command line. - - Example of a component function declaring file input and output:: - - def catboost_train_classifier( - training_data_path: InputPath('CSV'), # Path to input data file of type "CSV" - trained_model_path: OutputPath('CatBoostModel'), # Path to output data file of type "CatBoostModel" - number_of_trees: int = 100, # Small output of type "Integer" - ) -> NamedTuple('Outputs', [ - ('Accuracy', float), # Small output of type "Float" - ('Precision', float), # Small output of type "Float" - ('JobUri', 'URI'), # Small output of type "URI" - ]): - """Trains CatBoost classification model""" - ... - - return (accuracy, precision, recall) - ''' - - component_spec = _func_to_component_spec( - func=func, - base_image=base_image, - packages_to_install=packages_to_install, - ) - if annotations: - component_spec.metadata = structures.MetadataSpec( - annotations=annotations,) - - if output_component_file: - component_spec.save(output_component_file) - - return _create_task_factory_from_component_spec(component_spec) - - -def _module_is_builtin_or_standard(module_name: str) -> bool: - import sys - if module_name in sys.builtin_module_names: - return True - import distutils.sysconfig as sysconfig - import os - std_lib_dir = sysconfig.get_python_lib(standard_lib=True) - module_name_parts = module_name.split('.') - expected_module_path = os.path.join(std_lib_dir, *module_name_parts) - return os.path.exists(expected_module_path) or os.path.exists( - expected_module_path + '.py') diff --git a/sdk/python/kfp/deprecated/components/_python_to_graph_component.py b/sdk/python/kfp/deprecated/components/_python_to_graph_component.py deleted file mode 100644 index 074b0739de7..00000000000 --- a/sdk/python/kfp/deprecated/components/_python_to_graph_component.py +++ /dev/null @@ -1,213 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__all__ = [ - 'create_graph_component_from_pipeline_func', -] - -from collections import OrderedDict -from typing import Callable, Mapping, Optional - -from . import _components -from . import structures -from ._structures import ComponentSpec, GraphImplementation, GraphInputReference, GraphSpec, OutputSpec, TaskOutputArgument -from ._naming import _make_name_unique_by_adding_index -from ._python_op import _extract_component_interface -from ._components import _create_task_factory_from_component_spec - - -def create_graph_component_from_pipeline_func( - pipeline_func: Callable, - output_component_file: str = None, - embed_component_specs: bool = False, - annotations: Optional[Mapping[str, str]] = None, -) -> Callable: - """Creates graph component definition from a python pipeline function. The - component file can be published for sharing. - - Pipeline function is a function that only calls component functions and passes outputs to inputs. - This feature is experimental and lacks support for some of the DSL features like conditions and loops. - Only pipelines consisting of loaded components or python components are currently supported (no manually created ContainerOps or ResourceOps). - - .. warning:: - - Please note this feature is considered experimental! - - Args: - pipeline_func: Python function to convert - output_component_file: Path of the file where the component definition will be written. The `component.yaml` file can then be published for sharing. - embed_component_specs: Whether to embed component definitions or just reference them. Embedding makes the graph component self-contained. Default is False. - annotations: Optional. Allows adding arbitrary key-value data to the component specification. - - Returns: - A function representing the graph component. The component spec can be accessed using the .component_spec attribute. - The function will have the same parameters as the original function. - When called, the function will return a task object, corresponding to the graph component. - To reference the outputs of the task, use task.outputs["Output name"]. - - Example:: - - producer_op = load_component_from_file('producer/component.yaml') - processor_op = load_component_from_file('processor/component.yaml') - - def pipeline1(pipeline_param_1: int): - producer_task = producer_op() - processor_task = processor_op(pipeline_param_1, producer_task.outputs['Output 2']) - - return OrderedDict([ - ('Pipeline output 1', producer_task.outputs['Output 1']), - ('Pipeline output 2', processor_task.outputs['Output 2']), - ]) - - create_graph_component_from_pipeline_func(pipeline1, output_component_file='pipeline.component.yaml') - """ - component_spec = create_graph_component_spec_from_pipeline_func( - pipeline_func, embed_component_specs) - if annotations: - component_spec.metadata = structures.MetadataSpec( - annotations=annotations,) - if output_component_file: - from pathlib import Path - from ._yaml_utils import dump_yaml - component_dict = component_spec.to_dict() - component_yaml = dump_yaml(component_dict) - Path(output_component_file).write_text(component_yaml) - - return _create_task_factory_from_component_spec(component_spec) - - -def create_graph_component_spec_from_pipeline_func( - pipeline_func: Callable, - embed_component_specs: bool = False) -> ComponentSpec: - - component_spec = _extract_component_interface(pipeline_func) - # Checking the function parameters - they should not have file passing annotations. - input_specs = component_spec.inputs or [] - for input in input_specs: - if input._passing_style: - raise TypeError( - 'Graph component function parameter "{}" cannot have file-passing annotation "{}".' - .format(input.name, input._passing_style)) - - task_map = OrderedDict() #Preserving task order - - from ._components import _create_task_spec_from_component_and_arguments - - def task_construction_handler( - component_spec, - arguments, - component_ref, - ): - task = _create_task_spec_from_component_and_arguments( - component_spec=component_spec, - arguments=arguments, - component_ref=component_ref, - ) - - #Rewriting task ids so that they're same every time - task_id = task.component_ref.spec.name or "Task" - task_id = _make_name_unique_by_adding_index(task_id, task_map.keys(), - ' ') - for output_ref in task.outputs.values(): - output_ref.task_output.task_id = task_id - output_ref.task_output.task = None - task_map[task_id] = task - # Remove the component spec from component reference unless it will make the reference empty or unless explicitly asked by the user - if not embed_component_specs and any([ - task.component_ref.name, task.component_ref.url, - task.component_ref.digest - ]): - task.component_ref.spec = None - - return task #The handler is a transformation function, so it must pass the task through. - - # Preparing the pipeline_func arguments - # TODO: The key should be original parameter name if different - pipeline_func_args = { - input.name: GraphInputReference(input_name=input.name).as_argument() - for input in input_specs - } - - try: - #Setting the handler to fix and catch the tasks. - # FIX: The handler only hooks container component creation - old_handler = _components._container_task_constructor - _components._container_task_constructor = task_construction_handler - - #Calling the pipeline_func with GraphInputArgument instances as arguments - pipeline_func_result = pipeline_func(**pipeline_func_args) - finally: - _components._container_task_constructor = old_handler - - # Getting graph outputs - output_names = [output.name for output in (component_spec.outputs or [])] - - if len(output_names) == 1 and output_names[ - 0] == 'Output': # TODO: Check whether the NamedTuple syntax was used - pipeline_func_result = [pipeline_func_result] - - if isinstance(pipeline_func_result, tuple) and hasattr( - pipeline_func_result, - '_asdict'): # collections.namedtuple and typing.NamedTuple - pipeline_func_result = pipeline_func_result._asdict() - - if isinstance(pipeline_func_result, dict): - if output_names: - if set(output_names) != set(pipeline_func_result.keys()): - raise ValueError( - 'Returned outputs do not match outputs specified in the function signature: {} = {}' - .format( - str(set(pipeline_func_result.keys())), - str(set(output_names)))) - - if pipeline_func_result is None: - graph_output_value_map = {} - elif isinstance(pipeline_func_result, dict): - graph_output_value_map = OrderedDict(pipeline_func_result) - elif isinstance(pipeline_func_result, (list, tuple)): - if output_names: - if len(pipeline_func_result) != len(output_names): - raise ValueError( - 'Expected {} values from pipeline function, but got {}.' - .format(len(output_names), len(pipeline_func_result))) - graph_output_value_map = OrderedDict( - (name_value[0], name_value[1]) - for name_value in zip(output_names, pipeline_func_result)) - else: - graph_output_value_map = OrderedDict( - (output_value.task_output.output_name, output_value) - for output_value in pipeline_func_result - ) # TODO: Fix possible name non-uniqueness (e.g. use task id as prefix or add index to non-unique names) - else: - raise TypeError('Pipeline must return outputs as tuple or OrderedDict.') - - #Checking the pipeline_func output object types - for output_name, output_value in graph_output_value_map.items(): - if not isinstance(output_value, TaskOutputArgument): - raise TypeError( - 'Only TaskOutputArgument instances should be returned from graph component, but got "{}" = "{}".' - .format(output_name, str(output_value))) - - if not component_spec.outputs and graph_output_value_map: - component_spec.outputs = [ - OutputSpec(name=output_name, type=output_value.task_output.type) - for output_name, output_value in graph_output_value_map.items() - ] - - component_spec.implementation = GraphImplementation( - graph=GraphSpec( - tasks=task_map, - output_values=graph_output_value_map, - )) - return component_spec diff --git a/sdk/python/kfp/deprecated/components/_structures.py b/sdk/python/kfp/deprecated/components/_structures.py deleted file mode 100644 index 444c4494ae0..00000000000 --- a/sdk/python/kfp/deprecated/components/_structures.py +++ /dev/null @@ -1,896 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__all__ = [ - 'InputSpec', - 'OutputSpec', - 'InputValuePlaceholder', - 'InputPathPlaceholder', - 'OutputPathPlaceholder', - 'InputUriPlaceholder', - 'OutputUriPlaceholder', - 'InputMetadataPlaceholder', - 'InputOutputPortNamePlaceholder', - 'OutputMetadataPlaceholder', - 'ExecutorInputPlaceholder', - 'ConcatPlaceholder', - 'IsPresentPlaceholder', - 'IfPlaceholderStructure', - 'IfPlaceholder', - 'ContainerSpec', - 'ContainerImplementation', - 'ComponentSpec', - 'ComponentReference', - 'GraphInputReference', - 'GraphInputArgument', - 'TaskOutputReference', - 'TaskOutputArgument', - 'EqualsPredicate', - 'NotEqualsPredicate', - 'GreaterThanPredicate', - 'GreaterThanOrEqualPredicate', - 'LessThenPredicate', - 'LessThenOrEqualPredicate', - 'NotPredicate', - 'AndPredicate', - 'OrPredicate', - 'RetryStrategySpec', - 'CachingStrategySpec', - 'ExecutionOptionsSpec', - 'TaskSpec', - 'GraphSpec', - 'GraphImplementation', - 'PipelineRunSpec', -] - -from collections import OrderedDict - -from typing import Any, Dict, List, Mapping, Optional, Union - -from .modelbase import ModelBase - -PrimitiveTypes = Union[str, int, float, bool] -PrimitiveTypesIncludingNone = Optional[PrimitiveTypes] - -TypeSpecType = Union[str, Dict, List] - - -class InputSpec(ModelBase): - """Describes the component input specification.""" - - def __init__( - self, - name: str, - type: Optional[TypeSpecType] = None, - description: Optional[str] = None, - default: Optional[PrimitiveTypes] = None, - optional: Optional[bool] = False, - annotations: Optional[Dict[str, Any]] = None, - ): - super().__init__(locals()) - - -class OutputSpec(ModelBase): - """Describes the component output specification.""" - - def __init__( - self, - name: str, - type: Optional[TypeSpecType] = None, - description: Optional[str] = None, - annotations: Optional[Dict[str, Any]] = None, - ): - super().__init__(locals()) - - -class InputValuePlaceholder(ModelBase): #Non-standard attr names - """Represents the command-line argument placeholder that will be replaced - at run-time by the input argument value.""" - _serialized_names = { - 'input_name': 'inputValue', - } - - def __init__( - self, - input_name: str, - ): - super().__init__(locals()) - - -class InputPathPlaceholder(ModelBase): #Non-standard attr names - """Represents the command-line argument placeholder that will be replaced - at run-time by a local file path pointing to a file containing the input - argument value.""" - _serialized_names = { - 'input_name': 'inputPath', - } - - def __init__( - self, - input_name: str, - ): - super().__init__(locals()) - - -class OutputPathPlaceholder(ModelBase): #Non-standard attr names - """Represents the command-line argument placeholder that will be replaced - at run-time by a local file path pointing to a file where the program - should write its output data.""" - _serialized_names = { - 'output_name': 'outputPath', - } - - def __init__( - self, - output_name: str, - ): - super().__init__(locals()) - - -class InputUriPlaceholder(ModelBase): # Non-standard attr names - """Represents a placeholder for the URI of an input artifact. - - Represents the command-line argument placeholder that will be - replaced at run-time by the URI of the input artifact argument. - """ - _serialized_names = { - 'input_name': 'inputUri', - } - - def __init__( - self, - input_name: str, - ): - super().__init__(locals()) - - -class OutputUriPlaceholder(ModelBase): # Non-standard attr names - """Represents a placeholder for the URI of an output artifact. - - Represents the command-line argument placeholder that will be - replaced at run-time by a URI of the output artifac where the - program should write its output data. - """ - _serialized_names = { - 'output_name': 'outputUri', - } - - def __init__( - self, - output_name: str, - ): - super().__init__(locals()) - - -class InputMetadataPlaceholder(ModelBase): # Non-standard attr names - """Represents the file path to an input artifact metadata. - - During runtime, this command-line argument placeholder will be - replaced by the path where the metadata file associated with this - artifact has been written to. Currently only supported in v2 - components. - """ - _serialized_names = { - 'input_name': 'inputMetadata', - } - - def __init__(self, input_name: str): - super().__init__(locals()) - - -class InputOutputPortNamePlaceholder(ModelBase): # Non-standard attr names - """Represents the output port name of an input artifact. - - During compile time, this command-line argument placeholder will be - replaced by the actual output port name used by the producer task. - Currently only supported in v2 components. - """ - _serialized_names = { - 'input_name': 'inputOutputPortName', - } - - def __init__(self, input_name: str): - super().__init__(locals()) - - -class OutputMetadataPlaceholder(ModelBase): # Non-standard attr names - """Represents the output metadata JSON file location of this task. - - This file will encode the metadata information produced by this task: - - Artifacts metadata, but not the content of the artifact, and - - output parameters. - - Only supported in v2 components. - """ - _serialized_names = { - 'output_metadata': 'outputMetadata', - } - - def __init__(self, output_metadata: type(None) = None): - if output_metadata: - raise RuntimeError( - 'Output metadata placeholder cannot be associated with key') - super().__init__(locals()) - - def to_dict(self) -> Mapping[str, Any]: - # Override parent implementation. Otherwise it always returns {}. - return {'outputMetadata': None} - - -class ExecutorInputPlaceholder(ModelBase): # Non-standard attr names - """Represents the serialized ExecutorInput message at runtime. - - This placeholder will be replaced by a serialized - [ExecutorInput](https://github.com/kubeflow/pipelines/blob/61f9c2c328d245d89c9d9b8c923f24dbbd08cdc9/api/v2alpha1/pipeline_spec.proto#L730) - proto message at runtime, which includes parameters of the task, artifact - URIs and metadata. - """ - _serialized_names = { - 'executor_input': 'executorInput', - } - - def __init__(self, executor_input: type(None) = None): - if executor_input: - raise RuntimeError( - 'Executor input placeholder cannot be associated with input key' - '. Got %s' % executor_input) - super().__init__(locals()) - - def to_dict(self) -> Mapping[str, Any]: - # Override parent implementation. Otherwise it always returns {}. - return {'executorInput': None} - - -CommandlineArgumentType = Union[str, InputValuePlaceholder, - InputPathPlaceholder, OutputPathPlaceholder, - InputUriPlaceholder, OutputUriPlaceholder, - InputMetadataPlaceholder, - InputOutputPortNamePlaceholder, - OutputMetadataPlaceholder, - ExecutorInputPlaceholder, 'ConcatPlaceholder', - 'IfPlaceholder',] - - -class ConcatPlaceholder(ModelBase): #Non-standard attr names - """Represents the command-line argument placeholder that will be replaced - at run-time by the concatenated values of its items.""" - _serialized_names = { - 'items': 'concat', - } - - def __init__( - self, - items: List[CommandlineArgumentType], - ): - super().__init__(locals()) - - -class IsPresentPlaceholder(ModelBase): #Non-standard attr names - """Represents the command-line argument placeholder that will be replaced - at run-time by a boolean value specifying whether the caller has passed an - argument for the specified optional input.""" - _serialized_names = { - 'input_name': 'isPresent', - } - - def __init__( - self, - input_name: str, - ): - super().__init__(locals()) - - -IfConditionArgumentType = Union[bool, str, IsPresentPlaceholder, - InputValuePlaceholder] - - -class IfPlaceholderStructure(ModelBase): #Non-standard attr names - '''Used in by the IfPlaceholder - the command-line argument placeholder that will be replaced at run-time by the expanded value of either "then_value" or "else_value" depending on the submissio-time resolved value of the "cond" predicate.''' - _serialized_names = { - 'condition': 'cond', - 'then_value': 'then', - 'else_value': 'else', - } - - def __init__( - self, - condition: IfConditionArgumentType, - then_value: Union[CommandlineArgumentType, - List[CommandlineArgumentType]], - else_value: Optional[Union[CommandlineArgumentType, - List[CommandlineArgumentType]]] = None, - ): - super().__init__(locals()) - - -class IfPlaceholder(ModelBase): #Non-standard attr names - """Represents the command-line argument placeholder that will be replaced - at run-time by the expanded value of either "then_value" or "else_value" - depending on the submissio-time resolved value of the "cond" predicate.""" - _serialized_names = { - 'if_structure': 'if', - } - - def __init__( - self, - if_structure: IfPlaceholderStructure, - ): - super().__init__(locals()) - - -class ContainerSpec(ModelBase): - """Describes the container component implementation.""" - _serialized_names = { - 'file_outputs': - 'fileOutputs', #TODO: rename to something like legacy_unconfigurable_output_paths - } - - def __init__( - self, - image: str, - command: Optional[List[CommandlineArgumentType]] = None, - args: Optional[List[CommandlineArgumentType]] = None, - env: Optional[Mapping[str, str]] = None, - file_outputs: - Optional[Mapping[ - str, - str]] = None, #TODO: rename to something like legacy_unconfigurable_output_paths - ): - super().__init__(locals()) - - -class ContainerImplementation(ModelBase): - """Represents the container component implementation.""" - - def __init__( - self, - container: ContainerSpec, - ): - super().__init__(locals()) - - -ImplementationType = Union[ContainerImplementation, 'GraphImplementation'] - - -class MetadataSpec(ModelBase): - - def __init__( - self, - annotations: Optional[Dict[str, str]] = None, - labels: Optional[Dict[str, str]] = None, - ): - super().__init__(locals()) - - -class ComponentSpec(ModelBase): - """Component specification. - - Describes the metadata (name, description, annotations and labels), - the interface (inputs and outputs) and the implementation of the - component. - """ - - def __init__( - self, - name: Optional[str] = None, #? Move to metadata? - description: Optional[str] = None, #? Move to metadata? - metadata: Optional[MetadataSpec] = None, - inputs: Optional[List[InputSpec]] = None, - outputs: Optional[List[OutputSpec]] = None, - implementation: Optional[ImplementationType] = None, - version: Optional[str] = 'google.com/cloud/pipelines/component/v1', - #tags: Optional[Set[str]] = None, - ): - super().__init__(locals()) - self._post_init() - - def _post_init(self): - #Checking input names for uniqueness - self._inputs_dict = {} - if self.inputs: - for input in self.inputs: - if input.name in self._inputs_dict: - raise ValueError('Non-unique input name "{}"'.format( - input.name)) - self._inputs_dict[input.name] = input - - #Checking output names for uniqueness - self._outputs_dict = {} - if self.outputs: - for output in self.outputs: - if output.name in self._outputs_dict: - raise ValueError('Non-unique output name "{}"'.format( - output.name)) - self._outputs_dict[output.name] = output - - if isinstance(self.implementation, ContainerImplementation): - container = self.implementation.container - - if container.file_outputs: - for output_name, path in container.file_outputs.items(): - if output_name not in self._outputs_dict: - raise TypeError( - 'Unconfigurable output entry "{}" references non-existing output.' - .format({output_name: path})) - - def verify_arg(arg): - if arg is None: - pass - elif isinstance( - arg, (str, int, float, bool, OutputMetadataPlaceholder, - ExecutorInputPlaceholder)): - pass - elif isinstance(arg, list): - for arg2 in arg: - verify_arg(arg2) - elif isinstance( - arg, - (InputUriPlaceholder, InputValuePlaceholder, - InputPathPlaceholder, IsPresentPlaceholder, - InputMetadataPlaceholder, InputOutputPortNamePlaceholder)): - if arg.input_name not in self._inputs_dict: - raise TypeError( - 'Argument "{}" references non-existing input.' - .format(arg)) - elif isinstance(arg, - (OutputUriPlaceholder, OutputPathPlaceholder)): - if arg.output_name not in self._outputs_dict: - raise TypeError( - 'Argument "{}" references non-existing output.' - .format(arg)) - elif isinstance(arg, ConcatPlaceholder): - for arg2 in arg.items: - verify_arg(arg2) - elif isinstance(arg, IfPlaceholder): - verify_arg(arg.if_structure.condition) - verify_arg(arg.if_structure.then_value) - verify_arg(arg.if_structure.else_value) - else: - raise TypeError('Unexpected argument "{}"'.format(arg)) - - verify_arg(container.command) - verify_arg(container.args) - - if isinstance(self.implementation, GraphImplementation): - graph = self.implementation.graph - - if graph.output_values is not None: - for output_name, argument in graph.output_values.items(): - if output_name not in self._outputs_dict: - raise TypeError( - 'Graph output argument entry "{}" references non-existing output.' - .format({output_name: argument})) - - if graph.tasks is not None: - for task in graph.tasks.values(): - if task.arguments is not None: - for argument in task.arguments.values(): - if isinstance( - argument, GraphInputArgument - ) and argument.graph_input.input_name not in self._inputs_dict: - raise TypeError( - 'Argument "{}" references non-existing input.' - .format(argument)) - - def save(self, file_path: str): - """Saves the component definition to file. - - It can be shared online and later loaded using the - load_component function. - """ - from ._yaml_utils import dump_yaml - component_yaml = dump_yaml(self.to_dict()) - with open(file_path, 'w') as f: - f.write(component_yaml) - - -class ComponentReference(ModelBase): - """Component reference. - - Contains information that can be used to locate and load a component - by name, digest or URL - """ - - def __init__( - self, - name: Optional[str] = None, - digest: Optional[str] = None, - tag: Optional[str] = None, - url: Optional[str] = None, - spec: Optional[ComponentSpec] = None, - ): - super().__init__(locals()) - self._post_init() - - def _post_init(self) -> None: - if not any([self.name, self.digest, self.tag, self.url, self.spec]): - raise TypeError('Need at least one argument.') - - -class GraphInputReference(ModelBase): - """References the input of the graph (the scope is a single graph).""" - _serialized_names = { - 'input_name': 'inputName', - } - - def __init__( - self, - input_name: str, - type: - Optional[ - TypeSpecType] = None, # Can be used to override the reference data type - ): - super().__init__(locals()) - - def as_argument(self) -> 'GraphInputArgument': - return GraphInputArgument(graph_input=self) - - def with_type(self, type_spec: TypeSpecType) -> 'GraphInputReference': - return GraphInputReference( - input_name=self.input_name, - type=type_spec, - ) - - def without_type(self) -> 'GraphInputReference': - return self.with_type(None) - - -class GraphInputArgument(ModelBase): - """Represents the component argument value that comes from the graph - component input.""" - _serialized_names = { - 'graph_input': 'graphInput', - } - - def __init__( - self, - graph_input: GraphInputReference, - ): - super().__init__(locals()) - - -class TaskOutputReference(ModelBase): - """References the output of some task (the scope is a single graph).""" - _serialized_names = { - 'task_id': 'taskId', - 'output_name': 'outputName', - } - - def __init__( - self, - output_name: str, - task_id: - Optional[ - str] = None, # Used for linking to the upstream task in serialized component file. - task: - Optional[ - 'TaskSpec'] = None, # Used for linking to the upstream task in runtime since Task does not have an ID until inserted into a graph. - type: - Optional[ - TypeSpecType] = None, # Can be used to override the reference data type - ): - super().__init__(locals()) - if self.task_id is None and self.task is None: - raise TypeError('task_id and task cannot be None at the same time.') - - def with_type(self, type_spec: TypeSpecType) -> 'TaskOutputReference': - return TaskOutputReference( - output_name=self.output_name, - task_id=self.task_id, - task=self.task, - type=type_spec, - ) - - def without_type(self) -> 'TaskOutputReference': - return self.with_type(None) - - -class TaskOutputArgument(ModelBase - ): #Has additional constructor for convenience - """Represents the component argument value that comes from the output of - another task.""" - _serialized_names = { - 'task_output': 'taskOutput', - } - - def __init__( - self, - task_output: TaskOutputReference, - ): - super().__init__(locals()) - - @staticmethod - def construct( - task_id: str, - output_name: str, - ) -> 'TaskOutputArgument': - return TaskOutputArgument( - TaskOutputReference( - task_id=task_id, - output_name=output_name, - )) - - def with_type(self, type_spec: TypeSpecType) -> 'TaskOutputArgument': - return TaskOutputArgument( - task_output=self.task_output.with_type(type_spec),) - - def without_type(self) -> 'TaskOutputArgument': - return self.with_type(None) - - -ArgumentType = Union[PrimitiveTypes, GraphInputArgument, TaskOutputArgument] - - -class TwoOperands(ModelBase): - - def __init__( - self, - op1: ArgumentType, - op2: ArgumentType, - ): - super().__init__(locals()) - - -class BinaryPredicate(ModelBase): #abstract base type - - def __init__(self, operands: TwoOperands): - super().__init__(locals()) - - -class EqualsPredicate(BinaryPredicate): - """Represents the "equals" comparison predicate.""" - _serialized_names = {'operands': '=='} - - -class NotEqualsPredicate(BinaryPredicate): - """Represents the "not equals" comparison predicate.""" - _serialized_names = {'operands': '!='} - - -class GreaterThanPredicate(BinaryPredicate): - """Represents the "greater than" comparison predicate.""" - _serialized_names = {'operands': '>'} - - -class GreaterThanOrEqualPredicate(BinaryPredicate): - """Represents the "greater than or equal" comparison predicate.""" - _serialized_names = {'operands': '>='} - - -class LessThenPredicate(BinaryPredicate): - """Represents the "less than" comparison predicate.""" - _serialized_names = {'operands': '<'} - - -class LessThenOrEqualPredicate(BinaryPredicate): - """Represents the "less than or equal" comparison predicate.""" - _serialized_names = {'operands': '<='} - - -PredicateType = Union[ArgumentType, EqualsPredicate, NotEqualsPredicate, - GreaterThanPredicate, GreaterThanOrEqualPredicate, - LessThenPredicate, LessThenOrEqualPredicate, - 'NotPredicate', 'AndPredicate', 'OrPredicate',] - - -class TwoBooleanOperands(ModelBase): - - def __init__( - self, - op1: PredicateType, - op2: PredicateType, - ): - super().__init__(locals()) - - -class NotPredicate(ModelBase): - """Represents the "not" logical operation.""" - _serialized_names = {'operand': 'not'} - - def __init__(self, operand: PredicateType): - super().__init__(locals()) - - -class AndPredicate(ModelBase): - """Represents the "and" logical operation.""" - _serialized_names = {'operands': 'and'} - - def __init__(self, operands: TwoBooleanOperands): - super().__init__(locals()) - - -class OrPredicate(ModelBase): - """Represents the "or" logical operation.""" - _serialized_names = {'operands': 'or'} - - def __init__(self, operands: TwoBooleanOperands): - super().__init__(locals()) - - -class RetryStrategySpec(ModelBase): - _serialized_names = { - 'max_retries': 'maxRetries', - } - - def __init__( - self, - max_retries: int, - ): - super().__init__(locals()) - - -class CachingStrategySpec(ModelBase): - _serialized_names = { - 'max_cache_staleness': 'maxCacheStaleness', - } - - def __init__( - self, - max_cache_staleness: Optional[ - str] = None, # RFC3339 compliant duration: P30DT1H22M3S - ): - super().__init__(locals()) - - -class ExecutionOptionsSpec(ModelBase): - _serialized_names = { - 'retry_strategy': 'retryStrategy', - 'caching_strategy': 'cachingStrategy', - } - - def __init__( - self, - retry_strategy: Optional[RetryStrategySpec] = None, - caching_strategy: Optional[CachingStrategySpec] = None, - ): - super().__init__(locals()) - - -class TaskSpec(ModelBase): - """Task specification. - - Task is a "configured" component - a component supplied with arguments and other applied configuration changes. - """ - _serialized_names = { - 'component_ref': 'componentRef', - 'is_enabled': 'isEnabled', - 'execution_options': 'executionOptions' - } - - def __init__( - self, - component_ref: ComponentReference, - arguments: Optional[Mapping[str, ArgumentType]] = None, - is_enabled: Optional[PredicateType] = None, - execution_options: Optional[ExecutionOptionsSpec] = None, - annotations: Optional[Dict[str, Any]] = None, - ): - super().__init__(locals()) - #TODO: If component_ref is resolved to component spec, then check that the arguments correspond to the inputs - - def _init_outputs(self): - #Adding output references to the task - if self.component_ref.spec is None: - return - task_outputs = OrderedDict() - for output in self.component_ref.spec.outputs or []: - task_output_ref = TaskOutputReference( - output_name=output.name, - task=self, - type=output. - type, # TODO: Resolve type expressions. E.g. type: {TypeOf: Input 1} - ) - task_output_arg = TaskOutputArgument(task_output=task_output_ref) - task_outputs[output.name] = task_output_arg - - self.outputs = task_outputs - if len(task_outputs) == 1: - self.output = list(task_outputs.values())[0] - - -class GraphSpec(ModelBase): - """Describes the graph component implementation. - - It represents a graph of component tasks connected to the upstream - sources of data using the argument specifications. It also describes - the sources of graph output values. - """ - _serialized_names = { - 'output_values': 'outputValues', - } - - def __init__( - self, - tasks: Mapping[str, TaskSpec], - output_values: Mapping[str, ArgumentType] = None, - ): - super().__init__(locals()) - self._post_init() - - def _post_init(self): - #Checking task output references and preparing the dependency table - task_dependencies = {} - for task_id, task in self.tasks.items(): - dependencies = set() - task_dependencies[task_id] = dependencies - if task.arguments is not None: - for argument in task.arguments.values(): - if isinstance(argument, TaskOutputArgument): - dependencies.add(argument.task_output.task_id) - if argument.task_output.task_id not in self.tasks: - raise TypeError( - 'Argument "{}" references non-existing task.' - .format(argument)) - - #Topologically sorting tasks to detect cycles - task_dependents = {k: set() for k in task_dependencies.keys()} - for task_id, dependencies in task_dependencies.items(): - for dependency in dependencies: - task_dependents[dependency].add(task_id) - task_number_of_remaining_dependencies = { - k: len(v) for k, v in task_dependencies.items() - } - sorted_tasks = OrderedDict() - - def process_task(task_id): - if task_number_of_remaining_dependencies[ - task_id] == 0 and task_id not in sorted_tasks: - sorted_tasks[task_id] = self.tasks[task_id] - for dependent_task in task_dependents[task_id]: - task_number_of_remaining_dependencies[ - dependent_task] = task_number_of_remaining_dependencies[ - dependent_task] - 1 - process_task(dependent_task) - - for task_id in task_dependencies.keys(): - process_task(task_id) - if len(sorted_tasks) != len(task_dependencies): - tasks_with_unsatisfied_dependencies = { - k: v - for k, v in task_number_of_remaining_dependencies.items() - if v > 0 - } - task_wth_minimal_number_of_unsatisfied_dependencies = min( - tasks_with_unsatisfied_dependencies.keys(), - key=lambda task_id: tasks_with_unsatisfied_dependencies[task_id] - ) - raise ValueError('Task "{}" has cyclical dependency.'.format( - task_wth_minimal_number_of_unsatisfied_dependencies)) - - self._toposorted_tasks = sorted_tasks - - -class GraphImplementation(ModelBase): - """Represents the graph component implementation.""" - - def __init__( - self, - graph: GraphSpec, - ): - super().__init__(locals()) - - -class PipelineRunSpec(ModelBase): - """The object that can be sent to the backend to start a new Run.""" - _serialized_names = { - 'root_task': 'rootTask', - #'on_exit_task': 'onExitTask', - } - - def __init__( - self, - root_task: TaskSpec, - #on_exit_task: Optional[TaskSpec] = None, - ): - super().__init__(locals()) diff --git a/sdk/python/kfp/deprecated/components/_yaml_utils.py b/sdk/python/kfp/deprecated/components/_yaml_utils.py deleted file mode 100644 index 5dd0d42eac6..00000000000 --- a/sdk/python/kfp/deprecated/components/_yaml_utils.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import yaml -from collections import OrderedDict - - -def load_yaml(stream): - #!!! Yaml should only be loaded using this function. Otherwise the dict ordering may be broken in Python versions prior to 3.6 - #See https://stackoverflow.com/questions/5121931/in-python-how-can-you-load-yaml-mappings-as-ordereddicts/21912744#21912744 - - def ordered_load(stream, - Loader=yaml.SafeLoader, - object_pairs_hook=OrderedDict): - - class OrderedLoader(Loader): - pass - - def construct_mapping(loader, node): - loader.flatten_mapping(node) - return object_pairs_hook(loader.construct_pairs(node)) - - OrderedLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) - return yaml.load(stream, OrderedLoader) - - return ordered_load(stream) - - -def dump_yaml(data): - #See https://stackoverflow.com/questions/5121931/in-python-how-can-you-load-yaml-mappings-as-ordereddicts/21912744#21912744 - - def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): - - class OrderedDumper(Dumper): - pass - - def _dict_representer(dumper, data): - return dumper.represent_mapping( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items()) - - OrderedDumper.add_representer(OrderedDict, _dict_representer) - OrderedDumper.add_representer(dict, _dict_representer) - - #Hack to force the code (multi-line string) to be output using the '|' style. - def represent_str_or_text(self, data): - style = None - if data.find('\n') >= 0: #Multiple lines - #print('Switching style for multiline text:' + data) - style = '|' - if data.lower() in [ - 'y', 'n', 'yes', 'no', 'true', 'false', 'on', 'off' - ]: - style = '"' - return self.represent_scalar(u'tag:yaml.org,2002:str', data, style) - - OrderedDumper.add_representer(str, represent_str_or_text) - - return yaml.dump(data, stream, OrderedDumper, **kwds) - - return ordered_dump(data, default_flow_style=None) diff --git a/sdk/python/kfp/deprecated/components/modelbase.py b/sdk/python/kfp/deprecated/components/modelbase.py deleted file mode 100644 index b329c7939c4..00000000000 --- a/sdk/python/kfp/deprecated/components/modelbase.py +++ /dev/null @@ -1,394 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__all__ = [ - 'ModelBase', -] - -import inspect -from collections import abc, OrderedDict -from typing import Any, Dict, List, Mapping, MutableMapping, MutableSequence, Sequence, Type, TypeVar, Union, cast, get_type_hints - -T = TypeVar('T') - - -def verify_object_against_type(x: Any, typ: Type[T]) -> T: - """Verifies that the object is compatible to the specified type (types from - the typing package can be used).""" - #TODO: Merge with parse_object_from_struct_based_on_type which has almost the same code - if typ is type(None): - if x is None: - return x - else: - raise TypeError('Error: Object "{}" is not None.'.format(x)) - - if typ is Any or type(typ) is TypeVar: - return x - - try: #isinstance can fail for generics - if isinstance(x, typ): - return cast(typ, x) - except: - pass - - if hasattr(typ, '__origin__'): #Handling generic types - if typ.__origin__ is Union: #Optional == Union - exception_map = {} - possible_types = typ.__args__ - if type( - None - ) in possible_types and x is None: #Shortcut for Optional[] tests. Can be removed, but the exceptions will be more noisy. - return x - for possible_type in possible_types: - try: - verify_object_against_type(x, possible_type) - return x - except Exception as ex: - exception_map[possible_type] = ex - #exception_lines = ['Exception for type {}: {}.'.format(t, e) for t, e in exception_map.items()] - exception_lines = [str(e) for t, e in exception_map.items()] - exception_lines.append( - 'Error: Object "{}" is incompatible with type "{}".'.format( - x, typ)) - raise TypeError('\n'.join(exception_lines)) - - #not Union => not None - if x is None: - raise TypeError( - 'Error: None object is incompatible with type {}'.format(typ)) - - #assert isinstance(x, typ.__origin__) - generic_type = typ.__origin__ or getattr( - typ, '__extra__', None - ) #In python <3.7 typing.List.__origin__ == None; Python 3.7 has working __origin__, but no __extra__ TODO: Remove the __extra__ once we move to Python 3.7 - if generic_type in [ - list, List, abc.Sequence, abc.MutableSequence, Sequence, - MutableSequence - ] and type(x) is not str: #! str is also Sequence - if not isinstance(x, generic_type): - raise TypeError( - 'Error: Object "{}" is incompatible with type "{}"'.format( - x, typ)) - # In Python <3.7 Mapping.__args__ is None. - # In Python 3.9 typ.__args__ does not exist when the generic type does not have subscripts - type_args = typ.__args__ if getattr( - typ, '__args__', None) is not None else (Any, Any) - inner_type = type_args[0] - for item in x: - verify_object_against_type(item, inner_type) - return x - - elif generic_type in [ - dict, Dict, abc.Mapping, abc.MutableMapping, Mapping, - MutableMapping, OrderedDict - ]: - if not isinstance(x, generic_type): - raise TypeError( - 'Error: Object "{}" is incompatible with type "{}"'.format( - x, typ)) - # In Python <3.7 Mapping.__args__ is None. - # In Python 3.9 typ.__args__ does not exist when the generic type does not have subscripts - type_args = typ.__args__ if getattr( - typ, '__args__', None) is not None else (Any, Any) - inner_key_type = type_args[0] - inner_value_type = type_args[1] - for k, v in x.items(): - verify_object_against_type(k, inner_key_type) - verify_object_against_type(v, inner_value_type) - return x - - else: - raise TypeError( - 'Error: Unsupported generic type "{}". type.__origin__ or type.__extra__ == "{}"' - .format(typ, generic_type)) - - raise TypeError('Error: Object "{}" is incompatible with type "{}"'.format( - x, typ)) - - -def parse_object_from_struct_based_on_type(struct: Any, typ: Type[T]) -> T: - """Constructs an object from structure (usually dict) based on type. - - Supports list and dict types from the typing package plus Optional[] - and Union[] types. If some type is a class that has .from_dict class - method, that method is used for object construction. - """ - if typ is type(None): - if struct is None: - return None - else: - raise TypeError('Error: Structure "{}" is not None.'.format(struct)) - - if typ is Any or type(typ) is TypeVar: - return struct - - try: #isinstance can fail for generics - #if (isinstance(struct, typ) - # and not (typ is Sequence and type(struct) is str) #! str is also Sequence - # and not (typ is int and type(struct) is bool) #! bool is int - #): - if type(struct) is typ: - return struct - except: - pass - if hasattr(typ, 'from_dict'): - try: #More informative errors - return typ.from_dict(struct) - except Exception as ex: - raise TypeError( - 'Error: {}.from_dict(struct={}) failed with exception:\n{}' - .format(typ.__name__, struct, str(ex))) - if hasattr(typ, '__origin__'): #Handling generic types - if typ.__origin__ is Union: #Optional == Union - results = {} - exception_map = {} - # In Python 3.9 typ.__args__ does not exist when the generic type does not have subscripts - # Union without subscripts seems useless, but semantically it should be the same as Any. - possible_types = list(getattr(typ, '__args__', [Any])) - #if type(None) in possible_types and struct is None: #Shortcut for Optional[] tests. Can be removed, but the exceptions will be more noisy. - # return None - #Hack for Python <3.7 which for some reason "simplifies" Union[bool, int, ...] to just Union[int, ...] - if int in possible_types: - possible_types = possible_types + [bool] - for possible_type in possible_types: - try: - obj = parse_object_from_struct_based_on_type( - struct, possible_type) - results[possible_type] = obj - except Exception as ex: - if isinstance(ex, TypeError): - exception_map[possible_type] = ex - else: - exception_map[ - possible_type] = 'Unexpected exception when trying to convert structure "{}" to type "{}": {}: {}'.format( - struct, typ, type(ex), ex) - - #Single successful parsing. - if len(results) == 1: - return list(results.values())[0] - - if len(results) > 1: - raise TypeError( - 'Error: Structure "{}" is ambiguous. It can be parsed to multiple types: {}.' - .format(struct, list(results.keys()))) - - exception_lines = [str(e) for t, e in exception_map.items()] - exception_lines.append( - 'Error: Structure "{}" is incompatible with type "{}" - none of the types in Union are compatible.' - .format(struct, typ)) - raise TypeError('\n'.join(exception_lines)) - #not Union => not None - if struct is None: - raise TypeError( - 'Error: None structure is incompatible with type {}'.format( - typ)) - - #assert isinstance(x, typ.__origin__) - generic_type = typ.__origin__ or getattr( - typ, '__extra__', None - ) #In python <3.7 typing.List.__origin__ == None; Python 3.7 has working __origin__, but no __extra__ TODO: Remove the __extra__ once we move to Python 3.7 - if generic_type in [ - list, List, abc.Sequence, abc.MutableSequence, Sequence, - MutableSequence - ] and type(struct) is not str: #! str is also Sequence - if not isinstance(struct, generic_type): - raise TypeError( - 'Error: Structure "{}" is incompatible with type "{}" - it does not have list type.' - .format(struct, typ)) - # In Python <3.7 Mapping.__args__ is None. - # In Python 3.9 typ.__args__ does not exist when the generic type does not have subscripts - type_args = typ.__args__ if getattr( - typ, '__args__', None) is not None else (Any, Any) - inner_type = type_args[0] - return [ - parse_object_from_struct_based_on_type(item, inner_type) - for item in struct - ] - - elif generic_type in [ - dict, Dict, abc.Mapping, abc.MutableMapping, Mapping, - MutableMapping, OrderedDict - ]: #in Python <3.7 there is a difference between abc.Mapping and typing.Mapping - if not isinstance(struct, generic_type): - raise TypeError( - 'Error: Structure "{}" is incompatible with type "{}" - it does not have dict type.' - .format(struct, typ)) - # In Python <3.7 Mapping.__args__ is None. - # In Python 3.9 typ.__args__ does not exist when the generic type does not have subscripts - type_args = typ.__args__ if getattr( - typ, '__args__', None) is not None else (Any, Any) - inner_key_type = type_args[0] - inner_value_type = type_args[1] - return { - parse_object_from_struct_based_on_type(k, inner_key_type): - parse_object_from_struct_based_on_type(v, inner_value_type) - for k, v in struct.items() - } - - else: - raise TypeError( - 'Error: Unsupported generic type "{}". type.__origin__ or type.__extra__ == "{}"' - .format(typ, generic_type)) - - raise TypeError( - 'Error: Structure "{}" is incompatible with type "{}". Structure is not the instance of the type, the type does not have .from_dict method and is not generic.' - .format(struct, typ)) - - -def convert_object_to_struct(obj, serialized_names: Mapping[str, str] = {}): - """Converts an object to structure (usually a dict). - - Serializes all properties that do not start with underscores. If the - type of some property is a class that has .to_dict class method, - that method is used for conversion. Used by the ModelBase class. - """ - signature = inspect.signature(obj.__init__) #Needed for default values - result = {} - for python_name in signature.parameters: #TODO: Make it possible to specify the field ordering regardless of the presence of default values - value = getattr(obj, python_name) - if python_name.startswith('_'): - continue - attr_name = serialized_names.get(python_name, python_name) - if hasattr(value, "to_dict"): - result[attr_name] = value.to_dict() - elif isinstance(value, list): - result[attr_name] = [ - (x.to_dict() if hasattr(x, 'to_dict') else x) for x in value - ] - elif isinstance(value, dict): - result[attr_name] = { - k: (v.to_dict() if hasattr(v, 'to_dict') else v) - for k, v in value.items() - } - else: - param = signature.parameters.get(python_name, None) - if param is None or param.default == inspect.Parameter.empty or value != param.default: - result[attr_name] = value - - return result - - -def parse_object_from_struct_based_on_class_init( - cls: Type[T], - struct: Mapping, - serialized_names: Mapping[str, str] = {}) -> T: - """Constructs an object of specified class from structure (usually dict) - using the class.__init__ method. Converts all constructor arguments to - appropriate types based on the __init__ type hints. Used by the ModelBase - class. - - Arguments: - - serialized_names: specifies the mapping between __init__ parameter names and the structure key names for cases where these names are different (due to language syntax clashes or style differences). - """ - parameter_types = get_type_hints( - cls.__init__) #Properlty resolves forward references - - serialized_names_to_pythonic = {v: k for k, v in serialized_names.items()} - #If a pythonic name has a different original name, we forbid the pythonic name in the structure. Otherwise, this function would accept "python-styled" structures that should be invalid - forbidden_struct_keys = set( - serialized_names_to_pythonic.values()).difference( - serialized_names_to_pythonic.keys()) - args = {} - for original_name, value in struct.items(): - if original_name in forbidden_struct_keys: - raise ValueError( - 'Use "{}" key instead of pythonic key "{}" in the structure: {}.' - .format(serialized_names[original_name], original_name, struct)) - python_name = serialized_names_to_pythonic.get(original_name, - original_name) - param_type = parameter_types.get(python_name, None) - if param_type is not None: - args[python_name] = parse_object_from_struct_based_on_type( - value, param_type) - else: - args[python_name] = value - - return cls(**args) - - -class ModelBase: - """Base class for types that can be converted to JSON-like dict structures - or constructed from such structures. The object fields, their types and - default values are taken from the __init__ method arguments. Override the - _serialized_names mapping to control the key names of the serialized - structures. - - The derived class objects will have the .from_dict and .to_dict methods for conversion to or from structure. The base class constructor accepts the arguments map, checks the argument types and sets the object field values. - - Example derived class: - - class TaskSpec(ModelBase): - _serialized_names = { - 'component_ref': 'componentRef', - 'is_enabled': 'isEnabled', - } - - def __init__(self, - component_ref: ComponentReference, - arguments: Optional[Mapping[str, ArgumentType]] = None, - is_enabled: Optional[Union[ArgumentType, EqualsPredicate, NotEqualsPredicate]] = None, #Optional property with default value - ): - super().__init__(locals()) #Calling the ModelBase constructor to check the argument types and set the object field values. - - task_spec = TaskSpec.from_dict("{'componentRef': {...}, 'isEnabled: {'and': {...}}}") # = instance of TaskSpec - task_struct = task_spec.to_dict() #= "{'componentRef': {...}, 'isEnabled: {'and': {...}}}" - """ - _serialized_names = {} - - def __init__(self, args): - parameter_types = get_type_hints(self.__class__.__init__) - field_values = { - k: v - for k, v in args.items() - if k != 'self' and not k.startswith('_') - } - for k, v in field_values.items(): - parameter_type = parameter_types.get(k, None) - if parameter_type is not None: - try: - verify_object_against_type(v, parameter_type) - except Exception as e: - raise TypeError( - 'Argument for {} is not compatible with type "{}". Exception: {}' - .format(k, parameter_type, e)) - self.__dict__.update(field_values) - - @classmethod - def from_dict(cls: Type[T], struct: Mapping) -> T: - return parse_object_from_struct_based_on_class_init( - cls, struct, serialized_names=cls._serialized_names) - - def to_dict(self) -> Mapping: - return convert_object_to_struct( - self, serialized_names=self._serialized_names) - - def _get_field_names(self): - return list(inspect.signature(self.__init__).parameters) - - def __repr__(self): - return self.__class__.__name__ + '(' + ', '.join( - param + '=' + repr(getattr(self, param)) - for param in self._get_field_names()) + ')' - - def __eq__(self, other): - return self.__class__ == other.__class__ and { - k: getattr(self, k) for k in self._get_field_names() - } == {k: getattr(other, k) for k in other._get_field_names()} - - def __ne__(self, other): - return not self == other - - def __hash__(self): - return hash(repr(self)) diff --git a/sdk/python/kfp/deprecated/components/structures/__init__.py b/sdk/python/kfp/deprecated/components/structures/__init__.py deleted file mode 100644 index fed14559238..00000000000 --- a/sdk/python/kfp/deprecated/components/structures/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .._structures import * diff --git a/sdk/python/kfp/deprecated/components/structures/components.json_schema.json b/sdk/python/kfp/deprecated/components/structures/components.json_schema.json deleted file mode 100644 index dfda1b06615..00000000000 --- a/sdk/python/kfp/deprecated/components/structures/components.json_schema.json +++ /dev/null @@ -1,375 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-06/schema#", - "$id": "http://kubeflow.org/pipelines/components.json_schema.json", - "allOf": [{"$ref": "#/definitions/ComponentSpec"}], - - "definitions": { - "TypeSpecType": { - "oneOf": [ - {"type": "string"}, - {"type": "object", "additionalProperties": {"$ref": "#/definitions/TypeSpecType"}} - ] - }, - - "InputSpec": { - "description": "Describes the component input specification", - "type": "object", - "required": ["name"], - "properties": { - "name": {"type": "string"}, - "type": {"$ref": "#/definitions/TypeSpecType"}, - "description": {"type": "string"}, - "default": {"type": "string"}, - "optional": {"type": "boolean", "default": false} - }, - "additionalProperties": false - }, - - "OutputSpec": { - "description": "Describes the component output specification", - "type": "object", - "required": ["name"], - "properties": { - "name": {"type": "string"}, - "type": {"$ref": "#/definitions/TypeSpecType"}, - "description": {"type": "string"} - }, - "additionalProperties": false - }, - - "InputValuePlaceholder": { - "description": "Represents the command-line argument placeholder that will be replaced at run-time by the input argument value.", - "type": "object", - "required": ["inputValue"], - "properties": { - "inputValue" : { - "description": "Name of the input.", - "type": "string" - } - }, - "additionalProperties": false - }, - - "InputPathPlaceholder": { - "description": "Represents the command-line argument placeholder that will be replaced at run-time by a local file path pointing to a file containing the input argument value.", - "type": "object", - "required": ["inputPath"], - "properties": { - "inputPath" : { - "description": "Name of the input.", - "type": "string" - } - }, - "additionalProperties": false - }, - - "OutputPathPlaceholder": { - "description": "Represents the command-line argument placeholder that will be replaced at run-time by a local file path pointing to a file where the program should write its output data.", - "type": "object", - "required": ["outputPath"], - "properties": { - "outputPath" : { - "description": "Name of the output.", - "type": "string" - } - }, - "additionalProperties": false - }, - - "StringOrPlaceholder": { - "oneOf": [ - {"type": "string"}, - {"$ref": "#/definitions/InputValuePlaceholder"}, - {"$ref": "#/definitions/InputPathPlaceholder"}, - {"$ref": "#/definitions/OutputPathPlaceholder"}, - {"$ref": "#/definitions/ConcatPlaceholder"}, - {"$ref": "#/definitions/IfPlaceholder"} - ] - }, - - "ConcatPlaceholder": { - "description": "Represents the command-line argument placeholder that will be replaced at run-time by the concatenated values of its items.", - "type": "object", - "required": ["concat"], - "properties": { - "concat" : { - "description": "Items to concatenate", - "type": "array", - "items": {"$ref": "#/definitions/StringOrPlaceholder"} - } - }, - "additionalProperties": false - }, - - "IsPresentPlaceholder": { - "description": "Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input.", - "type": "object", - "properties": { - "isPresent": { - "description": "Name of the input.", - "type": "string" - } - }, - "additionalProperties": false - }, - - "IfConditionArgumentType": { - "oneOf": [ - {"$ref": "#/definitions/IsPresentPlaceholder"}, - {"type": "boolean"}, - {"type": "string"}, - {"$ref": "#/definitions/InputValuePlaceholder"} - ] - }, - - "IfPlaceholder": { - "description": "Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input.", - "type": "object", - "required": ["if"], - "properties": { - "if" : { - "type": "object", - "required": ["cond", "then"], - "properties": { - "cond": {"$ref": "#/definitions/IfConditionArgumentType"}, - "then": {"$ref": "#/definitions/StringOrPlaceholder"}, - "else": {"$ref": "#/definitions/StringOrPlaceholder"} - } - } - } - }, - - "ContainerSpec": { - "type": "object", - "required": ["image"], - "properties": { - "image": { - "description": "Docker image name.", - "$ref": "#/definitions/StringOrPlaceholder" - }, - "command": { - "description": "Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided.", - "type": "array", - "items": {"$ref": "#/definitions/StringOrPlaceholder"} - }, - "args": { - "description": "Arguments to the entrypoint. The docker image's CMD is used if this is not provided.", - "type": "array", - "items": {"$ref": "#/definitions/StringOrPlaceholder"} - }, - "env": { - "description": "List of environment variables to set in the container.", - "type": "object", - "additionalProperties": {"$ref": "#/definitions/StringOrPlaceholder"} - }, - "unconfigurableOutputPaths": { - "description": "Legacy. Deprecated. Can be used to specify local output file paths for containers that do not allow setting the path of some output using command-line.", - "type": "object", - "additionalProperties": {"type": "string"} - } - }, - "additionalProperties": false - }, - - "ContainerImplementation": { - "description": "Represents the container component implementation.", - "type": "object", - "required": ["container"], - "properties": { - "container": {"$ref": "#/definitions/ContainerSpec"} - } - }, - - "ImplementationType": { - "oneOf": [ - {"$ref": "#/definitions/ContainerImplementation"}, - {"$ref": "#/definitions/GraphImplementation"} - ] - }, - - "ComponentSpec": { - "description": "Component specification. Describes the metadata (name, description, source), the interface (inputs and outputs) and the implementation of the component.", - "type": "object", - "required": ["implementation"], - "properties": { - "name": {"type": "string"}, - "description": {"type": "string"}, - "inputs": {"type": "array", "items": {"$ref": "#/definitions/InputSpec"}}, - "outputs": {"type": "array", "items": {"$ref": "#/definitions/OutputSpec"}}, - "implementation": {"$ref": "#/definitions/ImplementationType"} - }, - "additionalProperties": false - }, - - "ComponentReference": { - "description": "Component reference. Contains information that can be used to locate and load a component by name, digest or URL", - "type": "object", - "properties": { - "name": {"type": "string"}, - "digest": {"type": "string"}, - "tag": {"type": "string"}, - "url": {"type": "string"}, - "spec": {"$ref": "#/definitions/ComponentSpec"} - }, - "additionalProperties": false - }, - - "GraphInputArgument": { - "description": "Represents the component argument value that comes from the graph component input.", - "type": "object", - "required": ["graphInput"], - "properties": { - "graphInput": { - "description": "References the input of the graph/pipeline.", - "type": "object", - "required": ["inputName"], - "properties": { - "inputName": {"type": "string"}, - "type": {"$ref": "#/definitions/TypeSpecType"} - }, - "additionalProperties": false - } - }, - "additionalProperties": false - }, - - "TaskOutputArgument": { - "description": "Represents the component argument value that comes from the output of a sibling task.", - "type": "object", - "required": ["taskOutput"], - "properties": { - "taskOutput": { - "description": "References the output of a sibling task.", - "type": "object", - "required": ["taskId", "outputName"], - "properties": { - "taskId": {"type": "string"}, - "outputName": {"type": "string"}, - "type": {"$ref": "#/definitions/TypeSpecType"} - }, - "additionalProperties": false - } - }, - "additionalProperties": false - }, - - "ArgumentType": { - "oneOf": [ - {"type": "string"}, - {"$ref": "#/definitions/GraphInputArgument"}, - {"$ref": "#/definitions/TaskOutputArgument"} - ] - }, - - "TwoArgumentOperands": { - "description": "Pair of operands for a binary operation.", - "type": "object", - "required": ["op1", "op2"], - "properties": { - "op1": {"$ref": "#/definitions/ArgumentType"}, - "op2": {"$ref": "#/definitions/ArgumentType"} - }, - "additionalProperties": false - }, - - "TwoLogicalOperands": { - "description": "Pair of operands for a binary logical operation.", - "type": "object", - "required": ["op1", "op2"], - "properties": { - "op1": {"$ref": "#/definitions/PredicateType"}, - "op2": {"$ref": "#/definitions/PredicateType"} - }, - "additionalProperties": false - }, - - "PredicateType": { - "oneOf": [ - {"type": "object", "required": ["=="], "properties": {"==": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - {"type": "object", "required": ["!="], "properties": {"!=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - {"type": "object", "required": [">"], "properties": {">": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - {"type": "object", "required": [">="], "properties": {">=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - {"type": "object", "required": ["<"], "properties": {"<": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - {"type": "object", "required": ["<="], "properties": {"<=": {"$ref": "#/definitions/TwoArgumentOperands"}}}, - - {"type": "object", "required": ["and"], "properties": {"and": {"$ref": "#/definitions/TwoLogicalOperands"}}}, - {"type": "object", "required": ["or"], "properties": {"or": {"$ref": "#/definitions/TwoLogicalOperands"}}}, - - {"type": "object", "required": ["not"], "properties": {"not": {"$ref": "#/definitions/PredicateType"}}} - ] - }, - - "RetryStrategySpec": { - "description": "Optional configuration that specifies how the task should be retried if it fails.", - "type": "object", - "properties": { - "maxRetries": {"type": "integer"} - }, - "additionalProperties": false - }, - - "CachingStrategySpec": { - "description": "Optional configuration that specifies how the task execution may be skipped if the output data exist in cache.", - "type": "object", - "properties": { - "maxCacheStaleness": {"type": "string", "format": "duration"} - }, - "additionalProperties": false - }, - - "ExecutionOptionsSpec": { - "description": "Optional configuration that specifies how the task should be executed. Can be used to set some platform-specific options.", - "type": "object", - "properties": { - "retryStrategy": {"$ref": "#/definitions/RetryStrategySpec"}, - "cachingStrategy": {"$ref": "#/definitions/CachingStrategySpec"} - }, - "additionalProperties": false - }, - - "TaskSpec": { - "description": "'Task specification. Task is a configured component - a component supplied with arguments and other applied configuration changes.", - "type": "object", - "required": ["componentRef"], - "properties": { - "componentRef": {"$ref": "#/definitions/ComponentReference"}, - "arguments": {"type": "object", "additionalProperties": {"$ref": "#/definitions/ArgumentType"}}, - "isEnabled": {"$ref": "#/definitions/PredicateType"}, - "executionOptions": {"$ref": "#/definitions/ExecutionOptionsSpec"} - }, - "additionalProperties": false - }, - - "GraphSpec": { - "description": "Describes the graph component implementation. It represents a graph of component tasks connected to the upstream sources of data using the argument specifications. It also describes the sources of graph output values.", - "type": "object", - "required": ["tasks"], - "properties": { - "tasks": {"type": "object", "additionalProperties": {"$ref": "#/definitions/TaskSpec"}}, - "outputValues": {"type": "object", "additionalProperties": {"$ref": "#/definitions/TaskOutputArgument"}} - }, - "additionalProperties": false - }, - - "GraphImplementation": { - "description": "Represents the graph component implementation.", - "type": "object", - "required": ["graph"], - "properties": { - "graph": {"$ref": "#/definitions/GraphSpec"} - }, - "additionalProperties": false - }, - - "PipelineRunSpec": { - "description": "The object that can be sent to the backend to start a new Run.", - "type": "object", - "required": ["rootTask"], - "properties": { - "rootTask": {"$ref": "#/definitions/TaskSpec"}, - "onExitTask": {"$ref": "#/definitions/TaskSpec"} - }, - "additionalProperties": false - } - } -} diff --git a/sdk/python/kfp/deprecated/components/structures/components.json_schema.outline.yaml b/sdk/python/kfp/deprecated/components/structures/components.json_schema.outline.yaml deleted file mode 100644 index 2b4087df709..00000000000 --- a/sdk/python/kfp/deprecated/components/structures/components.json_schema.outline.yaml +++ /dev/null @@ -1,69 +0,0 @@ -: - name: string - description: string - inputs: #InputSpec[] - : - name: string - type: TypeSpecType - description: string - default: string - optional: boolean - outputs: #OutputSpec[] - : - name: string - type: TypeSpecType - description: string - implementation: #ImplementationType - : - : - container: #ContainerSpec - image: StringOrPlaceholder - command: #StringOrPlaceholder[] - : - string: - : - inputValue: string - : - inputPath: string - : - outputPath: string - : - concat: StringOrPlaceholder[] - : - if: - cond: IfConditionArgumentType - then: StringOrPlaceholder[] - else: StringOrPlaceholder[] - args: StringOrPlaceholder[] - env: Map[str, StringOrPlaceholder] - unconfigurableOutputPaths: Map[str, str] - : - graph: #GraphSpec - outputValues: Map str -> TaskOutputArgument - tasks: #Map str -> TaskSpec - TaskSpec>: - componentRef: #ComponentReference - name: string - digest: string - tag: string - url: string - spec: ComponentSpec - arguments: #Map str -> ArgumentType - ArgumentType>: - : - string: - : - graphInput: - inputName: string - type: TypeSpecType - : - taskOutput: - taskId: string - outputName: string - type: TypeSpecType - isEnabled: PredicateType - executionOptions: #ExecutionOptionsSpec - retryStrategy: #RetryStrategySpec - maxRetries: integer - cachingStrategy: #CachingStrategySpec - maxCacheStaleness: string diff --git a/sdk/python/kfp/deprecated/components/structures/components.proto b/sdk/python/kfp/deprecated/components/structures/components.proto deleted file mode 100644 index 096f6bb0095..00000000000 --- a/sdk/python/kfp/deprecated/components/structures/components.proto +++ /dev/null @@ -1,216 +0,0 @@ -syntax = "proto3"; - -package org.kubeflow.pipelines.components.v1alpha1; -import "google/protobuf/struct.proto"; - -// Describes the component input specification -message InputSpec { - string name = 1; - TypeSpecType type = 2; - string description = 3; - string default = 4; - bool optional = 5; -} - -// Describes the component output specification -message OutputSpec { - string name = 1; - TypeSpecType type = 2; - string description = 3; -} - -message StringOrPlaceholder { - oneof oneof { - // Constant string - string constantValue = 1; - - // Represents the command-line argument placeholder that will be replaced at run-time by the input argument value. - string inputValue = 2; // == input value for - - // Represents the command-line argument placeholder that will be replaced at run-time by a local file path pointing to a file containing the input argument value. - string inputPath = 3; // == input path for - - // Represents the command-line argument placeholder that will be replaced at run-time by a local file path pointing to a file where the program should write its output data. - string outputPath = 4; // == output path for - - // Represents the command-line argument placeholder that will be replaced at run-time by the concatenated values of its items. - ConcatPlaceholder concat = 5; - - // Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input. - IfPlaceholder if = 6; - } -} - -message ContainerSpec { - // Docker image name. - string image = 1; - - // Entrypoint array. Not executed within a shell. The docker image's ENTRYPOINT is used if this is not provided. - repeated StringOrPlaceholder command = 2; - - // Arguments to the entrypoint. The docker image's CMD is used if this is not provided. - repeated StringOrPlaceholder args = 3; - - // List of environment variables to set in the container. - map env = 4; - - // Legacy. Deprecated. Can be used to specify output paths for programs that do not allow setting the path of some output using command-line. - map fileOutputs = 5; -} - -// Represents the container component implementation. -message ContainerImplementation { - ContainerSpec container = 1; -} - -// Component specification. Describes the metadata (name, description, source), the interface (inputs and outputs) and the implementation of the component. -message ComponentSpec { - string name = 1; - string description = 2; - repeated InputSpec inputs = 3; - repeated OutputSpec outputs = 4; - ImplementationType implementation = 5; - string version = 6; - MetadataSpec metadata = 7; -} - -// Component reference. Contains information that can be used to locate and load a component by name, digest or URL -message ComponentReference { - string name = 1; - string digest = 2; - string tag = 3; - string url = 4; - ComponentSpec spec = 5; -} - -// References the input of the graph. -message GraphInputReference { - string inputName = 1; - TypeSpecType type = 2; -} - -// References the output of a sibling task. -message TaskOutputReference { - string outputName = 1; - string taskId = 2; - TypeSpecType type = 3; -} - - -message TaskArgumentType { - oneof oneof { - // Constant task argument. - string string_value = 1; - - // Represents the task argument value that comes from the graph component input. - GraphInputReference graphInput = 2; - - // Represents the task argument value that comes from the output of a sibling task. - TaskOutputReference taskOutput = 3; - } -} - -// Task specification. Task is a configured component - a component supplied with arguments and other applied configuration changes. -message TaskSpec { - ComponentReference componentRef = 1; // Reference to the component from which the task is instantiated - - map arguments = 2; // Arguments to pass to the component - - PredicateType isEnabled = 3; // Specifies predicate to execute the task conditionally - - ExecutionOptionsSpec executionOptions = 4; -} - -// Describes the graph component implementation. It represents a graph of component tasks connected to the upstream sources of data using the argument specifications. It also describes the sources of graph output values. -message GraphSpec { - map tasks = 1; // List of tasks (components and their arguments) belonging to the graph. - map outputValues = 2; // Sources of the graph component output values. -} - -// The implementation of the component -message ImplementationType { - oneof oneof { - ContainerSpec container = 1; // Represents the component implementated as a containerized program - GraphSpec graph = 2; // Represents the component implementated as graph of connected tasks - } -} - -// The object that can be sent to the backend to start a new Run. -message PipelineRunSpec { - TaskSpec rootTask = 1; - TaskSpec onExitTask = 2; -} - -message TypeSpecType { - message Properties { - map type = 1; - } - - oneof oneof { - string type_name = 1; - Properties properties = 2; - } -} - -// Represents the command-line argument placeholder that will be replaced at run-time by the concatenated values of its items. -message ConcatPlaceholder { - // Items to concatenate - repeated StringOrPlaceholder items = 1; -} - -message IfConditionArgumentType { - oneof oneof { - bool boolean_value = 1; // e.g. True - string string_value = 2; // e.g. "true" - string isPresent = 3; // Checks whether an argument for some input was passed to the component // Represents the command-line argument placeholder that will be replaced at run-time by a boolean value specifying whether the caller has passed an argument for the specified optional input. - } -} - -message IfPlaceholder { - IfConditionArgumentType cond = 1; - repeated StringOrPlaceholder then = 2; - repeated StringOrPlaceholder else = 3; -} - -message MetadataSpec { - map annotations = 1; -} - -// Represents a logical expression. Used to specify condition for conditional task executed. -message PredicateType { - oneof oneof { - PredicateType not = 1; - - TwoLogicalOperands and = 2; - TwoLogicalOperands or = 3; - - TwoArgumentOperands eq = 4; - TwoArgumentOperands ne = 5; - TwoArgumentOperands gt = 6; - TwoArgumentOperands ge = 7; - TwoArgumentOperands lt = 8; - TwoArgumentOperands le = 9; - } -} - -// Pair of operands for a binary operation. -message TwoArgumentOperands { - TaskArgumentType op1 = 1; - TaskArgumentType op2 = 2; -} - -// Pair of operands for a binary logical operation. -message TwoLogicalOperands { - PredicateType op1 = 1; - PredicateType op2 = 2; -} - -// Optional configuration that specifies how the task should be retried if it fails. -message RetryStrategySpec { - int32 maxRetries = 1; -} - -// Optional configuration that specifies how the task should be executed. Can be used to set some platform-specific options. -message ExecutionOptionsSpec { - RetryStrategySpec retryStrategy = 1; -} diff --git a/sdk/python/kfp/deprecated/components/structures/generate_proto_code.sh b/sdk/python/kfp/deprecated/components/structures/generate_proto_code.sh deleted file mode 100755 index cceee8f586d..00000000000 --- a/sdk/python/kfp/deprecated/components/structures/generate_proto_code.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -e - -protos=( -k8s.io/api/core/v1/generated.proto -k8s.io/apimachinery/pkg/api/resource/generated.proto -k8s.io/apimachinery/pkg/apis/meta/v1/generated.proto -k8s.io/apimachinery/pkg/runtime/generated.proto -k8s.io/apimachinery/pkg/runtime/schema/generated.proto -k8s.io/apimachinery/pkg/util/intstr/generated.proto -) - -for proto in "${protos[@]}"; do - url=$(echo "$proto" | sed -E -e 's|k8s.io/([^/]+)/(.*)|https://raw.githubusercontent.com/kubernetes/\1/master/\2|') - mkdir -p "$(dirname "$proto")" - wget --quiet -O "$proto" "$url" -done - - - -protoc components.proto --python_out . -I . diff --git a/sdk/python/kfp/deprecated/components/type_annotation_utils.py b/sdk/python/kfp/deprecated/components/type_annotation_utils.py deleted file mode 100644 index 0370937b0df..00000000000 --- a/sdk/python/kfp/deprecated/components/type_annotation_utils.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for handling Python type annotation.""" - -import re -from typing import TypeVar, Union - -T = TypeVar('T') - - -def maybe_strip_optional_from_annotation(annotation: T) -> T: - """Strips 'Optional' from 'Optional[]' if applicable. - - For example:: - Optional[str] -> str - str -> str - List[int] -> List[int] - - Args: - annotation: The original type annotation which may or may not has - `Optional`. - - Returns: - The type inside Optional[] if Optional exists, otherwise the original type. - """ - if getattr(annotation, '__origin__', - None) is Union and annotation.__args__[1] is type(None): - return annotation.__args__[0] - return annotation - - -def get_short_type_name(type_name: str) -> str: - """Extracts the short form type name. - - This method is used for looking up serializer for a given type. - - For example:: - typing.List -> List - typing.List[int] -> List - typing.Dict[str, str] -> Dict - List -> List - str -> str - - Args: - type_name: The original type name. - - Returns: - The short form type name or the original name if pattern doesn't match. - """ - match = re.match(r'(typing\.)?(?P\w+)(?:\[.+\])?', type_name) - if match: - return match.group('type') - else: - return type_name diff --git a/sdk/python/kfp/deprecated/components/type_annotation_utils_test.py b/sdk/python/kfp/deprecated/components/type_annotation_utils_test.py deleted file mode 100644 index 01fa6290404..00000000000 --- a/sdk/python/kfp/deprecated/components/type_annotation_utils_test.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for kfp.components.type_annoation_utils.""" - -import unittest -from absl.testing import parameterized -from typing import Any, Dict, List, Optional - -from kfp.deprecated.components import type_annotation_utils - - -class TypeAnnotationUtilsTest(parameterized.TestCase): - - @parameterized.parameters( - { - 'original_annotation': str, - 'expected_annotation': str, - }, - { - 'original_annotation': 'MyCustomType', - 'expected_annotation': 'MyCustomType', - }, - { - 'original_annotation': List[int], - 'expected_annotation': List[int], - }, - { - 'original_annotation': Optional[str], - 'expected_annotation': str, - }, - { - 'original_annotation': Optional[Dict[str, float]], - 'expected_annotation': Dict[str, float], - }, - { - 'original_annotation': Optional[List[Dict[str, Any]]], - 'expected_annotation': List[Dict[str, Any]], - }, - ) - def test_maybe_strip_optional_from_annotation(self, original_annotation, - expected_annotation): - self.assertEqual( - expected_annotation, - type_annotation_utils.maybe_strip_optional_from_annotation( - original_annotation)) - - @parameterized.parameters( - { - 'original_type_name': 'str', - 'expected_type_name': 'str', - }, - { - 'original_type_name': 'typing.List[int]', - 'expected_type_name': 'List', - }, - { - 'original_type_name': 'List[int]', - 'expected_type_name': 'List', - }, - { - 'original_type_name': 'Dict[str, str]', - 'expected_type_name': 'Dict', - }, - { - 'original_type_name': 'List[Dict[str, str]]', - 'expected_type_name': 'List', - }, - ) - def test_get_short_type_name(self, original_type_name, expected_type_name): - self.assertEqual( - expected_type_name, - type_annotation_utils.get_short_type_name(original_type_name)) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/components_tests/__init__.py b/sdk/python/kfp/deprecated/components_tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/kfp/deprecated/components_tests/test_components.py b/sdk/python/kfp/deprecated/components_tests/test_components.py deleted file mode 100644 index 938ba2cec19..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_components.py +++ /dev/null @@ -1,1114 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import requests -import textwrap -import unittest -from pathlib import Path -from unittest import mock - -from kfp.deprecated import components as comp -from kfp.deprecated.components._components import _resolve_command_line_and_paths -from kfp.deprecated.components._yaml_utils import load_yaml -from kfp.deprecated.components.structures import ComponentSpec - - -class LoadComponentTestCase(unittest.TestCase): - - def _test_load_component_from_file(self, component_path: str): - task_factory1 = comp.load_component_from_file(component_path) - - arg1 = 3 - arg2 = 5 - task1 = task_factory1(arg1, arg2) - - self.assertEqual(task_factory1.__name__, 'Add') - self.assertEqual(task_factory1.__doc__.strip(), - 'Add\nReturns sum of two arguments') - - self.assertEqual( - task1.component_ref.spec.implementation.container.image, - 'python:3.5') - - resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, - task1.arguments) - self.assertEqual(resolved_cmd.args[0], str(arg1)) - self.assertEqual(resolved_cmd.args[1], str(arg2)) - - def test_load_component_from_yaml_file(self): - component_path = Path( - __file__).parent / 'test_data' / 'python_add.component.yaml' - self._test_load_component_from_file(str(component_path)) - - def test_load_component_from_zipped_yaml_file(self): - component_path = Path( - __file__).parent / 'test_data' / 'python_add.component.zip' - self._test_load_component_from_file(str(component_path)) - - def test_load_component_from_url(self): - component_path = Path( - __file__).parent / 'test_data' / 'python_add.component.yaml' - component_url = 'https://raw.githubusercontent.com/some/repo/components/component_group/python_add/component.yaml' - component_bytes = component_path.read_bytes() - component_dict = load_yaml(component_bytes) - - def mock_response_factory(url, params=None, **kwargs): - if url == component_url: - response = requests.Response() - response.url = component_url - response.status_code = 200 - response._content = component_bytes - return response - raise RuntimeError('Unexpected URL "{}"'.format(url)) - - with mock.patch('requests.get', mock_response_factory): - task_factory1 = comp.load_component_from_url(component_url) - - self.assertEqual( - task_factory1.__doc__, - component_dict['name'] + '\n' + component_dict['description']) - - arg1 = 3 - arg2 = 5 - task1 = task_factory1(arg1, arg2) - self.assertEqual( - task1.component_ref.spec.implementation.container.image, - component_dict['implementation']['container']['image']) - - resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, - task1.arguments) - self.assertEqual(resolved_cmd.args[0], str(arg1)) - self.assertEqual(resolved_cmd.args[1], str(arg2)) - - def test_loading_minimal_component(self): - component_text = '''\ -implementation: - container: - image: busybox -''' - component_dict = load_yaml(component_text) - task_factory1 = comp.load_component(text=component_text) - - self.assertEqual( - task_factory1.component_spec.implementation.container.image, - component_dict['implementation']['container']['image']) - - def test_digest_of_loaded_component(self): - component_text = textwrap.dedent('''\ - implementation: - container: - image: busybox - ''') - task_factory1 = comp.load_component_from_text(component_text) - task1 = task_factory1() - - self.assertEqual( - task1.component_ref.digest, - '1ede211233e869581d098673962c2c1e8c1e4cebb7cf5d7332c2f73cb4900823') - - def test_accessing_component_spec_from_task_factory(self): - component_text = '''\ -implementation: - container: - image: busybox -''' - task_factory1 = comp.load_component_from_text(component_text) - - actual_component_spec = task_factory1.component_spec - actual_component_spec_dict = actual_component_spec.to_dict() - expected_component_spec_dict = load_yaml(component_text) - expected_component_spec = ComponentSpec.from_dict( - expected_component_spec_dict) - self.assertEqual(expected_component_spec_dict, - actual_component_spec_dict) - self.assertEqual(expected_component_spec, task_factory1.component_spec) - - def test_fail_on_duplicate_input_names(self): - component_text = '''\ -inputs: -- {name: Data1} -- {name: Data1} -implementation: - container: - image: busybox -''' - with self.assertRaises(ValueError): - task_factory1 = comp.load_component_from_text(component_text) - - def test_fail_on_duplicate_output_names(self): - component_text = '''\ -outputs: -- {name: Data1} -- {name: Data1} -implementation: - container: - image: busybox -''' - with self.assertRaises(ValueError): - task_factory1 = comp.load_component_from_text(component_text) - - def test_handle_underscored_input_names(self): - component_text = '''\ -inputs: -- {name: Data} -- {name: _Data} -implementation: - container: - image: busybox -''' - task_factory1 = comp.load_component_from_text(component_text) - - def test_handle_underscored_output_names(self): - component_text = '''\ -outputs: -- {name: Data} -- {name: _Data} -implementation: - container: - image: busybox -''' - task_factory1 = comp.load_component_from_text(component_text) - - def test_handle_input_names_with_spaces(self): - component_text = '''\ -inputs: -- {name: Training data} -implementation: - container: - image: busybox -''' - task_factory1 = comp.load_component_from_text(component_text) - - def test_handle_output_names_with_spaces(self): - component_text = '''\ -outputs: -- {name: Training data} -implementation: - container: - image: busybox -''' - task_factory1 = comp.load_component_from_text(component_text) - - def test_handle_file_outputs_with_spaces(self): - component_text = '''\ -outputs: -- {name: Output data} -implementation: - container: - image: busybox - fileOutputs: - Output data: /outputs/output-data -''' - task_factory1 = comp.load_component_from_text(component_text) - - def test_handle_similar_input_names(self): - component_text = '''\ -inputs: -- {name: Input 1} -- {name: Input_1} -- {name: Input-1} -implementation: - container: - image: busybox -''' - task_factory1 = comp.load_component_from_text(component_text) - - def test_conflicting_name_renaming_stability(self): - # Checking that already pythonic input names are not renamed - # Checking that renaming is deterministic - component_text = textwrap.dedent('''\ - inputs: - - {name: Input 1} - - {name: Input_1} - - {name: Input-1} - - {name: input_1} # Last in the list, but is pythonic, so it should not be renamed - implementation: - container: - image: busybox - command: - - inputValue: Input 1 - - inputValue: Input_1 - - inputValue: Input-1 - - inputValue: input_1 - ''') - task_factory1 = comp.load_component(text=component_text) - task1 = task_factory1( - input_1_2='value_1_2', - input_1_3='value_1_3', - input_1_4='value_1_4', - input_1='value_1', # Expecting this input not to be renamed - ) - resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, - task1.arguments) - - self.assertEqual(resolved_cmd.command, - ['value_1_2', 'value_1_3', 'value_1_4', 'value_1']) - - def test_handle_duplicate_input_output_names(self): - component_text = '''\ -inputs: -- {name: Data} -outputs: -- {name: Data} -implementation: - container: - image: busybox -''' - task_factory1 = comp.load_component_from_text(component_text) - - def test_fail_on_unknown_value_argument(self): - component_text = '''\ -inputs: -- {name: Data} -implementation: - container: - image: busybox - args: - - {inputValue: Wrong} -''' - with self.assertRaises(TypeError): - task_factory1 = comp.load_component_from_text(component_text) - - def test_fail_on_unknown_file_output(self): - component_text = '''\ -outputs: -- {name: Data} -implementation: - container: - image: busybox - fileOutputs: - Wrong: '/outputs/output.txt' -''' - with self.assertRaises(TypeError): - task_factory1 = comp.load_component_from_text(component_text) - - def test_load_component_fail_on_no_sources(self): - with self.assertRaises(ValueError): - comp.load_component() - - def test_load_component_fail_on_multiple_sources(self): - with self.assertRaises(ValueError): - comp.load_component(filename='', text='') - - def test_load_component_fail_on_none_arguments(self): - with self.assertRaises(ValueError): - comp.load_component(filename=None, url=None, text=None) - - def test_load_component_from_file_fail_on_none_arg(self): - with self.assertRaises(TypeError): - comp.load_component_from_file(None) - - def test_load_component_from_url_fail_on_none_arg(self): - with self.assertRaises(TypeError): - comp.load_component_from_url(None) - - def test_load_component_from_text_fail_on_none_arg(self): - with self.assertRaises(TypeError): - comp.load_component_from_text(None) - - def test_input_value_resolving(self): - component_text = '''\ -inputs: -- {name: Data} -implementation: - container: - image: busybox - args: - - --data - - inputValue: Data -''' - task_factory1 = comp.load_component(text=component_text) - task1 = task_factory1('some-data') - resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, - task1.arguments) - - self.assertEqual(resolved_cmd.args, ['--data', 'some-data']) - - def test_automatic_output_resolving(self): - component_text = '''\ -outputs: -- {name: Data} -implementation: - container: - image: busybox - args: - - --output-data - - {outputPath: Data} -''' - task_factory1 = comp.load_component(text=component_text) - task1 = task_factory1() - resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, - task1.arguments) - - self.assertEqual(len(resolved_cmd.args), 2) - self.assertEqual(resolved_cmd.args[0], '--output-data') - self.assertTrue(resolved_cmd.args[1].startswith('/')) - - def test_input_path_placeholder_with_constant_argument(self): - component_text = '''\ -inputs: -- {name: input 1} -implementation: - container: - image: busybox - command: - - --input-data - - {inputPath: input 1} -''' - task_factory1 = comp.load_component_from_text(component_text) - task1 = task_factory1('Text') - resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, - task1.arguments) - - self.assertEqual(resolved_cmd.command, - ['--input-data', resolved_cmd.input_paths['input 1']]) - self.assertEqual(task1.arguments, {'input 1': 'Text'}) - - def test_optional_inputs_reordering(self): - """Tests optional input reordering. - - In python signature, optional arguments must come after the - required arguments. - """ - component_text = '''\ -inputs: -- {name: in1} -- {name: in2, optional: true} -- {name: in3} -implementation: - container: - image: busybox -''' - task_factory1 = comp.load_component_from_text(component_text) - import inspect - signature = inspect.signature(task_factory1) - actual_signature = list(signature.parameters.keys()) - self.assertSequenceEqual(actual_signature, ['in1', 'in3', 'in2'], str) - - def test_inputs_reordering_when_inputs_have_defaults(self): - """Tests reordering of inputs with default values. - - In python signature, optional arguments must come after the - required arguments. - """ - component_text = '''\ -inputs: -- {name: in1} -- {name: in2, default: val} -- {name: in3} -implementation: - container: - image: busybox -''' - task_factory1 = comp.load_component_from_text(component_text) - import inspect - signature = inspect.signature(task_factory1) - actual_signature = list(signature.parameters.keys()) - self.assertSequenceEqual(actual_signature, ['in1', 'in3', 'in2'], str) - - def test_inputs_reordering_stability(self): - """Tests input reordering stability. - - Required inputs and optional/default inputs should keep the - ordering. In python signature, optional arguments must come - after the required arguments. - """ - component_text = '''\ -inputs: -- {name: a1} -- {name: b1, default: val} -- {name: a2} -- {name: b2, optional: True} -- {name: a3} -- {name: b3, default: val} -- {name: a4} -- {name: b4, optional: True} -implementation: - container: - image: busybox -''' - task_factory1 = comp.load_component_from_text(component_text) - import inspect - signature = inspect.signature(task_factory1) - actual_signature = list(signature.parameters.keys()) - self.assertSequenceEqual( - actual_signature, ['a1', 'a2', 'a3', 'a4', 'b1', 'b2', 'b3', 'b4'], - str) - - def test_missing_optional_input_value_argument(self): - """Missing optional inputs should resolve to nothing.""" - component_text = '''\ -inputs: -- {name: input 1, optional: true} -implementation: - container: - image: busybox - command: - - a - - {inputValue: input 1} - - z -''' - task_factory1 = comp.load_component_from_text(component_text) - task1 = task_factory1() - resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, - task1.arguments) - - self.assertEqual(resolved_cmd.command, ['a', 'z']) - - def test_missing_optional_input_file_argument(self): - """Missing optional inputs should resolve to nothing.""" - component_text = '''\ -inputs: -- {name: input 1, optional: true} -implementation: - container: - image: busybox - command: - - a - - {inputPath: input 1} - - z -''' - task_factory1 = comp.load_component_from_text(component_text) - task1 = task_factory1() - resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, - task1.arguments) - - self.assertEqual(resolved_cmd.command, ['a', 'z']) - - def test_command_concat(self): - component_text = '''\ -inputs: -- {name: In1} -- {name: In2} -implementation: - container: - image: busybox - args: - - concat: [{inputValue: In1}, {inputValue: In2}] -''' - task_factory1 = comp.load_component(text=component_text) - task1 = task_factory1('some', 'data') - resolved_cmd = _resolve_command_line_and_paths(task1.component_ref.spec, - task1.arguments) - - self.assertEqual(resolved_cmd.args, ['somedata']) - - def test_command_if_boolean_true_then_else(self): - component_text = '''\ -implementation: - container: - image: busybox - args: - - if: - cond: true - then: --true-arg - else: --false-arg -''' - task_factory1 = comp.load_component(text=component_text) - task = task_factory1() - resolved_cmd = _resolve_command_line_and_paths(task.component_ref.spec, - task.arguments) - self.assertEqual(resolved_cmd.args, ['--true-arg']) - - def test_command_if_boolean_false_then_else(self): - component_text = '''\ -implementation: - container: - image: busybox - args: - - if: - cond: false - then: --true-arg - else: --false-arg -''' - task_factory1 = comp.load_component(text=component_text) - task = task_factory1() - resolved_cmd = _resolve_command_line_and_paths(task.component_ref.spec, - task.arguments) - self.assertEqual(resolved_cmd.args, ['--false-arg']) - - def test_command_if_true_string_then_else(self): - component_text = '''\ -implementation: - container: - image: busybox - args: - - if: - cond: 'true' - then: --true-arg - else: --false-arg -''' - task_factory1 = comp.load_component(text=component_text) - task = task_factory1() - resolved_cmd = _resolve_command_line_and_paths(task.component_ref.spec, - task.arguments) - self.assertEqual(resolved_cmd.args, ['--true-arg']) - - def test_command_if_false_string_then_else(self): - component_text = '''\ -implementation: - container: - image: busybox - args: - - if: - cond: 'false' - then: --true-arg - else: --false-arg -''' - task_factory1 = comp.load_component(text=component_text) - - task = task_factory1() - resolved_cmd = _resolve_command_line_and_paths(task.component_ref.spec, - task.arguments) - self.assertEqual(resolved_cmd.args, ['--false-arg']) - - def test_command_if_is_present_then(self): - component_text = '''\ -inputs: -- {name: In, optional: true} -implementation: - container: - image: busybox - args: - - if: - cond: {isPresent: In} - then: [--in, {inputValue: In}] - #else: --no-in -''' - task_factory1 = comp.load_component(text=component_text) - - task_then = task_factory1('data') - resolved_cmd_then = _resolve_command_line_and_paths( - task_then.component_ref.spec, task_then.arguments) - self.assertEqual(resolved_cmd_then.args, ['--in', 'data']) - - task_else = task_factory1() - resolved_cmd_else = _resolve_command_line_and_paths( - task_else.component_ref.spec, task_else.arguments) - self.assertEqual(resolved_cmd_else.args, []) - - def test_command_if_is_present_then_else(self): - component_text = '''\ -inputs: -- {name: In, optional: true} -implementation: - container: - image: busybox - args: - - if: - cond: {isPresent: In} - then: [--in, {inputValue: In}] - else: --no-in -''' - task_factory1 = comp.load_component(text=component_text) - - task_then = task_factory1('data') - resolved_cmd_then = _resolve_command_line_and_paths( - task_then.component_ref.spec, task_then.arguments) - self.assertEqual(resolved_cmd_then.args, ['--in', 'data']) - - task_else = task_factory1() - resolved_cmd_else = _resolve_command_line_and_paths( - task_else.component_ref.spec, task_else.arguments) - self.assertEqual(resolved_cmd_else.args, ['--no-in']) - - def test_command_if_input_value_then(self): - component_text = '''\ -inputs: -- {name: Do test, type: Boolean, optional: true} -- {name: Test data, type: Integer, optional: true} -- {name: Test parameter 1, optional: true} -implementation: - container: - image: busybox - args: - - if: - cond: {inputValue: Do test} - then: [--test-data, {inputValue: Test data}, --test-param1, {inputValue: Test parameter 1}] -''' - task_factory1 = comp.load_component(text=component_text) - - task_then = task_factory1(True, 'test_data.txt', '42') - resolved_cmd_then = _resolve_command_line_and_paths( - task_then.component_ref.spec, task_then.arguments) - self.assertEqual( - resolved_cmd_then.args, - ['--test-data', 'test_data.txt', '--test-param1', '42']) - - task_else = task_factory1() - resolved_cmd_else = _resolve_command_line_and_paths( - task_else.component_ref.spec, task_else.arguments) - self.assertEqual(resolved_cmd_else.args, []) - - def test_handle_default_values_in_task_factory(self): - component_text = '''\ -inputs: -- {name: Data, default: '123'} -implementation: - container: - image: busybox - args: - - {inputValue: Data} -''' - task_factory1 = comp.load_component_from_text(text=component_text) - - task1 = task_factory1() - resolved_cmd1 = _resolve_command_line_and_paths( - task1.component_ref.spec, task1.arguments) - self.assertEqual(resolved_cmd1.args, ['123']) - - task2 = task_factory1('456') - resolved_cmd2 = _resolve_command_line_and_paths( - task2.component_ref.spec, task2.arguments) - self.assertEqual(resolved_cmd2.args, ['456']) - - def test_check_task_spec_outputs_dictionary(self): - component_text = '''\ -outputs: -- {name: out 1} -- {name: out 2} -implementation: - container: - image: busybox - command: [touch, {outputPath: out 1}, {outputPath: out 2}] -''' - op = comp.load_component_from_text(component_text) - - task = op() - - self.assertEqual(list(task.outputs.keys()), ['out 1', 'out 2']) - - def test_check_task_object_no_output_attribute_when_0_outputs(self): - component_text = textwrap.dedent( - '''\ - implementation: - container: - image: busybox - command: [] - ''',) - - op = comp.load_component_from_text(component_text) - task = op() - - self.assertFalse(hasattr(task, 'output')) - - def test_check_task_object_has_output_attribute_when_1_output(self): - component_text = textwrap.dedent( - '''\ - outputs: - - {name: out 1} - implementation: - container: - image: busybox - command: [touch, {outputPath: out 1}] - ''',) - - op = comp.load_component_from_text(component_text) - task = op() - - self.assertEqual(task.output.task_output.output_name, 'out 1') - - def test_check_task_object_no_output_attribute_when_multiple_outputs(self): - component_text = textwrap.dedent( - '''\ - outputs: - - {name: out 1} - - {name: out 2} - implementation: - container: - image: busybox - command: [touch, {outputPath: out 1}, {outputPath: out 2}] - ''',) - - op = comp.load_component_from_text(component_text) - task = op() - - self.assertFalse(hasattr(task, 'output')) - - def test_prevent_passing_unserializable_objects_as_argument(self): - component_text = textwrap.dedent('''\ - inputs: - - {name: input 1} - - {name: input 2} - implementation: - container: - image: busybox - command: - - prog - - {inputValue: input 1} - - {inputPath: input 2} - ''') - component = comp.load_component_from_text(component_text) - # Passing normal values to component - task1 = component(input_1="value 1", input_2="value 2") - # Passing unserializable values to component - with self.assertRaises(TypeError): - component(input_1=task1, input_2="value 2") - with self.assertRaises(TypeError): - component(input_1=open, input_2="value 2") - with self.assertRaises(TypeError): - component(input_1="value 1", input_2=task1) - with self.assertRaises(TypeError): - component(input_1="value 1", input_2=open) - - def test_check_type_validation_of_task_spec_outputs(self): - producer_component_text = '''\ -outputs: -- {name: out1, type: Integer} -- {name: out2, type: String} -implementation: - container: - image: busybox - command: [touch, {outputPath: out1}, {outputPath: out2}] -''' - consumer_component_text = '''\ -inputs: -- {name: data, type: Integer} -implementation: - container: - image: busybox - command: [echo, {inputValue: data}] -''' - producer_op = comp.load_component_from_text(producer_component_text) - consumer_op = comp.load_component_from_text(consumer_component_text) - - producer_task = producer_op() - - consumer_op(producer_task.outputs['out1']) - consumer_op(producer_task.outputs['out2'].without_type()) - consumer_op(producer_task.outputs['out2'].with_type('Integer')) - with self.assertRaises(TypeError): - consumer_op(producer_task.outputs['out2']) - - def test_type_compatibility_check_for_simple_types(self): - component_a = '''\ -outputs: - - {name: out1, type: custom_type} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: custom_type} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1']) - - def test_type_compatibility_check_for_types_with_parameters(self): - component_a = '''\ -outputs: - - {name: out1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1']) - - def test_type_compatibility_check_when_using_positional_arguments(self): - """Tests that `op2(task1.output)` works as good as - `op2(in1=task1.output)`""" - component_a = '''\ -outputs: - - {name: out1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(a_task.outputs['out1']) - - def test_type_compatibility_check_when_input_type_is_missing(self): - component_a = '''\ -outputs: - - {name: out1, type: custom_type} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1']) - - def test_type_compatibility_check_when_argument_type_is_missing(self): - component_a = '''\ -outputs: - - {name: out1} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: custom_type} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1']) - - def test_fail_type_compatibility_check_when_simple_type_name_is_different( - self): - component_a = '''\ -outputs: - - {name: out1, type: type_A} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: type_Z} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - with self.assertRaises(TypeError): - b_task = task_factory_b(in1=a_task.outputs['out1']) - - def test_fail_type_compatibility_check_when_parametrized_type_name_is_different( - self): - component_a = '''\ -outputs: - - {name: out1, type: {parametrized_type_A: {property_a: value_a}}} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: {parametrized_type_Z: {property_a: value_a}}} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - with self.assertRaises(TypeError): - b_task = task_factory_b(in1=a_task.outputs['out1']) - - def test_fail_type_compatibility_check_when_type_property_value_is_different( - self): - component_a = '''\ -outputs: - - {name: out1, type: {parametrized_type: {property_a: value_a}}} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: {parametrized_type: {property_a: DIFFERENT VALUE}}} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - with self.assertRaises(TypeError): - b_task = task_factory_b(in1=a_task.outputs['out1']) - - @unittest.skip('Type compatibility check currently works the opposite way') - def test_type_compatibility_check_when_argument_type_has_extra_type_parameters( - self): - component_a = '''\ -outputs: - - {name: out1, type: {parametrized_type: {property_a: value_a, extra_property: extra_value}}} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: {parametrized_type: {property_a: value_a}}} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1']) - - @unittest.skip('Type compatibility check currently works the opposite way') - def test_fail_type_compatibility_check_when_argument_type_has_missing_type_parameters( - self): - component_a = '''\ -outputs: - - {name: out1, type: {parametrized_type: {property_a: value_a}}} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: {parametrized_type: {property_a: value_a, property_b: value_b}}} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - with self.assertRaises(TypeError): - b_task = task_factory_b(in1=a_task.outputs['out1']) - - def test_type_compatibility_check_not_failing_when_type_is_ignored(self): - component_a = '''\ -outputs: - - {name: out1, type: type_A} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: type_Z} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1'].without_type()) - - def test_type_compatibility_check_for_types_with_schema(self): - component_a = '''\ -outputs: - - {name: out1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: "^gs://.*$" } }}} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: "^gs://.*$" } }}} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1']) - - def test_fail_type_compatibility_check_for_types_with_different_schemas( - self): - component_a = '''\ -outputs: - - {name: out1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: AAA } }}} -implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] -''' - component_b = '''\ -inputs: - - {name: in1, type: {GCSPath: {openapi_schema_validator: {type: string, pattern: ZZZ } }}} -implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] -''' - task_factory_a = comp.load_component_from_text(component_a) - task_factory_b = comp.load_component_from_text(component_b) - a_task = task_factory_a() - - with self.assertRaises(TypeError): - b_task = task_factory_b(in1=a_task.outputs['out1']) - - def test_container_component_without_command_should_warn(self): - component_a = '''\ -name: component without command -inputs: - - {name: in1, type: String} -implementation: - container: - image: busybox -''' - - with self.assertWarnsRegex( - FutureWarning, - 'Container component must specify command to be compatible with ' - 'KFP v2 compatible mode and emissary executor'): - task_factory_a = comp.load_component_from_text(component_a) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/components_tests/test_data/component_with_0_inputs_and_2_outputs.component.yaml b/sdk/python/kfp/deprecated/components_tests/test_data/component_with_0_inputs_and_2_outputs.component.yaml deleted file mode 100644 index ea366a2f4bf..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_data/component_with_0_inputs_and_2_outputs.component.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: Component with 0 inputs and 2 outputs -outputs: -- {name: Output 1} -- {name: Output 2} -implementation: - container: - image: busybox - command: [sh, -c, ' - echo "Data 1" > $0 - echo "Data 2" > $1 - ' - ] - args: - - {outputPath: Output 1} - - {outputPath: Output 2} diff --git a/sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_0_outputs.component.yaml b/sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_0_outputs.component.yaml deleted file mode 100644 index 07473a4caba..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_0_outputs.component.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: Component with 2 inputs and 0 outputs -inputs: -- {name: Input parameter} -- {name: Input artifact} -implementation: - container: - image: busybox - command: [sh, -c, ' - echo "Input parameter = $0" - echo "Input artifact = $(< $1)" - ' - ] - args: - - {inputValue: Input parameter} - - {inputPath: Input artifact} diff --git a/sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_2_outputs.component.yaml b/sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_2_outputs.component.yaml deleted file mode 100644 index bd25f001561..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_data/component_with_2_inputs_and_2_outputs.component.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: Component with 2 inputs and 2 outputs -inputs: -- {name: Input parameter} -- {name: Input artifact} -outputs: -- {name: Output 1} -- {name: Output 2} -implementation: - container: - image: busybox - command: [sh, -c, ' - mkdir -p $(dirname "$2") - mkdir -p $(dirname "$3") - echo "$0" > "$2" - cp "$1" "$3" - ' - ] - args: - - {inputValue: Input parameter} - - {inputPath: Input artifact} - - {outputPath: Output 1} - - {outputPath: Output 2} diff --git a/sdk/python/kfp/deprecated/components_tests/test_data/module1.py b/sdk/python/kfp/deprecated/components_tests/test_data/module1.py deleted file mode 100644 index 5d36fe8a4c6..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_data/module1.py +++ /dev/null @@ -1,15 +0,0 @@ -module_level_variable = 10 - - -class ModuleLevelClass: - - def class_method(self, x): - return x * module_level_variable - - -def module_func(a: float) -> float: - return a * 5 - - -def module_func_with_deps(a: float, b: float) -> float: - return ModuleLevelClass().class_method(a) + module_func(b) diff --git a/sdk/python/kfp/deprecated/components_tests/test_data/module2_which_depends_on_module1.py b/sdk/python/kfp/deprecated/components_tests/test_data/module2_which_depends_on_module1.py deleted file mode 100644 index b1520488a96..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_data/module2_which_depends_on_module1.py +++ /dev/null @@ -1,5 +0,0 @@ -from .module1 import module_func_with_deps - - -def module2_func_with_deps(a: float, b: float) -> float: - return module_func_with_deps(a, b) + 10 diff --git a/sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.yaml b/sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.yaml deleted file mode 100644 index 2844e55796b..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.yaml +++ /dev/null @@ -1,35 +0,0 @@ -name: Add -description: | - Returns sum of two arguments -inputs: -- {name: a, type: float} -- {name: b, type: float} -outputs: -- {name: Output, type: float} -implementation: - container: - image: python:3.5 - command: - - python - - -c - - | - from pathlib import Path - - def add(a: float, b: float) -> float: - '''Returns sum of two arguments''' - return a + b - - def add_wrapper(a:float, b:float, output_file): - outputs = add(a, b) - if not isinstance(outputs, tuple): - outputs = (outputs,) - for idx, filename in enumerate([output_file]): - Path(filename).parent.mkdir(parents=True, exist_ok=True) - Path(filename).write_text(str(outputs[idx])) - - import fire - fire.Fire(add_wrapper) - args: - - {inputValue: a} - - {inputValue: b} - - {outputPath: Output} diff --git a/sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.zip b/sdk/python/kfp/deprecated/components_tests/test_data/python_add.component.zip deleted file mode 100644 index 951d7589740d181472118e9491f1e6a890d7bb43..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 547 zcmWIWW@Zs#W?Wn}wBKt3 zfxFMO)2_R0fBWoJo>rl%S&k#eM#mLJuQ(%)@b@0i68v(1<`KV6r55pvHhs_M$n*z` zl^3inU1}(Ig2n9K+XtJOcDVaJQ96=xu>62Ke^=Zj*W(+F^i=wGXEDoHwnQ=SXXo5? zPus5E{Fdus^|+Y5vfA7;HZDBg=-u1pa{i!qlFTHPqMlHFCjR$-wEphDT^}Uta7}Bu zaZ2zm)tfS23iamm`u4Og__5Vomt&!No3-caOV@8SUfLDrdR6L{km-h>^B13)az1P8 zud^nZ=MR`DIUbbH5O4Xdyi#C7OU0wPO>ge)GqyD=e)UKzCiP&^;+dts?B|SUWW=OA z<%E6%R_(fx)2vjo zXKVhgfVq<^zB+JTZ%`Mme1HGgmY=m}Ch|O<_wh(+%B&Fk&Qs5qY5w!ezax1lM)L66 z-kMi4-_P{Ar|Vb!#qizl#jh@Jc=de#s<%sL?PRaLoHu{9et NamedTuple('Outputs', [('model_path', str)]): - # Create dataset - create_dataset_task = automl_create_dataset_for_tables_op( - gcp_project_id=gcp_project_id, - gcp_region=gcp_region, - display_name=dataset_display_name, - ) - - # Import data - import_data_task = automl_import_data_from_bigquery_source_op( - dataset_path=create_dataset_task.outputs['dataset_path'], - input_uri=dataset_bq_input_uri, - ) - - # Prepare column schemas - split_column_specs = automl_split_dataset_table_column_names_op( - dataset_path=import_data_task.outputs['dataset_path'], - table_index=0, - target_column_name=target_column_name, - ) - - # Train a model - create_model_task = automl_create_model_for_tables_op( - gcp_project_id=gcp_project_id, - gcp_region=gcp_region, - display_name=model_display_name, - #dataset_id=create_dataset_task.outputs['dataset_id'], - dataset_id=import_data_task.outputs['dataset_path'], - target_column_path=split_column_specs.outputs['target_column_path'], - #input_feature_column_paths=None, # All non-target columns will be used if None is passed - input_feature_column_paths=split_column_specs.outputs['feature_column_paths'], - optimization_objective='MAXIMIZE_AU_PRC', - train_budget_milli_node_hours=train_budget_milli_node_hours, - ) #.after(import_data_task) - - # Batch prediction - batch_predict_task = automl_prediction_service_batch_predict_op( - model_path=create_model_task.outputs['model_path'], - bq_input_uri=batch_predict_bq_input_uri, - gcs_output_uri_prefix=batch_predict_gcs_output_uri_prefix, - ) - - return [create_model_task.outputs['model_path']] diff --git a/sdk/python/kfp/deprecated/components_tests/test_data_passing.py b/sdk/python/kfp/deprecated/components_tests/test_data_passing.py deleted file mode 100644 index 94aff682c2d..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_data_passing.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from kfp.deprecated.components import _data_passing - - -class DataPassingTest(unittest.TestCase): - - def test_serialize_value(self): - self.assertEqual( - '[1, 2, 3]', - _data_passing.serialize_value( - value=[1, 2, 3], type_name='typing.List[int]')) - - self.assertEqual( - '[4, 5, 6]', - _data_passing.serialize_value(value=[4, 5, 6], type_name='List')) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/components_tests/test_graph_components.py b/sdk/python/kfp/deprecated/components_tests/test_graph_components.py deleted file mode 100644 index 7b5575a354a..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_graph_components.py +++ /dev/null @@ -1,336 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from kfp.deprecated import components as comp -from kfp.deprecated.components.structures import ComponentReference, ComponentSpec, GraphImplementation, GraphInputReference, GraphSpec, InputSpec, OutputSpec, TaskOutputArgument, TaskSpec - -from kfp.deprecated.components._yaml_utils import load_yaml - - -class GraphComponentTestCase(unittest.TestCase): - - def test_handle_constructing_graph_component(self): - task1 = TaskSpec( - component_ref=ComponentReference(name='comp 1'), - arguments={'in1 1': 11}) - task2 = TaskSpec( - component_ref=ComponentReference(name='comp 2'), - arguments={ - 'in2 1': - 21, - 'in2 2': - TaskOutputArgument.construct( - task_id='task 1', output_name='out1 1') - }) - task3 = TaskSpec( - component_ref=ComponentReference(name='comp 3'), - arguments={ - 'in3 1': - TaskOutputArgument.construct( - task_id='task 2', output_name='out2 1'), - 'in3 2': - GraphInputReference(input_name='graph in 1').as_argument() - }) - - graph_component1 = ComponentSpec( - inputs=[ - InputSpec(name='graph in 1'), - InputSpec(name='graph in 2'), - ], - outputs=[ - OutputSpec(name='graph out 1'), - OutputSpec(name='graph out 2'), - ], - implementation=GraphImplementation( - graph=GraphSpec( - tasks={ - 'task 1': task1, - 'task 2': task2, - 'task 3': task3, - }, - output_values={ - 'graph out 1': - TaskOutputArgument.construct( - task_id='task 3', output_name='out3 1'), - 'graph out 2': - TaskOutputArgument.construct( - task_id='task 1', output_name='out1 2'), - }))) - - def test_handle_parsing_graph_component(self): - component_text = '''\ -inputs: -- {name: graph in 1} -- {name: graph in 2} -outputs: -- {name: graph out 1} -- {name: graph out 2} -implementation: - graph: - tasks: - task 1: - componentRef: {name: Comp 1} - arguments: - in1 1: 11 - task 2: - componentRef: {name: Comp 2} - arguments: - in2 1: 21 - in2 2: {taskOutput: {taskId: task 1, outputName: out1 1}} - task 3: - componentRef: {name: Comp 3} - arguments: - in3 1: {taskOutput: {taskId: task 2, outputName: out2 1}} - in3 2: {graphInput: {inputName: graph in 1}} - outputValues: - graph out 1: {taskOutput: {taskId: task 3, outputName: out3 1}} - graph out 2: {taskOutput: {taskId: task 1, outputName: out1 2}} -''' - struct = load_yaml(component_text) - ComponentSpec.from_dict(struct) - - def test_fail_on_cyclic_references(self): - component_text = '''\ -implementation: - graph: - tasks: - task 1: - componentRef: {name: Comp 1} - arguments: - in1 1: {taskOutput: {taskId: task 2, outputName: out2 1}} - task 2: - componentRef: {name: Comp 2} - arguments: - in2 1: {taskOutput: {taskId: task 1, outputName: out1 1}} -''' - struct = load_yaml(component_text) - with self.assertRaises(TypeError): - ComponentSpec.from_dict(struct) - - def test_handle_parsing_predicates(self): - component_text = '''\ -implementation: - graph: - tasks: - task 1: - componentRef: {name: Comp 1} - task 2: - componentRef: {name: Comp 2} - arguments: - in2 1: 21 - in2 2: {taskOutput: {taskId: task 1, outputName: out1 1}} - isEnabled: - not: - and: - op1: - '>': - op1: {taskOutput: {taskId: task 1, outputName: out1 1}} - op2: 0 - op2: - '==': - op1: {taskOutput: {taskId: task 1, outputName: out1 2}} - op2: 'head' -''' - struct = load_yaml(component_text) - ComponentSpec.from_dict(struct) - - def test_handle_parsing_task_execution_options_caching_strategy(self): - component_text = '''\ -implementation: - graph: - tasks: - task 1: - componentRef: {name: Comp 1} - executionOptions: - cachingStrategy: - maxCacheStaleness: P30D -''' - struct = load_yaml(component_text) - component_spec = ComponentSpec.from_dict(struct) - self.assertEqual( - component_spec.implementation.graph.tasks['task 1'] - .execution_options.caching_strategy.max_cache_staleness, 'P30D') - - def test_load_graph_component(self): - component_text = '''\ -inputs: -- {name: graph in 1} -- {name: graph in 2} -outputs: -- {name: graph out 1} -- {name: graph out 2} -- {name: graph out 3} -- {name: graph out 4} -implementation: - graph: - tasks: - task 1: - componentRef: - spec: - name: Component 1 - inputs: - - {name: in1_1} - outputs: - - {name: out1_1} - - {name: out1_2} - implementation: - container: - image: busybox - command: [sh, -c, 'echo "$0" > $1; echo "$0" > $2', {inputValue: in1_1}, {outputPath: out1_1}, {outputPath: out1_2}] - arguments: - in1_1: '11' - task 2: - componentRef: - spec: - name: Component 2 - inputs: - - {name: in2_1} - - {name: in2_2} - outputs: - - {name: out2_1} - implementation: - container: - image: busybox - command: [sh, -c, 'cat "$0" "$1" > $2', {inputValue: in2_1}, {inputValue: in2_2}, {outputPath: out2_1}] - arguments: - in2_1: '21' - in2_2: {taskOutput: {taskId: task 1, outputName: out1_1}} - task 3: - componentRef: - spec: - name: Component 3 - inputs: - - {name: in3_1} - - {name: in3_2} - outputs: - - {name: out3_1} - implementation: - container: - image: busybox - command: [sh, -c, 'cat "$0" "$1" > $2', {inputValue: in3_1}, {inputValue: in3_2}, {outputPath: out3_1}] - arguments: - in3_1: {taskOutput: {taskId: task 2, outputName: out2_1}} - in3_2: {graphInput: {inputName: graph in 1}} - outputValues: - graph out 1: {taskOutput: {taskId: task 3, outputName: out3_1}} - graph out 2: {taskOutput: {taskId: task 1, outputName: out1_2}} - graph out 3: {graphInput: {inputName: graph in 2}} - graph out 4: '42' -''' - op = comp.load_component_from_text(component_text) - task = op('graph 1', 'graph 2') - self.assertEqual(len(task.outputs), 4) - - def test_load_nested_graph_components(self): - component_text = '''\ -inputs: -- {name: graph in 1} -- {name: graph in 2} -outputs: -- {name: graph out 1} -- {name: graph out 2} -- {name: graph out 3} -- {name: graph out 4} -implementation: - graph: - tasks: - task 1: - componentRef: - spec: - name: Component 1 - inputs: - - {name: in1_1} - outputs: - - {name: out1_1} - - {name: out1_2} - implementation: - container: - image: busybox - command: [sh, -c, 'echo "$0" > $1; echo "$0" > $2', {inputValue: in1_1}, {outputPath: out1_1}, {outputPath: out1_2}] - arguments: - in1_1: '11' - task 2: - componentRef: - spec: - name: Component 2 - inputs: - - {name: in2_1} - - {name: in2_2} - outputs: - - {name: out2_1} - implementation: - container: - image: busybox - command: [sh, -c, 'cat "$0" "$1" > $2', {inputValue: in2_1}, {inputValue: in2_2}, {outputPath: out2_1}] - arguments: - in2_1: '21' - in2_2: {taskOutput: {taskId: task 1, outputName: out1_1}} - task 3: - componentRef: - spec: - inputs: - - {name: in3_1} - - {name: in3_2} - outputs: - - {name: out3_1} - implementation: - graph: - tasks: - graph subtask: - componentRef: - spec: - name: Component 3 - inputs: - - {name: in3_1} - - {name: in3_2} - outputs: - - {name: out3_1} - implementation: - container: - image: busybox - command: [sh, -c, 'cat "$0" "$1" > $2', {inputValue: in3_1}, {inputValue: in3_2}, {outputPath: out3_1}] - arguments: - in3_1: {graphInput: {inputName: in3_1}} - in3_2: {graphInput: {inputName: in3_1}} - outputValues: - out3_1: {taskOutput: {taskId: graph subtask, outputName: out3_1}} - arguments: - in3_1: {taskOutput: {taskId: task 2, outputName: out2_1}} - in3_2: {graphInput: {inputName: graph in 1}} - outputValues: - graph out 1: {taskOutput: {taskId: task 3, outputName: out3_1}} - graph out 2: {taskOutput: {taskId: task 1, outputName: out1_2}} - graph out 3: {graphInput: {inputName: graph in 2}} - graph out 4: '42' -''' - op = comp.load_component_from_text(component_text) - old_value = comp._components._always_expand_graph_components = True - try: - task = op('graph 1', 'graph 2') - finally: - comp._components._always_expand_graph_components = old_value - self.assertIn( - 'out3_1', str(task.outputs['graph out 1']) - ) # Checks that the outputs coming from tasks in nested subgraphs are properly resolved. - self.assertIn('out1_2', str(task.outputs['graph out 2'])) - self.assertEqual(task.outputs['graph out 3'], 'graph 2') - self.assertEqual(task.outputs['graph out 4'], '42') - - -#TODO: Test task name conversion to Argo-compatible names - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/components_tests/test_python_op.py b/sdk/python/kfp/deprecated/components_tests/test_python_op.py deleted file mode 100644 index a116ad645ac..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_python_op.py +++ /dev/null @@ -1,1244 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import subprocess -import sys -import tempfile -import unittest -from contextlib import contextmanager -from pathlib import Path -from typing import Callable, NamedTuple, Sequence - -from kfp.deprecated import components as comp -from kfp.deprecated.components import (InputBinaryFile, InputPath, - InputTextFile, OutputBinaryFile, - OutputPath, OutputTextFile) -from kfp.deprecated.components._components import _resolve_command_line_and_paths -from kfp.deprecated.components.structures import InputSpec, OutputSpec - - -def add_two_numbers(a: float, b: float) -> float: - """Returns sum of two arguments.""" - return a + b - - -@contextmanager -def components_local_output_dir_context(output_dir: str): - old_dir = comp._components._outputs_dir - try: - comp._components._outputs_dir = output_dir - yield output_dir - finally: - comp._components._outputs_dir = old_dir - - -@contextmanager -def components_override_input_output_dirs_context(inputs_dir: str, - outputs_dir: str): - old_inputs_dir = comp._components._inputs_dir - old_outputs_dir = comp._components._outputs_dir - try: - if inputs_dir: - comp._components._inputs_dir = inputs_dir - if outputs_dir: - comp._components._outputs_dir = outputs_dir - yield - finally: - comp._components._inputs_dir = old_inputs_dir - comp._components._outputs_dir = old_outputs_dir - - -module_level_variable = 10 - - -class ModuleLevelClass: - - def class_method(self, x): - return x * module_level_variable - - -def module_func(a: float) -> float: - return a * 5 - - -def module_func_with_deps(a: float, b: float) -> float: - return ModuleLevelClass().class_method(a) + module_func(b) - - -def dummy_in_0_out_0(): - pass - - -class PythonOpTestCase(unittest.TestCase): - - def helper_test_2_in_1_out_component_using_local_call( - self, func, op, arguments=[3., 5.]): - expected = func(arguments[0], arguments[1]) - if isinstance(expected, tuple): - expected = expected[0] - expected_str = str(expected) - - with tempfile.TemporaryDirectory() as temp_dir_name: - with components_local_output_dir_context(temp_dir_name): - task = op(arguments[0], arguments[1]) - resolved_cmd = _resolve_command_line_and_paths( - task.component_ref.spec, - task.arguments, - ) - - full_command = resolved_cmd.command + resolved_cmd.args - # Setting the component python interpreter to the current one. - # Otherwise the components are executed in different environment. - # Some components (e.g. the ones that use code pickling) are sensitive to this. - for i in range(2): - if full_command[i] == 'python3': - full_command[i] = sys.executable - subprocess.run(full_command, check=True) - - output_path = list(resolved_cmd.output_paths.values())[0] - actual_str = Path(output_path).read_text() - - self.assertEqual(float(actual_str), float(expected_str)) - - def helper_test_2_in_2_out_component_using_local_call( - self, func, op, output_names): - arg1 = float(3) - arg2 = float(5) - - expected_tuple = func(arg1, arg2) - expected1_str = str(expected_tuple[0]) - expected2_str = str(expected_tuple[1]) - - with tempfile.TemporaryDirectory() as temp_dir_name: - with components_local_output_dir_context(temp_dir_name): - task = op(arg1, arg2) - resolved_cmd = _resolve_command_line_and_paths( - task.component_ref.spec, - task.arguments, - ) - - full_command = resolved_cmd.command + resolved_cmd.args - # Setting the component python interpreter to the current one. - # Otherwise the components are executed in different environment. - # Some components (e.g. the ones that use code pickling) are sensitive to this. - for i in range(2): - if full_command[i] == 'python3': - full_command[i] = sys.executable - - subprocess.run(full_command, check=True) - - (output_path1, - output_path2) = (resolved_cmd.output_paths[output_names[0]], - resolved_cmd.output_paths[output_names[1]]) - actual1_str = Path(output_path1).read_text() - actual2_str = Path(output_path2).read_text() - - self.assertEqual(float(actual1_str), float(expected1_str)) - self.assertEqual(float(actual2_str), float(expected2_str)) - - def helper_test_component_created_from_func_using_local_call( - self, func: Callable, arguments: dict): - task_factory = comp.func_to_container_op(func) - self.helper_test_component_against_func_using_local_call( - func, task_factory, arguments) - - def helper_test_component_against_func_using_local_call( - self, func: Callable, op: Callable, arguments: dict): - # ! This function cannot be used when component has output types that use custom serialization since it will compare non-serialized function outputs with serialized component outputs. - # Evaluating the function to get the expected output values - expected_output_values_list = func(**arguments) - if not isinstance(expected_output_values_list, Sequence) or isinstance( - expected_output_values_list, str): - expected_output_values_list = [str(expected_output_values_list)] - expected_output_values_list = [ - str(value) for value in expected_output_values_list - ] - - output_names = [output.name for output in op.component_spec.outputs] - expected_output_values_dict = dict( - zip(output_names, expected_output_values_list)) - - self.helper_test_component_using_local_call( - op, arguments, expected_output_values_dict) - - def helper_test_component_using_local_call( - self, - component_task_factory: Callable, - arguments: dict = None, - expected_output_values: dict = None): - arguments = arguments or {} - expected_output_values = expected_output_values or {} - with tempfile.TemporaryDirectory() as temp_dir_name: - # Creating task from the component. - # We do it in a special context that allows us to control the output file locations. - inputs_path = Path(temp_dir_name) / 'inputs' - outputs_path = Path(temp_dir_name) / 'outputs' - with components_override_input_output_dirs_context( - str(inputs_path), str(outputs_path)): - task = component_task_factory(**arguments) - resolved_cmd = _resolve_command_line_and_paths( - task.component_ref.spec, - task.arguments, - ) - - # Preparing input files - for input_name, input_file_path in (resolved_cmd.input_paths or - {}).items(): - Path(input_file_path).parent.mkdir(parents=True, exist_ok=True) - Path(input_file_path).write_text(str(arguments[input_name])) - - # Constructing the full command-line from resolved command+args - full_command = resolved_cmd.command + resolved_cmd.args - # Setting the component python interpreter to the current one. - # Otherwise the components are executed in different environment. - # Some components (e.g. the ones that use code pickling) are sensitive to this. - for i in range(2): - if full_command[i] == 'python3': - full_command[i] = sys.executable - - # Executing the command-line locally - subprocess.run(full_command, check=True) - - actual_output_values_dict = { - output_name: Path(output_path).read_text() for output_name, - output_path in resolved_cmd.output_paths.items() - } - - self.assertDictEqual(actual_output_values_dict, expected_output_values) - - def test_func_to_container_op_local_call(self): - func = add_two_numbers - op = comp.func_to_container_op(func) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - def test_func_to_container_op_output_component_file(self): - func = add_two_numbers - with tempfile.TemporaryDirectory() as temp_dir_name: - component_path = str(Path(temp_dir_name) / 'component.yaml') - comp.func_to_container_op( - func, output_component_file=component_path) - op = comp.load_component_from_file(component_path) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - def test_func_to_component_file(self): - func = add_two_numbers - with tempfile.TemporaryDirectory() as temp_dir_name: - component_path = str(Path(temp_dir_name) / 'component.yaml') - comp._python_op.func_to_component_file( - func, output_component_file=component_path) - op = comp.load_component_from_file(component_path) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - def test_indented_func_to_container_op_local_call(self): - - def add_two_numbers_indented(a: float, b: float) -> float: - """Returns sum of two arguments.""" - return a + b - - func = add_two_numbers_indented - op = comp.func_to_container_op(func) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - def test_func_to_container_op_call_other_func(self): - extra_variable = 10 - - class ExtraClass: - - def class_method(self, x): - return x * extra_variable - - def extra_func(a: float) -> float: - return a * 5 - - def main_func(a: float, b: float) -> float: - return ExtraClass().class_method(a) + extra_func(b) - - func = main_func - op = comp.func_to_container_op(func, use_code_pickling=True) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - def test_func_to_container_op_check_nothing_extra_captured(self): - - def f1(): - pass - - def f2(): - pass - - def main_func(a: float, b: float) -> float: - f1() - try: - eval('f2()') - except: - return a + b - raise AssertionError( - "f2 should not be captured, because it's not a dependency.") - - expected_func = lambda a, b: a + b - op = comp.func_to_container_op(main_func, use_code_pickling=True) - - self.helper_test_2_in_1_out_component_using_local_call( - expected_func, op) - - def test_func_to_container_op_call_other_func_global(self): - func = module_func_with_deps - op = comp.func_to_container_op(func, use_code_pickling=True) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - def test_func_to_container_op_with_imported_func(self): - from kfp.deprecated.components_tests.test_data.module1 import \ - module_func_with_deps as module1_func_with_deps - func = module1_func_with_deps - op = comp.func_to_container_op(func, use_code_pickling=True) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - def test_func_to_container_op_with_imported_func2(self): - from kfp.deprecated.components_tests.test_data import module1, module2_which_depends_on_module1 - func = module2_which_depends_on_module1.module2_func_with_deps - op = comp.func_to_container_op( - func, - use_code_pickling=True, - modules_to_capture=[ - module1.__name__, # '*.components_tests.test_data.module1' - func. - __module__, # '*.components_tests.test_data.module2_which_depends_on_module1' - ]) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - def test_func_to_container_op_multiple_named_typed_outputs(self): - from typing import NamedTuple - - def add_multiply_two_numbers( - a: float, b: float - ) -> NamedTuple('DummyName', [('sum', float), ('product', float)]): - """Returns sum and product of two arguments.""" - return (a + b, a * b) - - func = add_multiply_two_numbers - op = comp.func_to_container_op(func) - - self.helper_test_2_in_2_out_component_using_local_call( - func, op, output_names=['sum', 'product']) - - def test_extract_component_interface(self): - from typing import NamedTuple - - def my_func( # noqa: F722 - required_param, - int_param: int = 42, - float_param: float = 3.14, - str_param: str = 'string', - bool_param: bool = True, - none_param=None, - custom_type_param: 'Custom type' = None, - custom_struct_type_param: { - 'CustomType': { - 'param1': 'value1', - 'param2': 'value2' - } - } = None, - ) -> NamedTuple( - 'DummyName', - [ - #('required_param',), # All typing.NamedTuple fields must have types - ('int_param', int), - ('float_param', float), - ('str_param', str), - ('bool_param', bool), - #('custom_type_param', 'Custom type'), #SyntaxError: Forward reference must be an expression -- got 'Custom type' - ('custom_type_param', 'CustomType'), - #('custom_struct_type_param', {'CustomType': {'param1': 'value1', 'param2': 'value2'}}), # TypeError: NamedTuple('Name', [(f0, t0), (f1, t1), ...]); each t must be a type Got {'CustomType': {'param1': 'value1', 'param2': 'value2'}} - ]): - """Function docstring.""" - - component_spec = comp._python_op._extract_component_interface(my_func) - - self.assertEqual( - component_spec.inputs, - [ - InputSpec(name='required_param'), - InputSpec( - name='int_param', - type='Integer', - default='42', - optional=True), - InputSpec( - name='float_param', - type='Float', - default='3.14', - optional=True), - InputSpec( - name='str_param', - type='String', - default='string', - optional=True), - InputSpec( - name='bool_param', - type='Boolean', - default='True', - optional=True), - InputSpec(name='none_param', - optional=True), # No default='None' - InputSpec( - name='custom_type_param', type='Custom type', - optional=True), - InputSpec( - name='custom_struct_type_param', - type={ - 'CustomType': { - 'param1': 'value1', - 'param2': 'value2' - } - }, - optional=True), - ]) - self.assertEqual( - component_spec.outputs, - [ - OutputSpec(name='int_param', type='Integer'), - OutputSpec(name='float_param', type='Float'), - OutputSpec(name='str_param', type='String'), - OutputSpec(name='bool_param', type='Boolean'), - #OutputSpec(name='custom_type_param', type='Custom type', default='None'), - OutputSpec(name='custom_type_param', type='CustomType'), - #OutputSpec(name='custom_struct_type_param', type={'CustomType': {'param1': 'value1', 'param2': 'value2'}}, optional=True), - ]) - - self.maxDiff = None - self.assertDictEqual( - component_spec.to_dict(), - { - 'name': - 'My func', - 'description': - 'Function docstring.', - 'inputs': [ - { - 'name': 'required_param' - }, - { - 'name': 'int_param', - 'type': 'Integer', - 'default': '42', - 'optional': True - }, - { - 'name': 'float_param', - 'type': 'Float', - 'default': '3.14', - 'optional': True - }, - { - 'name': 'str_param', - 'type': 'String', - 'default': 'string', - 'optional': True - }, - { - 'name': 'bool_param', - 'type': 'Boolean', - 'default': 'True', - 'optional': True - }, - { - 'name': 'none_param', - 'optional': True - }, # No default='None' - { - 'name': 'custom_type_param', - 'type': 'Custom type', - 'optional': True - }, - { - 'name': 'custom_struct_type_param', - 'type': { - 'CustomType': { - 'param1': 'value1', - 'param2': 'value2' - } - }, - 'optional': True - }, - ], - 'outputs': [ - { - 'name': 'int_param', - 'type': 'Integer' - }, - { - 'name': 'float_param', - 'type': 'Float' - }, - { - 'name': 'str_param', - 'type': 'String' - }, - { - 'name': 'bool_param', - 'type': 'Boolean' - }, - { - 'name': 'custom_type_param', - 'type': 'CustomType' - }, - #{'name': 'custom_struct_type_param', 'type': {'CustomType': {'param1': 'value1', 'param2': 'value2'}}, 'optional': True}, - ] - }) - - @unittest.skip #TODO: #Simplified multi-output syntax is not implemented yet - def test_func_to_container_op_multiple_named_typed_outputs_using_list_syntax( - self): - - def add_multiply_two_numbers( - a: float, b: float) -> [('sum', float), ('product', float)]: - """Returns sum and product of two arguments.""" - return (a + b, a * b) - - func = add_multiply_two_numbers - op = comp.func_to_container_op(func) - - self.helper_test_2_in_2_out_component_using_local_call( - func, op, output_names=['sum', 'product']) - - def test_func_to_container_op_named_typed_outputs_with_underscores(self): - from typing import NamedTuple - - def add_two_numbers_name2( - a: float, - b: float) -> NamedTuple('DummyName', [('output_data', float)]): - """Returns sum of two arguments.""" - return (a + b,) - - func = add_two_numbers_name2 - op = comp.func_to_container_op(func) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - @unittest.skip #Python does not allow NamedTuple with spaces in names: ValueError: Type names and field names must be valid identifiers: 'Output data' - def test_func_to_container_op_named_typed_outputs_with_spaces(self): - from typing import NamedTuple - - def add_two_numbers_name3( - a: float, - b: float) -> NamedTuple('DummyName', [('Output data', float)]): - """Returns sum of two arguments.""" - return (a + b,) - - func = add_two_numbers_name3 - op = comp.func_to_container_op(func) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - def test_handling_same_input_output_names(self): - from typing import NamedTuple - - def add_multiply_two_numbers( - a: float, b: float - ) -> NamedTuple('DummyName', [('a', float), ('b', float)]): - """Returns sum and product of two arguments.""" - return (a + b, a * b) - - func = add_multiply_two_numbers - op = comp.func_to_container_op(func) - - self.helper_test_2_in_2_out_component_using_local_call( - func, op, output_names=['a', 'b']) - - def test_handling_same_input_default_output_names(self): - - def add_two_numbers_indented(a: float, Output: float) -> float: - """Returns sum of two arguments.""" - return a + Output - - func = add_two_numbers_indented - op = comp.func_to_container_op(func) - - self.helper_test_2_in_1_out_component_using_local_call(func, op) - - def test_legacy_python_component_name_description_overrides(self): - # Deprecated feature - - expected_name = 'Sum component name' - expected_description = 'Sum component description' - expected_image = 'org/image' - - def add_two_numbers_decorated( - a: float, - b: float, - ) -> float: - """Returns sum of two arguments.""" - return a + b - - # Deprecated features - add_two_numbers_decorated._component_human_name = expected_name - add_two_numbers_decorated._component_description = expected_description - add_two_numbers_decorated._component_base_image = expected_image - - func = add_two_numbers_decorated - op = comp.func_to_container_op(func) - - component_spec = op.component_spec - - self.assertEqual(component_spec.name, expected_name) - self.assertEqual(component_spec.description.strip(), - expected_description.strip()) - self.assertEqual(component_spec.implementation.container.image, - expected_image) - - def test_saving_default_values(self): - from typing import NamedTuple - - def add_multiply_two_numbers( - a: float = 3, - b: float = 5 - ) -> NamedTuple('DummyName', [('sum', float), ('product', float)]): - """Returns sum and product of two arguments.""" - return (a + b, a * b) - - func = add_multiply_two_numbers - component_spec = comp._python_op._func_to_component_spec(func) - - self.assertEqual(component_spec.inputs[0].default, '3') - self.assertEqual(component_spec.inputs[1].default, '5') - - def test_handling_of_descriptions(self): - - def pipeline(env_var: str, - secret_name: str, - secret_key: str = None) -> None: - """Pipeline to Demonstrate Usage of Secret. - - Args: - env_var: Name of the variable inside the Pod - secret_name: Name of the Secret in the namespace - """ - - component_spec = comp._python_op._func_to_component_spec(pipeline) - self.assertEqual(component_spec.description, - 'Pipeline to Demonstrate Usage of Secret.') - self.assertEqual(component_spec.inputs[0].description, - 'Name of the variable inside the Pod') - self.assertEqual(component_spec.inputs[1].description, - 'Name of the Secret in the namespace') - self.assertIsNone(component_spec.inputs[2].description) - - def test_handling_default_value_of_none(self): - - def assert_is_none(arg=None): - assert arg is None - - func = assert_is_none - op = comp.func_to_container_op(func) - self.helper_test_component_using_local_call(op) - - def test_handling_complex_default_values(self): - - def assert_values_are_default( - singleton_param=None, - function_param=ascii, - dict_param: dict = {'b': [2, 3, 4]}, - func_call_param='_'.join(['a', 'b', 'c']), - ): - assert singleton_param is None - assert function_param is ascii - assert dict_param == {'b': [2, 3, 4]} - assert func_call_param == '_'.join(['a', 'b', 'c']) - - func = assert_values_are_default - op = comp.func_to_container_op(func) - self.helper_test_component_using_local_call(op) - - def test_handling_boolean_arguments(self): - - def assert_values_are_true_false( - bool1: bool, - bool2: bool, - ) -> int: - assert bool1 is True - assert bool2 is False - return 1 - - func = assert_values_are_true_false - op = comp.func_to_container_op(func) - self.helper_test_2_in_1_out_component_using_local_call( - func, op, arguments=[True, False]) - - def test_handling_list_dict_arguments(self): - - def assert_values_are_same( - list_param: list, - dict_param: dict, - ) -> int: - import unittest - unittest.TestCase().assertEqual( - list_param, - ["string", 1, 2.2, True, False, None, [3, 4], { - 's': 5 - }]) - unittest.TestCase().assertEqual( - dict_param, { - 'str': "string", - 'int': 1, - 'float': 2.2, - 'false': False, - 'true': True, - 'none': None, - 'list': [3, 4], - 'dict': { - 's': 4 - } - }) - return 1 - - # ! JSON map keys are always strings. Python converts all keys to strings without warnings - func = assert_values_are_same - op = comp.func_to_container_op(func) - self.helper_test_2_in_1_out_component_using_local_call( - func, - op, - arguments=[ - ["string", 1, 2.2, True, False, None, [3, 4], { - 's': 5 - }], - { - 'str': "string", - 'int': 1, - 'float': 2.2, - 'false': False, - 'true': True, - 'none': None, - 'list': [3, 4], - 'dict': { - 's': 4 - } - }, - ]) - - def test_fail_on_handling_list_arguments_containing_python_objects(self): - """Checks that lists containing python objects not having .to_struct() - raise error during serialization.""" - - class MyClass: - pass - - def consume_list(list_param: list,) -> int: - return 1 - - def consume_dict(dict_param: dict,) -> int: - return 1 - - list_op = comp.create_component_from_func(consume_list) - dict_op = comp.create_component_from_func(consume_dict) - - with self.assertRaises(Exception): - list_op([1, MyClass(), 3]) - - with self.assertRaises(Exception): - dict_op({'k1': MyClass()}) - - def test_handling_list_arguments_containing_serializable_python_objects( - self): - """Checks that lists containing python objects with .to_struct() can be - properly serialized.""" - - class MyClass: - - def to_struct(self): - return {'foo': [7, 42]} - - def assert_values_are_correct( - list_param: list, - dict_param: dict, - ) -> int: - import unittest - unittest.TestCase().assertEqual(list_param, - [1, { - 'foo': [7, 42] - }, 3]) - unittest.TestCase().assertEqual(dict_param, - {'k1': { - 'foo': [7, 42] - }}) - return 1 - - task_factory = comp.create_component_from_func( - assert_values_are_correct) - - self.helper_test_component_using_local_call( - task_factory, - arguments=dict( - list_param=[1, MyClass(), 3], - dict_param={'k1': MyClass()}, - ), - expected_output_values={'Output': '1'}, - ) - - def test_handling_base64_pickle_arguments(self): - - def assert_values_are_same( - obj1: 'Base64Pickle', # noqa: F821 - obj2: 'Base64Pickle', # noqa: F821 - ) -> int: - import unittest - unittest.TestCase().assertEqual(obj1['self'], obj1) - unittest.TestCase().assertEqual(obj2, open) - return 1 - - func = assert_values_are_same - op = comp.func_to_container_op(func) - - recursive_obj = {} - recursive_obj['self'] = recursive_obj - self.helper_test_2_in_1_out_component_using_local_call( - func, op, arguments=[ - recursive_obj, - open, - ]) - - def test_handling_list_dict_output_values(self): - - def produce_list() -> list: - return ["string", 1, 2.2, True, False, None, [3, 4], {'s': 5}] - - # ! JSON map keys are always strings. Python converts all keys to strings without warnings - task_factory = comp.func_to_container_op(produce_list) - - import json - expected_output = json.dumps( - ["string", 1, 2.2, True, False, None, [3, 4], { - 's': 5 - }]) - - self.helper_test_component_using_local_call( - task_factory, - arguments={}, - expected_output_values={'Output': expected_output}) - - def test_input_path(self): - - def consume_file_path(number_file_path: InputPath(int)) -> int: - with open(number_file_path) as f: - string_data = f.read() - return int(string_data) - - task_factory = comp.func_to_container_op(consume_file_path) - - self.assertEqual(task_factory.component_spec.inputs[0].type, 'Integer') - - self.helper_test_component_using_local_call( - task_factory, - arguments={'number': "42"}, - expected_output_values={'Output': '42'}) - - def test_input_text_file(self): - - def consume_file_path(number_file: InputTextFile(int)) -> int: - string_data = number_file.read() - assert isinstance(string_data, str) - return int(string_data) - - task_factory = comp.func_to_container_op(consume_file_path) - - self.assertEqual(task_factory.component_spec.inputs[0].type, 'Integer') - - self.helper_test_component_using_local_call( - task_factory, - arguments={'number': "42"}, - expected_output_values={'Output': '42'}) - - def test_input_binary_file(self): - - def consume_file_path(number_file: InputBinaryFile(int)) -> int: - bytes_data = number_file.read() - assert isinstance(bytes_data, bytes) - return int(bytes_data) - - task_factory = comp.func_to_container_op(consume_file_path) - - self.assertEqual(task_factory.component_spec.inputs[0].type, 'Integer') - - self.helper_test_component_using_local_call( - task_factory, - arguments={'number': "42"}, - expected_output_values={'Output': '42'}) - - def test_output_path(self): - - def write_to_file_path(number_file_path: OutputPath(int)): - with open(number_file_path, 'w') as f: - f.write(str(42)) - - task_factory = comp.func_to_container_op(write_to_file_path) - - self.assertFalse(task_factory.component_spec.inputs) - self.assertEqual(len(task_factory.component_spec.outputs), 1) - self.assertEqual(task_factory.component_spec.outputs[0].type, 'Integer') - - self.helper_test_component_using_local_call( - task_factory, arguments={}, expected_output_values={'number': '42'}) - - def test_output_text_file(self): - - def write_to_file_path(number_file: OutputTextFile(int)): - number_file.write(str(42)) - - task_factory = comp.func_to_container_op(write_to_file_path) - - self.assertFalse(task_factory.component_spec.inputs) - self.assertEqual(len(task_factory.component_spec.outputs), 1) - self.assertEqual(task_factory.component_spec.outputs[0].type, 'Integer') - - self.helper_test_component_using_local_call( - task_factory, arguments={}, expected_output_values={'number': '42'}) - - def test_output_binary_file(self): - - def write_to_file_path(number_file: OutputBinaryFile(int)): - number_file.write(b'42') - - task_factory = comp.func_to_container_op(write_to_file_path) - - self.assertFalse(task_factory.component_spec.inputs) - self.assertEqual(len(task_factory.component_spec.outputs), 1) - self.assertEqual(task_factory.component_spec.outputs[0].type, 'Integer') - - self.helper_test_component_using_local_call( - task_factory, arguments={}, expected_output_values={'number': '42'}) - - def test_output_path_plus_return_value(self): - - def write_to_file_path(number_file_path: OutputPath(int)) -> str: - with open(number_file_path, 'w') as f: - f.write(str(42)) - return 'Hello' - - task_factory = comp.func_to_container_op(write_to_file_path) - - self.assertFalse(task_factory.component_spec.inputs) - self.assertEqual(len(task_factory.component_spec.outputs), 2) - self.assertEqual(task_factory.component_spec.outputs[0].type, 'Integer') - self.assertEqual(task_factory.component_spec.outputs[1].type, 'String') - - self.helper_test_component_using_local_call( - task_factory, - arguments={}, - expected_output_values={ - 'number': '42', - 'Output': 'Hello' - }) - - def test_all_data_passing_ways(self): - - def write_to_file_path( - file_input1_path: InputPath(str), - file_input2_file: InputTextFile(str), - file_output1_path: OutputPath(str), - file_output2_file: OutputTextFile(str), - value_input1: str = 'foo', - value_input2: str = 'foo', - ) -> NamedTuple('Outputs', [ - ('return_output1', str), - ('return_output2', str), - ]): - with open(file_input1_path, 'r') as file_input1_file: - with open(file_output1_path, 'w') as file_output1_file: - file_output1_file.write(file_input1_file.read()) - - file_output2_file.write(file_input2_file.read()) - - return (value_input1, value_input2) - - task_factory = comp.func_to_container_op(write_to_file_path) - - self.assertEqual( - set(input.name for input in task_factory.component_spec.inputs), - {'file_input1', 'file_input2', 'value_input1', 'value_input2'}) - self.assertEqual( - set(output.name for output in task_factory.component_spec.outputs), - { - 'file_output1', 'file_output2', 'return_output1', - 'return_output2' - }) - - self.helper_test_component_using_local_call( - task_factory, - arguments={ - 'file_input1': 'file_input1_value', - 'file_input2': 'file_input2_value', - 'value_input1': 'value_input1_value', - 'value_input2': 'value_input2_value', - }, - expected_output_values={ - 'file_output1': 'file_input1_value', - 'file_output2': 'file_input2_value', - 'return_output1': 'value_input1_value', - 'return_output2': 'value_input2_value', - }, - ) - - def test_optional_input_path(self): - - def consume_file_path(number_file_path: InputPath(int) = None) -> int: - result = -1 - if number_file_path: - with open(number_file_path) as f: - string_data = f.read() - result = int(string_data) - return result - - task_factory = comp.func_to_container_op(consume_file_path) - - self.helper_test_component_using_local_call( - task_factory, arguments={}, expected_output_values={'Output': '-1'}) - - self.helper_test_component_using_local_call( - task_factory, - arguments={'number': "42"}, - expected_output_values={'Output': '42'}) - - def test_fail_on_input_path_non_none_default(self): - - def read_from_file_path( - file_path: InputPath(int) = '/tmp/something') -> str: - return file_path - - with self.assertRaises(ValueError): - task_factory = comp.func_to_container_op(read_from_file_path) - - def test_fail_on_output_path_default(self): - - def write_to_file_path(file_path: OutputPath(int) = None) -> str: - return file_path - - with self.assertRaises(ValueError): - task_factory = comp.func_to_container_op(write_to_file_path) - - def test_annotations_stripping(self): - import collections - import typing - - MyFuncOutputs = typing.NamedTuple('Outputs', [('sum', int), - ('product', int)]) - - class CustomType1: - pass - - def my_func( - param1: CustomType1 = None, # This caused failure previously - param2: collections - .OrderedDict = None, # This caused failure previously - ) -> MyFuncOutputs: # This caused failure previously - assert param1 == None - assert param2 == None - return (8, 15) - - task_factory = comp.create_component_from_func(my_func) - - self.helper_test_component_using_local_call( - task_factory, - arguments={}, - expected_output_values={ - 'sum': '8', - 'product': '15' - }) - - def test_file_input_name_conversion(self): - # Checking the input name conversion rules for file inputs: - # For InputPath, the "_path" suffix is removed - # For Input*, the "_file" suffix is removed - - def consume_file_path( - number: int, - number_1a_path: str, - number_1b_file: str, - number_1c_file_path: str, - number_1d_path_file: str, - number_2a_path: InputPath(str), - number_2b_file: InputPath(str), - number_2c_file_path: InputPath(str), - number_2d_path_file: InputPath(str), - number_3a_path: InputTextFile(str), - number_3b_file: InputTextFile(str), - number_3c_file_path: InputTextFile(str), - number_3d_path_file: InputTextFile(str), - number_4a_path: InputBinaryFile(str), - number_4b_file: InputBinaryFile(str), - number_4c_file_path: InputBinaryFile(str), - number_4d_path_file: InputBinaryFile(str), - output_number_2a_path: OutputPath(str), - output_number_2b_file: OutputPath(str), - output_number_2c_file_path: OutputPath(str), - output_number_2d_path_file: OutputPath(str), - output_number_3a_path: OutputTextFile(str), - output_number_3b_file: OutputTextFile(str), - output_number_3c_file_path: OutputTextFile(str), - output_number_3d_path_file: OutputTextFile(str), - output_number_4a_path: OutputBinaryFile(str), - output_number_4b_file: OutputBinaryFile(str), - output_number_4c_file_path: OutputBinaryFile(str), - output_number_4d_path_file: OutputBinaryFile(str), - ): - pass - - task_factory = comp.func_to_container_op(consume_file_path) - actual_input_names = [ - input.name for input in task_factory.component_spec.inputs - ] - actual_output_names = [ - output.name for output in task_factory.component_spec.outputs - ] - - self.assertEqual([ - 'number', - 'number_1a_path', - 'number_1b_file', - 'number_1c_file_path', - 'number_1d_path_file', - 'number_2a', - 'number_2b', - 'number_2c', - 'number_2d_path', - 'number_3a_path', - 'number_3b', - 'number_3c_file_path', - 'number_3d_path', - 'number_4a_path', - 'number_4b', - 'number_4c_file_path', - 'number_4d_path', - ], actual_input_names) - - self.assertEqual([ - 'output_number_2a', - 'output_number_2b', - 'output_number_2c', - 'output_number_2d_path', - 'output_number_3a_path', - 'output_number_3b', - 'output_number_3c_file_path', - 'output_number_3d_path', - 'output_number_4a_path', - 'output_number_4b', - 'output_number_4c_file_path', - 'output_number_4d_path', - ], actual_output_names) - - def test_packages_to_install_feature(self): - task_factory = comp.func_to_container_op( - dummy_in_0_out_0, packages_to_install=['six', 'pip']) - - self.helper_test_component_using_local_call( - task_factory, arguments={}, expected_output_values={}) - - task_factory2 = comp.func_to_container_op( - dummy_in_0_out_0, - packages_to_install=[ - 'bad-package-0ee7cf93f396cd5072603dec154425cd53bf1c681c7c7605c60f8faf7799b901' - ]) - with self.assertRaises(Exception): - self.helper_test_component_using_local_call( - task_factory2, arguments={}, expected_output_values={}) - - def test_component_annotations(self): - - def some_func(): - pass - - annotations = { - 'key1': 'value1', - 'key2': 'value2', - } - task_factory = comp.create_component_from_func( - some_func, annotations=annotations) - component_spec = task_factory.component_spec - self.assertEqual(component_spec.metadata.annotations, annotations) - - def test_code_with_escapes(self): - - def my_func(): - """Hello \n world.""" - - task_factory = comp.create_component_from_func(my_func) - self.helper_test_component_using_local_call( - task_factory, arguments={}, expected_output_values={}) - - def test_end_to_end_python_component_pipeline(self): - #Defining the Python function - def add(a: float, b: float) -> float: - """Returns sum of two arguments.""" - return a + b - - with tempfile.TemporaryDirectory() as temp_dir_name: - add_component_file = str( - Path(temp_dir_name).joinpath('add.component.yaml')) - - #Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance) - add_op = comp.func_to_container_op( - add, - base_image='python:3.5', - output_component_file=add_component_file) - - #Checking that the component artifact is usable: - add_op2 = comp.load_component_from_file(add_component_file) - - #Building the pipeline - def calc_pipeline( - a1, - a2='7', - a3='17', - ): - task_1 = add_op(a1, a2) - task_2 = add_op2(a1, a2) - task_3 = add_op(task_1.outputs['Output'], - task_2.outputs['Output']) - task_4 = add_op2(task_3.outputs['Output'], a3) - - #Instantiating the pipleine: - calc_pipeline(42) - - def test_argument_serialization_failure(self): - from typing import Sequence - - def my_func(args: Sequence[int]): - args - - task_factory = comp.create_component_from_func(my_func) - with self.assertRaisesRegex( - TypeError, - r'There are no registered serializers for type "(typing.)?Sequence(\[int\])?"' - ): - self.helper_test_component_using_local_call( - task_factory, arguments={'args': [1, 2, 3]}) - - def test_argument_serialization_success(self): - from typing import List - - def my_func(args: List[int]): - args - - task_factory = comp.create_component_from_func(my_func) - self.helper_test_component_using_local_call( - task_factory, arguments={'args': [1, 2, 3]}) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/components_tests/test_python_pipeline_to_graph_component.py b/sdk/python/kfp/deprecated/components_tests/test_python_pipeline_to_graph_component.py deleted file mode 100644 index 8109f308128..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_python_pipeline_to_graph_component.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -from collections import OrderedDict -from pathlib import Path - -from kfp.deprecated import components as comp -from kfp.deprecated.components._python_to_graph_component import create_graph_component_spec_from_pipeline_func - - -class PythonPipelineToGraphComponentTestCase(unittest.TestCase): - - def test_handle_creating_graph_component_from_pipeline_that_uses_container_components( - self): - test_data_dir = Path(__file__).parent / 'test_data' - producer_op = comp.load_component_from_file( - str(test_data_dir / - 'component_with_0_inputs_and_2_outputs.component.yaml')) - processor_op = comp.load_component_from_file( - str(test_data_dir / - 'component_with_2_inputs_and_2_outputs.component.yaml')) - consumer_op = comp.load_component_from_file( - str(test_data_dir / - 'component_with_2_inputs_and_0_outputs.component.yaml')) - - def pipeline1(pipeline_param_1: int): - producer_task = producer_op() - processor_task = processor_op(pipeline_param_1, - producer_task.outputs['Output 2']) - consumer_task = consumer_op(processor_task.outputs['Output 1'], - processor_task.outputs['Output 2']) - - return OrderedDict( - [ # You can safely return normal dict in python 3.6+ - ('Pipeline output 1', producer_task.outputs['Output 1']), - ('Pipeline output 2', processor_task.outputs['Output 2']), - ]) - - graph_component = create_graph_component_spec_from_pipeline_func( - pipeline1) - - self.assertEqual(len(graph_component.inputs), 1) - self.assertListEqual( - [input.name for input in graph_component.inputs], - ['pipeline_param_1' - ]) #Relies on human name conversion function stability - self.assertListEqual( - [output.name for output in graph_component.outputs], - ['Pipeline output 1', 'Pipeline output 2']) - self.assertEqual(len(graph_component.implementation.graph.tasks), 3) - - def test_create_component_from_real_pipeline_retail_product_stockout_prediction( - self): - from kfp.deprecated.components_tests.test_data.retail_product_stockout_prediction_pipeline import retail_product_stockout_prediction_pipeline - - graph_component = create_graph_component_spec_from_pipeline_func( - retail_product_stockout_prediction_pipeline) - - import yaml - expected_component_spec_path = str( - Path(__file__).parent / 'test_data' / - 'retail_product_stockout_prediction_pipeline.component.yaml') - with open(expected_component_spec_path) as f: - expected_dict = yaml.safe_load(f) - - self.assertEqual(expected_dict, graph_component.to_dict()) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/components_tests/test_structure_model_base.py b/sdk/python/kfp/deprecated/components_tests/test_structure_model_base.py deleted file mode 100644 index 54e418da84a..00000000000 --- a/sdk/python/kfp/deprecated/components_tests/test_structure_model_base.py +++ /dev/null @@ -1,303 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from typing import List, Dict, Union, Optional -from kfp.deprecated.components.modelbase import ModelBase - - -class TestModel1(ModelBase): - _serialized_names = { - 'prop_1': 'prop1', - 'prop_2': 'prop 2', - 'prop_3': '@@', - } - - def __init__( - self, - prop_0: str, - prop_1: Optional[str] = None, - prop_2: Union[int, str, bool] = '', - prop_3: 'TestModel1' = None, - prop_4: Optional[Dict[str, 'TestModel1']] = None, - prop_5: Optional[Union['TestModel1', List['TestModel1'], - Dict[str, 'TestModel1']]] = None, - prop_6: Optional[Union[str, List, Dict]] = None, - ): - #print(locals()) - super().__init__(locals()) - - -class StructureModelBaseTestCase(unittest.TestCase): - - def test_handle_type_check_for_simple_builtin(self): - self.assertEqual(TestModel1(prop_0='value 0').prop_0, 'value 0') - - with self.assertRaises(TypeError): - TestModel1(prop_0=1) - - with self.assertRaises(TypeError): - TestModel1(prop_0=None) - - with self.assertRaises(TypeError): - TestModel1(prop_0=TestModel1(prop_0='value 0')) - - def test_handle_type_check_for_optional_builtin(self): - self.assertEqual( - TestModel1(prop_0='', prop_1='value 1').prop_1, 'value 1') - self.assertEqual(TestModel1(prop_0='', prop_1=None).prop_1, None) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_1=1) - - with self.assertRaises(TypeError): - TestModel1( - prop_0='', prop_1=TestModel1(prop_0='', prop_1='value 1')) - - def test_handle_type_check_for_union_builtin(self): - self.assertEqual( - TestModel1(prop_0='', prop_2='value 2').prop_2, 'value 2') - self.assertEqual(TestModel1(prop_0='', prop_2=22).prop_2, 22) - self.assertEqual(TestModel1(prop_0='', prop_2=True).prop_2, True) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_2=None) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_2=22.22) - - with self.assertRaises(TypeError): - TestModel1( - prop_0='', prop_2=TestModel1(prop_0='', prop_2='value 2')) - - def test_handle_type_check_for_class(self): - val3 = TestModel1(prop_0='value 0') - self.assertEqual(TestModel1(prop_0='', prop_3=val3).prop_3, val3) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_3=1) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_3='value 3') - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_3=[val3]) - - def test_handle_type_check_for_dict_class(self): - val4 = TestModel1(prop_0='value 0') - self.assertEqual( - TestModel1(prop_0='', prop_4={ - 'key 4': val4 - }).prop_4['key 4'], val4) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_4=1) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_4='value 4') - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_4=[val4]) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_4={42: val4}) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_4={'key 4': [val4]}) - - def test_handle_type_check_for_union_dict_class(self): - val5 = TestModel1(prop_0='value 0') - self.assertEqual(TestModel1(prop_0='', prop_5=val5).prop_5, val5) - self.assertEqual(TestModel1(prop_0='', prop_5=[val5]).prop_5[0], val5) - self.assertEqual( - TestModel1(prop_0='', prop_5={ - 'key 5': val5 - }).prop_5['key 5'], val5) - self.assertEqual(TestModel1(prop_0='', prop_5=None).prop_5, None) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_5=1) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_5='value 5') - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_5={'key 5': 'value 5'}) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_5={42: val5}) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_5={'key 5': [val5]}) - - def test_handle_type_check_for_open_generic_classes(self): - val6 = TestModel1(prop_0='value 0') - self.assertEqual(TestModel1(prop_0='', prop_6='val6').prop_6, 'val6') - self.assertEqual(TestModel1(prop_0='', prop_6=[val6]).prop_6[0], val6) - self.assertEqual( - TestModel1(prop_0='', prop_6={ - 'key 6': val6 - }).prop_6['key 6'], val6) - self.assertEqual(TestModel1(prop_0='', prop_6=None).prop_6, None) - - with self.assertRaises(TypeError): - TestModel1(prop_0='', prop_6=1) - - def test_handle_from_to_dict_for_simple_builtin(self): - struct0 = {'prop_0': 'value 0'} - obj0 = TestModel1.from_dict(struct0) - self.assertEqual(obj0.prop_0, 'value 0') - self.assertDictEqual(obj0.to_dict(), struct0) - - with self.assertRaises(AttributeError): #TypeError: - TestModel1.from_dict(None) - - with self.assertRaises(AttributeError): #TypeError: - TestModel1.from_dict('') - - with self.assertRaises(TypeError): - TestModel1.from_dict({}) - - with self.assertRaises(TypeError): - TestModel1.from_dict({'prop0': 'value 0'}) - - def test_handle_from_to_dict_for_optional_builtin(self): - struct11 = {'prop_0': '', 'prop1': 'value 1'} - obj11 = TestModel1.from_dict(struct11) - self.assertEqual(obj11.prop_1, struct11['prop1']) - self.assertDictEqual(obj11.to_dict(), struct11) - - struct12 = {'prop_0': '', 'prop1': None} - obj12 = TestModel1.from_dict(struct12) - self.assertEqual(obj12.prop_1, None) - self.assertDictEqual(obj12.to_dict(), {'prop_0': ''}) - - with self.assertRaises(TypeError): - TestModel1.from_dict({'prop_0': '', 'prop 1': ''}) - - with self.assertRaises(TypeError): - TestModel1.from_dict({'prop_0': '', 'prop1': 1}) - - def test_handle_from_to_dict_for_union_builtin(self): - struct21 = {'prop_0': '', 'prop 2': 'value 2'} - obj21 = TestModel1.from_dict(struct21) - self.assertEqual(obj21.prop_2, struct21['prop 2']) - self.assertDictEqual(obj21.to_dict(), struct21) - - struct22 = {'prop_0': '', 'prop 2': 22} - obj22 = TestModel1.from_dict(struct22) - self.assertEqual(obj22.prop_2, struct22['prop 2']) - self.assertDictEqual(obj22.to_dict(), struct22) - - struct23 = {'prop_0': '', 'prop 2': True} - obj23 = TestModel1.from_dict(struct23) - self.assertEqual(obj23.prop_2, struct23['prop 2']) - self.assertDictEqual(obj23.to_dict(), struct23) - - with self.assertRaises(TypeError): - TestModel1.from_dict({'prop_0': 'ZZZ', 'prop 2': None}) - - with self.assertRaises(TypeError): - TestModel1.from_dict({'prop_0': '', 'prop 2': 22.22}) - - def test_handle_from_to_dict_for_class(self): - val3 = TestModel1(prop_0='value 0') - - struct31 = { - 'prop_0': '', - '@@': val3.to_dict() - } #{'prop_0': '', '@@': TestModel1(prop_0='value 0')} is also valid for from_dict, but this cannot happen when parsing for real - obj31 = TestModel1.from_dict(struct31) - self.assertEqual(obj31.prop_3, val3) - self.assertDictEqual(obj31.to_dict(), struct31) - - with self.assertRaises(TypeError): - TestModel1.from_dict({'prop_0': '', '@@': 'value 3'}) - - def test_handle_from_to_dict_for_dict_class(self): - val4 = TestModel1(prop_0='value 0') - - struct41 = {'prop_0': '', 'prop_4': {'val 4': val4.to_dict()}} - obj41 = TestModel1.from_dict(struct41) - self.assertEqual(obj41.prop_4['val 4'], val4) - self.assertDictEqual(obj41.to_dict(), struct41) - - with self.assertRaises(TypeError): - TestModel1.from_dict({'prop_0': '', 'prop_4': {44: val4.to_dict()}}) - - def test_handle_from_to_dict_for_union_dict_class(self): - val5 = TestModel1(prop_0='value 0') - - struct51 = {'prop_0': '', 'prop_5': val5.to_dict()} - obj51 = TestModel1.from_dict(struct51) - self.assertEqual(obj51.prop_5, val5) - self.assertDictEqual(obj51.to_dict(), struct51) - - struct52 = {'prop_0': '', 'prop_5': [val5.to_dict()]} - obj52 = TestModel1.from_dict(struct52) - self.assertListEqual(obj52.prop_5, [val5]) - self.assertDictEqual(obj52.to_dict(), struct52) - - with self.assertRaises(TypeError): - TestModel1.from_dict({'prop_0': '', 'prop_5': {44: val5.to_dict()}}) - - with self.assertRaises(TypeError): - TestModel1.from_dict({ - 'prop_0': '', - 'prop_5': [val5.to_dict(), None] - }) - - def test_handle_from_to_dict_for_open_generic_class(self): - value61 = "value 6 1" - struct61 = {'prop_0': '', 'prop_6': value61} - obj61 = TestModel1.from_dict(struct61) - self.assertEqual(obj61.prop_6, value61) - self.assertEqual(obj61.to_dict(), struct61) - - value62 = ["value 6 2"] - struct62 = {'prop_0': '', 'prop_6': value62} - obj62 = TestModel1.from_dict(struct62) - self.assertEqual(obj62.prop_6, value62) - self.assertEqual(obj62.to_dict(), struct62) - - value63 = {"key 6 3": "value 6 3"} - struct63 = {'prop_0': '', 'prop_6': value63} - obj63 = TestModel1.from_dict(struct63) - self.assertEqual(obj63.prop_6, value63) - self.assertEqual(obj63.to_dict(), struct63) - - with self.assertRaises(TypeError): - TestModel1.from_dict({'prop_0': '', 'prop_6': 64}) - - def test_handle_comparisons(self): - - class A(ModelBase): - - def __init__(self, a, b): - super().__init__(locals()) - - self.assertEqual(A(1, 2), A(1, 2)) - self.assertNotEqual(A(1, 2), A(1, 3)) - - class B(ModelBase): - - def __init__(self, a, b): - super().__init__(locals()) - - self.assertNotEqual(A(1, 2), B(1, 2)) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/containers/__init__.py b/sdk/python/kfp/deprecated/containers/__init__.py deleted file mode 100644 index 70c8c657f2b..00000000000 --- a/sdk/python/kfp/deprecated/containers/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the speci - -from ._build_image_api import * -from ._container_builder import * diff --git a/sdk/python/kfp/deprecated/containers/_build_image_api.py b/sdk/python/kfp/deprecated/containers/_build_image_api.py deleted file mode 100644 index 0ae1f0233ee..00000000000 --- a/sdk/python/kfp/deprecated/containers/_build_image_api.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the speci - -__all__ = [ - 'build_image_from_working_dir', - 'default_image_builder', -] - -import logging -import os -import re -import shutil -import tempfile - -from ._cache import calculate_recursive_dir_hash, try_read_value_from_cache, write_value_to_cache -from ._container_builder import ContainerBuilder - -default_base_image = 'gcr.io/deeplearning-platform-release/tf-cpu.1-14' - -_container_work_dir = '/python_env' - -default_image_builder = ContainerBuilder() - - -def _generate_dockerfile_text(context_dir: str, - dockerfile_path: str, - base_image: str = None) -> str: - # Generating the Dockerfile - logging.info('Generating the Dockerfile') - - requirements_rel_path = 'requirements.txt' - requirements_path = os.path.join(context_dir, requirements_rel_path) - requirements_file_exists = os.path.exists(requirements_path) - - if not base_image: - base_image = default_base_image - if callable(base_image): - base_image = base_image() - - dockerfile_lines = [] - dockerfile_lines.append('FROM {}'.format(base_image)) - dockerfile_lines.append('WORKDIR {}'.format(_container_work_dir)) - if requirements_file_exists: - dockerfile_lines.append('COPY {} .'.format(requirements_rel_path)) - dockerfile_lines.append( - 'RUN python3 -m pip install -r {}'.format(requirements_rel_path)) - dockerfile_lines.append('COPY . .') - - return '\n'.join(dockerfile_lines) - - -def build_image_from_working_dir(image_name: str = None, - working_dir: str = None, - file_filter_re: str = r'.*\.py', - timeout: int = 1000, - base_image: str = None, - builder: ContainerBuilder = None) -> str: - """Builds and pushes a new container image that captures the current python - working directory. - - This function recursively scans the working directory and captures the following files in the container image context: - - * :code:`requirements.txt` files - * All python files (can be overridden by passing a different `file_filter_re` argument) - - The function generates Dockerfile that starts from a python container image, install packages from requirements.txt (if present) and copies all the captured python files to the container image. - The Dockerfile can be overridden by placing a custom Dockerfile in the root of the working directory. - - Args: - image_name: Optional. The image repo name where the new container image will be pushed. The name will be generated if not not set. - working_dir: Optional. The directory that will be captured. The current directory will be used if omitted. - file_filter_re: Optional. A regular expression that will be used to decide which files to include in the container building context. - timeout: Optional. The image building timeout in seconds. - base_image: Optional. The container image to use as the base for the new image. If not set, the Google Deep Learning Tensorflow CPU image will be used. - builder: Optional. An instance of :py:class:`kfp.containers.ContainerBuilder` or compatible class that will be used to build the image. - The default builder uses "kubeflow-pipelines-container-builder" service account in "kubeflow" namespace. It works with Kubeflow Pipelines clusters installed in "kubeflow" namespace using Google Cloud Marketplace or Standalone with version > 0.4.0. - If your Kubeflow Pipelines is installed in a different namespace, you should use :code:`ContainerBuilder(namespace='', ...)`. - - Depending on how you installed Kubeflow Pipelines, you need to configure your :code:`ContainerBuilder` instance's namespace and service_account: - - * For clusters installed with Kubeflow >= 0.7, use :code:`ContainerBuilder(namespace='', service_account='default-editor', ...)`. You can omit the namespace if you use kfp sdk from in-cluster notebook, it uses notebook namespace by default. - * For clusters installed with Kubeflow < 0.7, use :code:`ContainerBuilder(service_account='default', ...)`. - * For clusters installed using Google Cloud Marketplace or Standalone with version <= 0.4.0, use :code:`ContainerBuilder(namespace='' service_account='default')` - You may refer to `installation guide `_ for more details about different installation options. - - Returns: - The full name of the container image including the hash digest. E.g. :code:`gcr.io/my-org/my-image@sha256:86c1...793c`. - """ - current_dir = working_dir or os.getcwd() - with tempfile.TemporaryDirectory() as context_dir: - logging.info( - 'Creating the build context directory: {}'.format(context_dir)) - - # Copying all *.py and requirements.txt files - for dirpath, dirnames, filenames in os.walk(current_dir): - dst_dirpath = os.path.join(context_dir, - os.path.relpath(dirpath, current_dir)) - os.makedirs(dst_dirpath, exist_ok=True) - for file_name in filenames: - if re.match(file_filter_re, - file_name) or file_name == 'requirements.txt': - src_path = os.path.join(dirpath, file_name) - dst_path = os.path.join(dst_dirpath, file_name) - shutil.copy(src_path, dst_path) - - src_dockerfile_path = os.path.join(current_dir, 'Dockerfile') - dst_dockerfile_path = os.path.join(context_dir, 'Dockerfile') - if os.path.exists(src_dockerfile_path): - if base_image: - raise ValueError( - 'Cannot specify base_image when using custom Dockerfile (which already specifies the base image).' - ) - shutil.copy(src_dockerfile_path, dst_dockerfile_path) - else: - dockerfile_text = _generate_dockerfile_text(context_dir, - dst_dockerfile_path, - base_image) - with open(dst_dockerfile_path, 'w') as f: - f.write(dockerfile_text) - - cache_name = 'build_image_from_working_dir' - cache_key = calculate_recursive_dir_hash(context_dir) - cached_image_name = try_read_value_from_cache(cache_name, cache_key) - if cached_image_name: - return cached_image_name - - if builder is None: - builder = default_image_builder - image_name = builder.build( - local_dir=context_dir, - target_image=image_name, - timeout=timeout, - ) - if image_name: - write_value_to_cache(cache_name, cache_key, image_name) - return image_name diff --git a/sdk/python/kfp/deprecated/containers/_cache.py b/sdk/python/kfp/deprecated/containers/_cache.py deleted file mode 100644 index 59cca67936e..00000000000 --- a/sdk/python/kfp/deprecated/containers/_cache.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the speci - -import hashlib -import os -import shutil -import tempfile -from pathlib import Path - - -def calculate_file_hash(file_path: str): - block_size = 64 * 1024 - sha = hashlib.sha256() - with open(file_path, 'rb') as fp: - while True: - data = fp.read(block_size) - if not data: - break - sha.update(data) - return sha.hexdigest() - - -def calculate_recursive_dir_hash(root_dir_path: str): - path_hashes = {} - for dirpath, dirnames, filenames in os.walk(root_dir_path): - for file_name in filenames: - file_path = os.path.join(dirpath, file_name) - rel_file_path = os.path.relpath(file_path, root_dir_path) - file_hash = calculate_file_hash(file_path) - path_hashes[rel_file_path] = file_hash - binary_path_hash_lines = sorted( - path.encode('utf-8') + b'\t' + path_hash.encode('utf-8') + b'\n' - for path, path_hash in path_hashes.items()) - binary_path_hash_doc = b''.join(binary_path_hash_lines) - - full_hash = hashlib.sha256(binary_path_hash_doc).hexdigest() - return full_hash - - -def try_read_value_from_cache(cache_type: str, key: str) -> str: - cache_file_path = Path(tempfile.tempdir) / cache_type / key - if cache_file_path.exists(): - return cache_file_path.read_text() - return None - - -def write_value_to_cache(cache_type: str, key: str, value: str): - cache_file_path = Path(tempfile.tempdir) / cache_type / key - if cache_file_path.exists(): - old_value = cache_file_path.read_text() - if value != old_value: - import warnings - warnings.warn( - 'Overwriting existing cache entry "{}" with value "{}" != "{}".' - .format(key, value, old_value)) - cache_file_path.parent.mkdir(parents=True, exist_ok=True) - cache_file_path.write_text(value) - - -def clear_cache(cache_type: str): - cache_file_path = Path(tempfile.tempdir) / cache_type - if cache_file_path.exists(): - shutil.rmtree(cache_file_path) diff --git a/sdk/python/kfp/deprecated/containers/_component_builder.py b/sdk/python/kfp/deprecated/containers/_component_builder.py deleted file mode 100644 index 08e16ec2bd7..00000000000 --- a/sdk/python/kfp/deprecated/containers/_component_builder.py +++ /dev/null @@ -1,429 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys -import tempfile -import logging -import shutil -from collections import OrderedDict -from typing import Callable, Dict, List, Optional - -from deprecated.sphinx import deprecated - -from ..components._components import _create_task_factory_from_component_spec -from ..components._python_op import _func_to_component_spec -from ._container_builder import ContainerBuilder -from kfp.deprecated.components import _structures -from kfp.deprecated.containers import entrypoint - -V2_COMPONENT_ANNOTATION = 'pipelines.kubeflow.org/component_v2' -_PROGRAM_LAUNCHER_CMD = 'program_path=$(mktemp)\nprintf "%s" "$0" > ' \ - '"$program_path"\npython3 -u "$program_path" "$@"\n' - - -class VersionedDependency(object): - """DependencyVersion specifies the versions.""" - - def __init__(self, name, version=None, min_version=None, max_version=None): - """if version is specified, no need for min_version or max_version; if - both are specified, version is adopted.""" - self._name = name - if version is not None: - self._min_version = version - self._max_version = version - else: - self._min_version = min_version - self._max_version = max_version - - @property - def name(self): - return self._name - - @property - def min_version(self): - return self._min_version - - @min_version.setter - def min_version(self, min_version): - self._min_version = min_version - - def has_min_version(self): - return self._min_version != None - - @property - def max_version(self): - return self._max_version - - @max_version.setter - def max_version(self, max_version): - self._max_version = max_version - - def has_max_version(self): - return self._max_version != None - - def has_versions(self): - return (self.has_min_version()) or (self.has_max_version()) - - -class DependencyHelper(object): - """DependencyHelper manages software dependency information.""" - - def __init__(self): - self._PYTHON_PACKAGE = 'PYTHON_PACKAGE' - self._dependency = {self._PYTHON_PACKAGE: OrderedDict()} - - @property - def python_packages(self): - return self._dependency[self._PYTHON_PACKAGE] - - def add_python_package(self, dependency, override=True): - """add_single_python_package adds a dependency for the python package. - - Args: - name: package name - version: it could be a specific version(1.10.0), or a range(>=1.0,<=2.0) - if not specified, the default is resolved automatically by the pip system. - override: whether to override the version if already existing in the dependency. - """ - if dependency.name in self.python_packages and not override: - return - self.python_packages[dependency.name] = dependency - - def generate_pip_requirements(self, target_file): - """write the python packages to a requirement file the generated file - follows the order of which the packages are added.""" - with open(target_file, 'w') as f: - for name, version in self.python_packages.items(): - version = self.python_packages[name] - version_str = '' - if version.has_min_version(): - version_str += ' >= ' + version.min_version + ',' - if version.has_max_version(): - version_str += ' <= ' + version.max_version + ',' - f.write(name + version_str.rstrip(',') + '\n') - - -def _dependency_to_requirements(dependency=[], filename='requirements.txt'): - """ - Generates a requirement file based on the dependency - Args: - dependency (list): a list of VersionedDependency, which includes the package name and versions - filename (str): requirement file name, default as requirements.txt - """ - dependency_helper = DependencyHelper() - for version in dependency: - dependency_helper.add_python_package(version) - dependency_helper.generate_pip_requirements(filename) - - -def _generate_dockerfile(filename: str, - base_image: str, - requirement_filename: Optional[str] = None, - add_files: Optional[Dict[str, str]] = None): - """ - generates dockerfiles - Args: - filename (str): target file name for the dockerfile. - base_image (str): the base image name. - requirement_filename (str): requirement file name - add_files (Dict[str, str]): Map containing the files thats should be added to the container. add_files maps the build context relative source paths to the container destination paths. - """ - with open(filename, 'w') as f: - f.write('FROM ' + base_image + '\n') - f.write( - 'RUN apt-get update -y && apt-get install --no-install-recommends -y -q python3 python3-pip python3-setuptools\n' - ) - if requirement_filename is not None: - f.write('ADD ' + requirement_filename + ' /ml/requirements.txt\n') - f.write('RUN python3 -m pip install -r /ml/requirements.txt\n') - - for src_path, dst_path in (add_files or {}).items(): - f.write('ADD ' + src_path + ' ' + dst_path + '\n') - - -def _configure_logger(logger): - """_configure_logger configures the logger such that the info level logs go - to the stdout and the error(or above) level logs go to the stderr. - - It is important for the Jupyter notebook log rendering - """ - if hasattr(_configure_logger, 'configured'): - # Skip the logger configuration the second time this function - # is called to avoid multiple streamhandlers bound to the logger. - return - setattr(_configure_logger, 'configured', 'true') - logger.setLevel(logging.INFO) - info_handler = logging.StreamHandler(stream=sys.stdout) - info_handler.addFilter(lambda record: record.levelno <= logging.INFO) - info_handler.setFormatter( - logging.Formatter( - '%(asctime)s:%(levelname)s:%(message)s', - datefmt='%Y-%m-%d %H:%M:%S')) - error_handler = logging.StreamHandler(sys.stderr) - error_handler.addFilter(lambda record: record.levelno > logging.INFO) - error_handler.setFormatter( - logging.Formatter( - '%(asctime)s:%(levelname)s:%(message)s', - datefmt='%Y-%m-%d %H:%M:%S')) - logger.addHandler(info_handler) - logger.addHandler(error_handler) - - -def _purge_program_launching_code( - commands: List[str], - entrypoint_container_path: Optional[str] = None, - is_v2: bool = False) -> str: - """Replaces the inline Python code with calling a local program. - - For example, - Before: sh -ec '... && python3 -u ...' 'import sys ...' --param1 ... - After: python -u /ml/main.py --param1 ... - - Args: - commands: The container commands to be replaced. - entrypoint_container_path: The path to the entrypoint program in the - container. - is_v2: Whether the component being generated is a v2 component. Default is - False. - - Returns: - The originally generated inline Python code. - """ - if not (is_v2 or entrypoint_container_path): - raise ValueError( - 'Only v2 component has default entrypoint path. ' - 'Conventional KFP component needs to specify container ' - 'entrypoint explicitly. For example, /ml/main.py') - program_launcher_index = commands.index(_PROGRAM_LAUNCHER_CMD) - # When there're preinstallation package specified when converting to component - # spec the index will be 3, otherwise it'll be 2. - assert program_launcher_index in [2, 3] - program_code_index = program_launcher_index + 1 - result = commands[program_code_index] - if is_v2: - # TODO: Implement the v2 component entrypoint on KFP. - # The following are just placeholders. - commands[program_code_index] = 'kfp.containers.entrypoint' - commands.pop(program_launcher_index) - commands[program_launcher_index - 1] = '-m' - commands[program_launcher_index - 2] = 'python' - else: - commands[program_code_index] = entrypoint_container_path - commands.pop(program_launcher_index) - commands[program_launcher_index - 1] = '-u' # -ec => -u - # sh => python3 or python2 - commands[program_launcher_index - 2] = 'python' - - return result - - -def build_python_component( - component_func: Callable, - target_image: str, - base_image: Optional[str] = None, - dependency: Optional[List[VersionedDependency]] = None, - staging_gcs_path: Optional[str] = None, - timeout: int = 600, - namespace: Optional[str] = None, - target_component_file: Optional[str] = None, - is_v2: bool = False): - """build_component automatically builds a container image for the - component_func based on the base_image and pushes to the target_image. - - Args: - component_func (python function): The python function to build components - upon. - base_image (str): Docker image to use as a base image. - target_image (str): Full URI to push the target image. - staging_gcs_path (str): GCS blob that can store temporary build files. - target_image (str): The target image path. - timeout (int): The timeout for the image build(in secs), default is 600 - seconds. - namespace (str): The namespace within which to run the kubernetes Kaniko - job. If the job is running on GKE and value is None the underlying - functions will use the default namespace from GKE. - dependency (list): The list of VersionedDependency, which includes the - package name and versions, default is empty. - target_component_file (str): The path to save the generated component YAML - spec. - is_v2: Whether or not generating a v2 KFP component, default - is false. - - Raises: - ValueError: The function is not decorated with python_component decorator or - the python_version is neither python2 nor python3 - """ - - _configure_logger(logging.getLogger()) - - if component_func is None: - raise ValueError('component_func must not be None') - if target_image is None: - raise ValueError('target_image must not be None') - - if staging_gcs_path is None: - raise ValueError('staging_gcs_path must not be None') - - if base_image is None: - base_image = getattr(component_func, '_component_base_image', None) - if base_image is None: - from ..components._python_op import default_base_image_or_builder - base_image = default_base_image_or_builder - if isinstance(base_image, Callable): - base_image = base_image() - if not dependency: - dependency = [] - - logging.info('Build an image that is based on ' + base_image + - ' and push the image to ' + target_image) - - component_spec = _func_to_component_spec( - component_func, base_image=base_image) - - if is_v2: - # TODO: Remove this warning once we make v2 component compatible with KFP - # v1 stack. - logging.warning( - 'Currently V2 component is only compatible with v2 KFP.') - # Annotate the component to be a V2 one. - if not component_spec.metadata: - component_spec.metadata = _structures.MetadataSpec() - if not component_spec.metadata.annotations: - component_spec.metadata.annotations = {} - component_spec.metadata.annotations[V2_COMPONENT_ANNOTATION] = 'true' - - command_line_args = component_spec.implementation.container.command - - # The relative path to put the Python program code. - program_path = 'ml/main.py' - # The relative path used when building a V2 component. - v2_entrypoint_path = None - # Python program code extracted from the component spec. - program_code = None - - if is_v2: - - program_code = _purge_program_launching_code( - commands=command_line_args, is_v2=True) - - # Override user program args for new-styled component. - # TODO: The actual program args will be changed after we support v2 - # component on KFP. - # For v2 component, the received command line args are fixed as follows: - # --executor_input_str - # {Executor input pb message at runtime} - # --function_name - # {The name of user defined function} - # --output_metadata_path - # {The place to write output metadata JSON file} - program_args = [ - '--executor_input_str', - _structures.ExecutorInputPlaceholder(), - '--{}'.format(entrypoint.FN_NAME_ARG), component_func.__name__, - '--output_metadata_path', - _structures.OutputMetadataPlaceholder() - ] - - component_spec.implementation.container.args = program_args - else: - program_code = _purge_program_launching_code( - commands=command_line_args, - entrypoint_container_path='/' + program_path) - - arc_docker_filename = 'Dockerfile' - arc_requirement_filename = 'requirements.txt' - - with tempfile.TemporaryDirectory() as local_build_dir: - # Write the program code to a file in the context directory - local_python_filepath = os.path.join(local_build_dir, program_path) - os.makedirs(os.path.dirname(local_python_filepath), exist_ok=True) - - with open(local_python_filepath, 'w') as f: - f.write(program_code) - - # Generate the python package requirements file in the context directory - local_requirement_filepath = os.path.join(local_build_dir, - arc_requirement_filename) - if is_v2: - # For v2 components, KFP are expected to be packed in the container. - dependency.append( - VersionedDependency(name='kfp', min_version='1.4.0')) - - _dependency_to_requirements(dependency, local_requirement_filepath) - - # Generate Dockerfile in the context directory - local_docker_filepath = os.path.join(local_build_dir, - arc_docker_filename) - add_files = {program_path: '/' + program_path} - - _generate_dockerfile( - local_docker_filepath, - base_image, - arc_requirement_filename, - add_files=add_files) - - logging.info('Building and pushing container image.') - container_builder = ContainerBuilder(staging_gcs_path, target_image, - namespace) - image_name_with_digest = container_builder.build( - local_build_dir, arc_docker_filename, target_image, timeout) - - component_spec.implementation.container.image = image_name_with_digest - - # Optionally writing the component definition to a local file for sharing - target_component_file = target_component_file or getattr( - component_func, '_component_target_component_file', None) - if target_component_file: - component_spec.save(target_component_file) - - task_factory_function = _create_task_factory_from_component_spec( - component_spec) - return task_factory_function - - -@deprecated( - version='0.1.32', - reason='`build_docker_image` is deprecated. Use `kfp.containers.build_image_from_working_dir` instead.' -) -def build_docker_image(staging_gcs_path, - target_image, - dockerfile_path, - timeout=600, - namespace=None): - """build_docker_image automatically builds a container image based on the - specification in the dockerfile and pushes to the target_image. - - Args: - staging_gcs_path (str): GCS blob that can store temporary build files - target_image (str): gcr path to push the final image - dockerfile_path (str): local path to the dockerfile - timeout (int): the timeout for the image build(in secs), default is 600 seconds - namespace (str): the namespace within which to run the kubernetes kaniko job. Default is None. If the - job is running on GKE and value is None the underlying functions will use the default namespace from GKE. - """ - _configure_logger(logging.getLogger()) - - with tempfile.TemporaryDirectory() as local_build_dir: - dockerfile_rel_path = 'Dockerfile' - dst_dockerfile_path = os.path.join(local_build_dir, dockerfile_rel_path) - shutil.copyfile(dockerfile_path, dst_dockerfile_path) - - container_builder = ContainerBuilder( - staging_gcs_path, target_image, namespace=namespace) - image_name_with_digest = container_builder.build( - local_build_dir, dockerfile_rel_path, target_image, timeout) - - logging.info('Build image complete.') - return image_name_with_digest diff --git a/sdk/python/kfp/deprecated/containers/_container_builder.py b/sdk/python/kfp/deprecated/containers/_container_builder.py deleted file mode 100644 index aa36ffd95bd..00000000000 --- a/sdk/python/kfp/deprecated/containers/_container_builder.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__all__ = [ - 'ContainerBuilder', -] - -import logging -import tarfile -import tempfile -import os -import uuid - -SERVICEACCOUNT_NAMESPACE = '/var/run/secrets/kubernetes.io/serviceaccount/namespace' -GCS_STAGING_BLOB_DEFAULT_PREFIX = 'kfp_container_build_staging' -GCR_DEFAULT_IMAGE_SUFFIX = 'kfp_container' -KANIKO_EXECUTOR_IMAGE_DEFAULT = 'gcr.io/kaniko-project/executor@sha256:78d44ec4e9cb5545d7f85c1924695c89503ded86a59f92c7ae658afa3cff5400' - - -def _get_project_id(): - import requests - URL = "http://metadata.google.internal/computeMetadata/v1/project/project-id" - headers = {'Metadata-Flavor': 'Google'} - r = requests.get(url=URL, headers=headers) - if not r.ok: - raise RuntimeError( - 'ContainerBuilder failed to retrieve the project id.') - return r.text - - -def _get_instance_id(): - import requests - URL = "http://metadata.google.internal/computeMetadata/v1/instance/id" - headers = {'Metadata-Flavor': 'Google'} - r = requests.get(url=URL, headers=headers) - if not r.ok: - raise RuntimeError( - 'ContainerBuilder failed to retrieve the instance id.') - return r.text - - -class ContainerBuilder(object): - """ContainerBuilder helps build a container image.""" - - def __init__(self, - gcs_staging=None, - default_image_name=None, - namespace=None, - service_account='kubeflow-pipelines-container-builder', - kaniko_executor_image=KANIKO_EXECUTOR_IMAGE_DEFAULT, - k8s_client_configuration=None): - """ - Args: - gcs_staging (str): GCS bucket/blob that can store temporary build files, - default is gs://PROJECT_ID/kfp_container_build_staging. You have to - specify this when it doesn't run in cluster. - default_image_name (str): Target container image name that will be used by the build method if the target_image argument is not specified. - namespace (str): Kubernetes namespace where the container builder pod is launched, - default is the same namespace as the notebook service account in cluster - or 'kubeflow' if not in cluster. If using the full Kubeflow - deployment and not in cluster, you should specify your own user namespace. - service_account (str): Kubernetes service account the pod uses for container building, - The default value is "kubeflow-pipelines-container-builder". It works with Kubeflow Pipelines clusters installed using Google Cloud Marketplace or Standalone with version > 0.4.0. - The service account should have permission to read and write from staging gcs path and upload built images to gcr.io. - kaniko_executor_image (str): Docker image used to run kaniko executor. Defaults to gcr.io/kaniko-project/executor:v0.10.0. - k8s_client_configuration (kubernetes.Configuration): Kubernetes client configuration object to be used when talking with Kubernetes API. - This is optional. If not specified, it will use the default configuration. This can be used to personalize the client used to talk to the Kubernetes server and change authentication parameters. - """ - self._gcs_staging = gcs_staging - self._gcs_staging_checked = False - self._default_image_name = default_image_name - self._namespace = namespace - self._service_account = service_account - self._kaniko_image = kaniko_executor_image - self._k8s_client_configuration = k8s_client_configuration - - def _get_namespace(self): - if self._namespace is None: - # Configure the namespace - if os.path.exists(SERVICEACCOUNT_NAMESPACE): - with open(SERVICEACCOUNT_NAMESPACE, 'r') as f: - self._namespace = f.read() - else: - self._namespace = 'kubeflow' - return self._namespace - - def _get_staging_location(self): - if self._gcs_staging_checked: - return self._gcs_staging - - # Configure the GCS staging bucket - if self._gcs_staging is None: - try: - gcs_bucket = _get_project_id() - except: - raise ValueError( - 'Cannot get the Google Cloud project ID, please specify the gcs_staging argument.' - ) - self._gcs_staging = 'gs://' + gcs_bucket + '/' + GCS_STAGING_BLOB_DEFAULT_PREFIX - else: - from pathlib import PurePath - path = PurePath(self._gcs_staging).parts - if len(path) < 2 or not path[0].startswith('gs'): - raise ValueError('Error: {} should be a GCS path.'.format( - self._gcs_staging)) - gcs_bucket = path[1] - from ._gcs_helper import GCSHelper - GCSHelper.create_gcs_bucket_if_not_exist(gcs_bucket) - self._gcs_staging_checked = True - return self._gcs_staging - - def _get_default_image_name(self): - if self._default_image_name is None: - # KubeFlow Jupyter notebooks have environment variable with the notebook ID - try: - nb_id = os.environ.get('NB_PREFIX', _get_instance_id()) - except: - raise ValueError('Please provide the default_image_name.') - nb_id = nb_id.replace('/', '-').strip('-') - self._default_image_name = os.path.join('gcr.io', _get_project_id(), - nb_id, - GCR_DEFAULT_IMAGE_SUFFIX) - return self._default_image_name - - def _generate_kaniko_spec(self, context, docker_filename, target_image): - """_generate_kaniko_yaml generates kaniko job yaml based on a template - yaml.""" - content = { - 'apiVersion': 'v1', - 'metadata': { - 'generateName': 'kaniko-', - 'namespace': self._get_namespace(), - 'annotations': { - 'sidecar.istio.io/inject': 'false' - }, - }, - 'kind': 'Pod', - 'spec': { - 'restartPolicy': 'Never', - 'containers': [{ - 'name': 'kaniko', - 'args': [ - '--cache=true', - '--dockerfile=' + docker_filename, - '--context=' + context, - '--destination=' + target_image, - '--digest-file=/dev/termination-log', # This is suggested by the Kaniko devs as a way to return the image digest from Kaniko Pod. See https://github.com/GoogleContainerTools/kaniko#--digest-file - ], - 'image': self._kaniko_image, - }], - 'serviceAccountName': self._service_account - } - } - return content - - def _wrap_dir_in_tarball(self, tarball_path, dir_name): - """_wrap_files_in_tarball creates a tarball for all the files in the - directory.""" - if not tarball_path.endswith('.tar.gz'): - raise ValueError('the tarball path should end with .tar.gz') - with tarfile.open(tarball_path, 'w:gz') as tarball: - tarball.add(dir_name, arcname='') - - def build(self, - local_dir, - docker_filename: str = 'Dockerfile', - target_image=None, - timeout=1000): - """ - Args: - local_dir (str): local directory that stores all the necessary build files - docker_filename (str): the path of the Dockerfile relative to the local_dir - target_image (str): The container image name where the data will be pushed. Can include tag. If not specified, the function will use the default_image_name specified when creating ContainerBuilder. - timeout (int): time out in seconds. Default: 1000 - """ - target_image = target_image or self._get_default_image_name() - # Prepare build context - with tempfile.TemporaryDirectory() as local_build_dir: - from ._gcs_helper import GCSHelper - logging.info('Generate build files.') - local_tarball_path = os.path.join(local_build_dir, - 'docker.tmp.tar.gz') - self._wrap_dir_in_tarball(local_tarball_path, local_dir) - # Upload to the context - context = os.path.join(self._get_staging_location(), - str(uuid.uuid4()) + '.tar.gz') - GCSHelper.upload_gcs_file(local_tarball_path, context) - - # Run kaniko job - kaniko_spec = self._generate_kaniko_spec( - context=context, - docker_filename=docker_filename, - target_image=target_image) - logging.info('Start a kaniko job for build.') - from ._k8s_job_helper import K8sJobHelper - k8s_helper = K8sJobHelper(self._k8s_client_configuration) - result_pod_obj = k8s_helper.run_job(kaniko_spec, timeout) - logging.info('Kaniko job complete.') - - # Clean up - GCSHelper.remove_gcs_blob(context) - - # Returning image name with digest - (image_repo, _, image_tag) = target_image.partition(':') - # When Kaniko build completes successfully, the termination message is the hash digest of the newly built image. Otherwise it's empty. See https://github.com/GoogleContainerTools/kaniko#--digest-file https://kubernetes.io/docs/tasks/debug-application-cluster/determine-reason-pod-failure/#customizing-the-termination-message - termination_message = [ - status.state.terminated.message - for status in result_pod_obj.status.container_statuses - if status.name == 'kaniko' - ][0] # Note: Using status.state instead of status.last_state since last_state entries can still be None - image_digest = termination_message - if not image_digest.startswith('sha256:'): - raise RuntimeError( - "Kaniko returned invalid image digest: {}".format( - image_digest)) - strict_image_name = image_repo + '@' + image_digest - logging.info( - 'Built and pushed image: {}.'.format(strict_image_name)) - return strict_image_name diff --git a/sdk/python/kfp/deprecated/containers/_gcs_helper.py b/sdk/python/kfp/deprecated/containers/_gcs_helper.py deleted file mode 100644 index a49734abf82..00000000000 --- a/sdk/python/kfp/deprecated/containers/_gcs_helper.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pathlib -import tempfile - - -class GCSHelper(object): - """GCSHelper manages the connection with the GCS storage.""" - - @staticmethod - def get_blob_from_gcs_uri(gcs_path): - """ - Args: - gcs_path (str) : gcs blob path - Returns: - gcs_blob: gcs blob object(https://github.com/googleapis/google-cloud-python/blob/5c9bb42cb3c9250131cfeef6e0bafe8f4b7c139f/storage/google/cloud/storage/blob.py#L105) - """ - from google.cloud import storage - pure_path = pathlib.PurePath(gcs_path) - gcs_bucket = pure_path.parts[1] - gcs_blob = '/'.join(pure_path.parts[2:]) - client = storage.Client() - bucket = client.get_bucket(gcs_bucket) - blob = bucket.blob(gcs_blob) - return blob - - @staticmethod - def upload_gcs_file(local_path, gcs_path): - """ - Args: - local_path (str): local file path - gcs_path (str) : gcs blob path - """ - blob = GCSHelper.get_blob_from_gcs_uri(gcs_path) - blob.upload_from_filename(local_path) - - @staticmethod - def write_to_gcs_path(path: str, content: str) -> None: - """Writes serialized content to a GCS location. - - Args: - path: GCS path to write to. - content: The content to be written. - """ - fd, temp_path = tempfile.mkstemp() - try: - with os.fdopen(fd, 'w') as tmp: - tmp.write(content) - - if not GCSHelper.get_blob_from_gcs_uri(path): - pure_path = pathlib.PurePath(path) - gcs_bucket = pure_path.parts[1] - GCSHelper.create_gcs_bucket_if_not_exist(gcs_bucket) - - GCSHelper.upload_gcs_file(temp_path, path) - finally: - os.remove(temp_path) - - @staticmethod - def remove_gcs_blob(gcs_path): - """ - Args: - gcs_path (str) : gcs blob path - """ - blob = GCSHelper.get_blob_from_gcs_uri(gcs_path) - blob.delete() - - @staticmethod - def download_gcs_blob(local_path, gcs_path): - """ - Args: - local_path (str): local file path - gcs_path (str) : gcs blob path - """ - blob = GCSHelper.get_blob_from_gcs_uri(gcs_path) - blob.download_to_filename(local_path) - - @staticmethod - def read_from_gcs_path(gcs_path: str) -> str: - """Reads the content of a file hosted on GCS.""" - fd, temp_path = tempfile.mkstemp() - try: - GCSHelper.download_gcs_blob(temp_path, gcs_path) - with os.fdopen(fd, 'r') as tmp: - result = tmp.read() - finally: - os.remove(temp_path) - return result - - @staticmethod - def create_gcs_bucket_if_not_exist(gcs_bucket): - """ - Args: - gcs_bucket (str) : gcs bucket name - """ - from google.cloud import storage - from google.cloud.exceptions import NotFound - client = storage.Client() - try: - client.get_bucket(gcs_bucket) - except NotFound: - client.create_bucket(gcs_bucket) diff --git a/sdk/python/kfp/deprecated/containers/_k8s_job_helper.py b/sdk/python/kfp/deprecated/containers/_k8s_job_helper.py deleted file mode 100644 index eb66de7126d..00000000000 --- a/sdk/python/kfp/deprecated/containers/_k8s_job_helper.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from datetime import datetime -from kubernetes import client as k8s_client -from kubernetes import config -import time -import logging -import os - - -class K8sJobHelper(object): - """Kubernetes Helper.""" - - def __init__(self, k8s_client_configuration=None): - if not self._configure_k8s(k8s_client_configuration): - raise Exception('K8sHelper __init__ failure') - - def _configure_k8s(self, k8s_client_configuration=None): - k8s_config_file = os.environ.get('KUBECONFIG') - if k8s_config_file: - try: - logging.info('Loading kubernetes config from the file %s', - k8s_config_file) - config.load_kube_config( - config_file=k8s_config_file, - client_configuration=k8s_client_configuration) - except Exception as e: - raise RuntimeError( - 'Can not load kube config from the file %s, error: %s', - k8s_config_file, e) - else: - try: - config.load_incluster_config() - logging.info('Initialized with in-cluster config.') - except: - logging.info( - 'Cannot find in-cluster config, trying the local kubernetes config. ' - ) - try: - config.load_kube_config( - client_configuration=k8s_client_configuration) - logging.info( - 'Found local kubernetes config. Initialized with kube_config.' - ) - except: - raise RuntimeError( - 'Forgot to run the gcloud command? Check out the link: \ - https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl for more information' - ) - self._api_client = k8s_client.ApiClient() - self._corev1 = k8s_client.CoreV1Api(self._api_client) - return True - - def _create_k8s_job(self, yaml_spec): - """_create_k8s_job creates a kubernetes job based on the yaml spec.""" - pod = k8s_client.V1Pod( - metadata=k8s_client.V1ObjectMeta( - generate_name=yaml_spec['metadata']['generateName'], - annotations=yaml_spec['metadata']['annotations'])) - container = k8s_client.V1Container( - name=yaml_spec['spec']['containers'][0]['name'], - image=yaml_spec['spec']['containers'][0]['image'], - args=yaml_spec['spec']['containers'][0]['args']) - pod.spec = k8s_client.V1PodSpec( - restart_policy=yaml_spec['spec']['restartPolicy'], - containers=[container], - service_account_name=yaml_spec['spec']['serviceAccountName']) - try: - api_response = self._corev1.create_namespaced_pod( - yaml_spec['metadata']['namespace'], pod) - return api_response.metadata.name, True - except k8s_client.rest.ApiException as e: - logging.exception( - "Exception when calling CoreV1Api->create_namespaced_pod: {}\n" - .format(str(e))) - return '', False - - def _wait_for_k8s_job(self, pod_name, yaml_spec, timeout): - """_wait_for_k8s_job waits for the job to complete.""" - status = 'running' - start_time = datetime.now() - while status in ['pending', 'running']: - # Pod pending values: https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1PodStatus.md - try: - api_response = self._corev1.read_namespaced_pod( - pod_name, yaml_spec['metadata']['namespace']) - status = api_response.status.phase.lower() - time.sleep(5) - elapsed_time = (datetime.now() - start_time).seconds - logging.info('{} seconds: waiting for job to complete'.format( - elapsed_time)) - if elapsed_time > timeout: - logging.info('Kubernetes job timeout') - return False - except k8s_client.rest.ApiException as e: - logging.exception( - 'Exception when calling CoreV1Api->read_namespaced_pod: {}\n' - .format(str(e))) - return False - return status == 'succeeded' - - def _delete_k8s_job(self, pod_name, yaml_spec): - """_delete_k8s_job deletes a pod.""" - try: - api_response = self._corev1.delete_namespaced_pod( - pod_name, - yaml_spec['metadata']['namespace'], - body=k8s_client.V1DeleteOptions()) - except k8s_client.rest.ApiException as e: - logging.exception( - 'Exception when calling CoreV1Api->delete_namespaced_pod: {}\n' - .format(str(e))) - - def _read_pod_log(self, pod_name, yaml_spec): - try: - api_response = self._corev1.read_namespaced_pod_log( - pod_name, yaml_spec['metadata']['namespace']) - except k8s_client.rest.ApiException as e: - logging.exception( - 'Exception when calling CoreV1Api->read_namespaced_pod_log: {}\n' - .format(str(e))) - return False - return api_response - - def _read_pod_status(self, pod_name, namespace): - try: - # Using read_namespaced_pod due to the following error: "pods \"kaniko-p2phh\" is forbidden: User \"system:serviceaccount:kubeflow:jupyter-notebook\" cannot get pods/status in the namespace \"kubeflow\"" - #api_response = self._corev1.read_namespaced_pod_status(pod_name, namespace) - api_response = self._corev1.read_namespaced_pod(pod_name, namespace) - except k8s_client.rest.ApiException as e: - logging.exception( - 'Exception when calling CoreV1Api->read_namespaced_pod_status: {}\n' - .format(str(e))) - return False - return api_response - - def run_job(self, yaml_spec, timeout=600): - """run_job runs a kubernetes job and clean up afterwards.""" - pod_name, succ = self._create_k8s_job(yaml_spec) - namespace = yaml_spec['metadata']['namespace'] - if not succ: - raise RuntimeError('Kubernetes job creation failed.') - # timeout in seconds - succ = self._wait_for_k8s_job(pod_name, yaml_spec, timeout) - if not succ: - logging.info('Kubernetes job failed.') - print(self._read_pod_log(pod_name, yaml_spec)) - raise RuntimeError('Kubernetes job failed.') - status_obj = self._read_pod_status(pod_name, namespace) - self._delete_k8s_job(pod_name, yaml_spec) - return status_obj diff --git a/sdk/python/kfp/deprecated/containers/entrypoint.py b/sdk/python/kfp/deprecated/containers/entrypoint.py deleted file mode 100644 index 26c7294491a..00000000000 --- a/sdk/python/kfp/deprecated/containers/entrypoint.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, NamedTuple, Optional, Union - -import fire -from google.protobuf import json_format - -from kfp.deprecated.containers import entrypoint_utils -from kfp.deprecated.dsl import artifact -from kfp.pipeline_spec import pipeline_spec_pb2 - -FN_SOURCE = 'ml/main.py' -FN_NAME_ARG = 'function_name' - -PARAM_METADATA_SUFFIX = '_input_param_metadata_file' -ARTIFACT_METADATA_SUFFIX = '_input_artifact_metadata_file' -FIELD_NAME_SUFFIX = '_input_field_name' -ARGO_PARAM_SUFFIX = '_input_argo_param' -INPUT_URI_SUFFIX = '_input_uri' -PRODUCER_POD_ID_SUFFIX = '_pod_id' -OUTPUT_NAME_SUFFIX = '_input_output_name' - -OUTPUT_PARAM_PATH_SUFFIX = '_parameter_output_path' -OUTPUT_ARTIFACT_PATH_SUFFIX = '_artifact_output_uri' - -METADATA_FILE_ARG = 'executor_metadata_json_file' - - -class InputParam(object): - """POD that holds an input parameter.""" - - def __init__(self, - value: Optional[Union[str, float, int]] = None, - metadata_file: Optional[str] = None, - field_name: Optional[str] = None): - """Instantiates an InputParam object. - - Args: - value: The actual value of the parameter. - metadata_file: The location of the metadata JSON file output by the - producer step. - field_name: The output name of the producer. - - Raises: - ValueError: when neither of the following is true: - 1) value is provided, and metadata_file and field_name are not; or - 2) both metadata_file and field_name are provided, and value is not. - """ - if not (value is not None and not (metadata_file or field_name) or - (metadata_file and field_name and value is None)): - raise ValueError( - 'Either value or both metadata_file and field_name ' - 'needs to be provided. Got value={value}, field_name=' - '{field_name}, metadata_file={metadata_file}'.format( - value=value, - field_name=field_name, - metadata_file=metadata_file)) - if value is not None: - self._value = value - else: - # Parse the value by inspecting the producer's metadata JSON file. - self._value = entrypoint_utils.get_parameter_from_output( - metadata_file, field_name) - - self._metadata_file = metadata_file - self._field_name = field_name - - # Following properties are read-only - @property - def value(self) -> Union[float, str, int]: - return self._value - - @property - def metadata_file(self) -> str: - return self._metadata_file - - @property - def field_name(self) -> str: - return self._field_name - - -class InputArtifact(object): - """POD that holds an input artifact.""" - - def __init__(self, - uri: Optional[str] = None, - metadata_file: Optional[str] = None, - output_name: Optional[str] = None): - """Instantiates an InputParam object. - - Args: - uri: The uri holds the input artifact. - metadata_file: The location of the metadata JSON file output by the - producer step. - output_name: The output name of the artifact in producer step. - - Raises: - ValueError: when neither of the following is true: - 1) uri is provided, and metadata_file and output_name are not; or - 2) both metadata_file and output_name are provided, and uri is not. - """ - if not ((uri and not (metadata_file or output_name) or - (metadata_file and output_name and not uri))): - raise ValueError( - 'Either uri or both metadata_file and output_name ' - 'needs to be provided. Got uri={uri}, output_name=' - '{output_name}, metadata_file={metadata_file}'.format( - uri=uri, - output_name=output_name, - metadata_file=metadata_file)) - - self._metadata_file = metadata_file - self._output_name = output_name - if uri: - self._uri = uri - else: - self._uri = self.get_artifact().uri - - # Following properties are read-only. - @property - def uri(self) -> str: - return self._uri - - @property - def metadata_file(self) -> str: - return self._metadata_file - - @property - def output_name(self) -> str: - return self._output_name - - def get_artifact(self) -> artifact.Artifact: - """Gets an artifact object by parsing metadata or creating one from - uri.""" - if self.metadata_file and self.output_name: - return entrypoint_utils.get_artifact_from_output( - self.metadata_file, self.output_name) - else: - # Provide an empty schema when returning a raw Artifact. - result = artifact.Artifact( - instance_schema=artifact.DEFAULT_ARTIFACT_SCHEMA) - result.uri = self.uri - return result - - -def _write_output_metadata_file(fn_res: Union[int, str, float, NamedTuple], - output_artifacts: Dict[str, artifact.Artifact], - output_metadata_path: str): - """Writes the output metadata file to the designated place.""" - # If output_params is a singleton value, needs to transform it to a mapping. - output_parameters = {} - if isinstance(fn_res, (int, str, float)): - output_parameters['output'] = fn_res - else: - # When multiple outputs, we'll need to match each field to the output paths. - for idx, output_name in enumerate(fn_res._fields): - output_parameters[output_name] = fn_res[idx] - - executor_output = entrypoint_utils.get_executor_output( - output_artifacts=output_artifacts, output_params=output_parameters) - - with open(output_metadata_path, 'w') as f: - f.write(json_format.MessageToJson(executor_output)) - - return executor_output - - -def main(executor_input_str: str, - function_name: str, - output_metadata_path: Optional[str] = None): - """Container entrypoint used by KFP Python function based component. - - executor_input_str: A serialized ExecutorInput proto message. - function_name: The name of the user-defined function. - output_metadata_path: A local path where the output metadata JSON file should - be written to. - """ - executor_input = pipeline_spec_pb2.ExecutorInput() - json_format.Parse(text=executor_input_str, message=executor_input) - output_metadata_path = output_metadata_path or executor_input.outputs.output_file - parameter_dict = {} # kwargs to be passed to UDF. - for name, input_param in executor_input.inputs.parameters.items(): - parameter_dict[name] = entrypoint_utils.get_python_value(input_param) - - for name, input_artifacts in executor_input.inputs.artifacts.items(): - parameter_dict[name] = artifact.Artifact.get_from_runtime_artifact( - input_artifacts.artifacts[0]) - - # Also, determine a way to inspect the function signature to decide the type - # of output artifacts. - fn = entrypoint_utils.import_func_from_source(FN_SOURCE, function_name) - - # In the ExeuctorInput message passed into the entrypoint, the output artifact - # URIs are already specified. The output artifact is constructed according to - # the specified URIs + type information retrieved from function signature. - output_uris = {} - for name, output_artifacts in executor_input.outputs.artifacts.items(): - output_uris[name] = output_artifacts.artifacts[0].uri - - output_artifacts = entrypoint_utils.get_output_artifacts(fn, output_uris) - for name, art in output_artifacts.items(): - parameter_dict[name] = art - - # Execute the user function. fn_res is expected to contain output parameters - # only. It's either an namedtuple or a single primitive value. - fn_res = fn(**parameter_dict) - - _write_output_metadata_file( - fn_res=fn_res, - output_artifacts=output_artifacts, - output_metadata_path=output_metadata_path) - - -if __name__ == '__main__': - fire.Fire(main) diff --git a/sdk/python/kfp/deprecated/containers/entrypoint_utils.py b/sdk/python/kfp/deprecated/containers/entrypoint_utils.py deleted file mode 100644 index 4a7f30298aa..00000000000 --- a/sdk/python/kfp/deprecated/containers/entrypoint_utils.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from absl import logging -import importlib -import sys -from typing import Callable, Dict, Optional, Union -from google.protobuf import json_format - -from kfp.deprecated.components import _python_op -from kfp.deprecated.containers import _gcs_helper -from kfp.pipeline_spec import pipeline_spec_pb2 -from kfp.deprecated.dsl import artifact - -# If path starts with one of those, consider files are in remote filesystem. -_REMOTE_FS_PREFIX = ['gs://', 'hdfs://', 's3://'] - -# Constant user module name when importing the function from a Python file. -_USER_MODULE = 'user_module' - - -def get_parameter_from_output(file_path: str, param_name: str): - """Gets a parameter value by its name from output metadata JSON.""" - output = pipeline_spec_pb2.ExecutorOutput() - json_format.Parse( - text=_gcs_helper.GCSHelper.read_from_gcs_path(file_path), - message=output) - value = output.parameters[param_name] - return getattr(value, value.WhichOneof('value')) - - -def get_artifact_from_output(file_path: str, - output_name: str) -> artifact.Artifact: - """Gets an artifact object from output metadata JSON.""" - output = pipeline_spec_pb2.ExecutorOutput() - json_format.Parse( - text=_gcs_helper.GCSHelper.read_from_gcs_path(file_path), - message=output) - # Currently we bear the assumption that each output contains only one artifact - json_str = json_format.MessageToJson( - output.artifacts[output_name].artifacts[0], sort_keys=True) - - # Convert runtime_artifact to Python artifact - return artifact.Artifact.deserialize(json_str) - - -def import_func_from_source(source_path: str, fn_name: str) -> Callable: - """Imports a function from a Python file. - - The implementation is borrowed from - https://github.com/tensorflow/tfx/blob/8f25a4d1cc92dfc8c3a684dfc8b82699513cafb5/tfx/utils/import_utils.py#L50 - - Args: - source_path: The local path to the Python source file. - fn_name: The function name, which can be found in the source file. - - Return: A Python function object. - - Raises: - ImportError when failed to load the source file or cannot find the function - with the given name. - """ - if any([source_path.startswith(prefix) for prefix in _REMOTE_FS_PREFIX]): - raise RuntimeError( - 'Only local source file can be imported. Please make ' - 'sure the user code is built into executor container. ' - 'Got file path: %s' % source_path) - try: - loader = importlib.machinery.SourceFileLoader( - fullname=_USER_MODULE, - path=source_path, - ) - spec = importlib.util.spec_from_loader( - loader.name, loader, origin=source_path) - module = importlib.util.module_from_spec(spec) - sys.modules[loader.name] = module - loader.exec_module(module) - except IOError: - raise ImportError( - '{} in {} not found in import_func_from_source()'.format( - fn_name, source_path)) - try: - return getattr(module, fn_name) - except AttributeError: - raise ImportError( - '{} in {} not found in import_func_from_source()'.format( - fn_name, source_path)) - - -def get_output_artifacts( - fn: Callable, output_uris: Dict[str, - str]) -> Dict[str, artifact.Artifact]: - """Gets the output artifacts from function signature and provided URIs. - - Args: - fn: A user-provided function, whose signature annotates the type of output - artifacts. - output_uris: The mapping from output artifact name to its URI. - - Returns: - A mapping from output artifact name to Python artifact objects. - """ - # Inspect the function signature to determine the set of output artifact. - spec = _python_op._extract_component_interface(fn) - - result = {} # Mapping from output name to artifacts. - for output in spec.outputs: - if (getattr(output, '_passing_style', - None) == _python_op.OutputArtifact): - # Creates an artifact according to its name - type_name = getattr(output, 'type', None) - if not type_name: - continue - - try: - artifact_cls = getattr( - importlib.import_module( - artifact.KFP_ARTIFACT_ONTOLOGY_MODULE), type_name) - - except (AttributeError, ImportError, ValueError): - logging.warning(( - 'Could not load artifact class %s.%s; using fallback deserialization' - ' for the relevant artifact. Please make sure that any artifact ' - 'classes can be imported within your container or environment.' - ), artifact.KFP_ARTIFACT_ONTOLOGY_MODULE, type_name) - artifact_cls = artifact.Artifact - - if artifact_cls == artifact.Artifact: - # Provide an empty schema if instantiating an bare-metal artifact. - art = artifact_cls( - instance_schema=artifact.DEFAULT_ARTIFACT_SCHEMA) - else: - art = artifact_cls() - - art.uri = output_uris[output.name] - result[output.name] = art - - return result - - -def _get_pipeline_value( - value: Union[int, float, str]) -> Optional[pipeline_spec_pb2.Value]: - """Converts Python primitive value to pipeline value pb.""" - if value is None: - return None - - result = pipeline_spec_pb2.Value() - if isinstance(value, int): - result.int_value = value - elif isinstance(value, float): - result.double_value = value - elif isinstance(value, str): - result.string_value = value - else: - raise TypeError('Got unknown type of value: {}'.format(value)) - - return result - - -def get_python_value(value: pipeline_spec_pb2.Value) -> Union[int, float, str]: - """Gets Python value from pipeline value pb message.""" - return getattr(value, value.WhichOneof('value')) - - -def get_executor_output( - output_artifacts: Dict[str, artifact.Artifact], - output_params: Dict[str, Union[int, float, str]] -) -> pipeline_spec_pb2.ExecutorOutput: - """Gets the output metadata message.""" - result = pipeline_spec_pb2.ExecutorOutput() - - for name, art in output_artifacts.items(): - result.artifacts[name].CopyFrom( - pipeline_spec_pb2.ArtifactList(artifacts=[art.runtime_artifact])) - - for name, param in output_params.items(): - result.parameters[name].CopyFrom(_get_pipeline_value(param)) - - return result diff --git a/sdk/python/kfp/deprecated/containers_tests/__init__.py b/sdk/python/kfp/deprecated/containers_tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/kfp/deprecated/containers_tests/component_builder_test.py b/sdk/python/kfp/deprecated/containers_tests/component_builder_test.py deleted file mode 100644 index 29fc4f83f81..00000000000 --- a/sdk/python/kfp/deprecated/containers_tests/component_builder_test.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for kfp.containers._component_builder module.""" -import os -import shutil -import tempfile -import unittest -from unittest import mock - -from kfp.deprecated.containers import _component_builder -from kfp.deprecated.containers import _container_builder -from kfp.deprecated import components - -_TEST_TARGET_IMAGE = 'gcr.io/my-project/my-image' -_TEST_STAGING_LOCATION = 'gs://my-project/tmp' - - -class ComponentBuilderTest(unittest.TestCase): - - def setUp(self) -> None: - self._tmp_dir = tempfile.mkdtemp() - self._old_dir = os.getcwd() - with open( - os.path.join( - os.path.dirname(__file__), 'testdata', - 'expected_component.yaml'), 'r') as f: - self._expected_component_yaml = f.read() - os.chdir(self._tmp_dir) - self.addCleanup(os.chdir, self._old_dir) - self.addCleanup(shutil.rmtree, self._tmp_dir) - - @mock.patch.object( - _container_builder.ContainerBuilder, - 'build', - return_value='gcr.io/my-project/my-image:123456', - autospec=True) - def testBuildV2PythonComponent(self, mock_build): - self.maxDiff = 2400 - - def test_function(test_param: str, - test_artifact: components.InputArtifact('Dataset'), - test_output: components.OutputArtifact('Model')): - pass - - _component_builder.build_python_component( - component_func=test_function, - target_image=_TEST_TARGET_IMAGE, - staging_gcs_path=_TEST_STAGING_LOCATION, - target_component_file='component.yaml', - is_v2=True) - - with open('component.yaml', 'r') as f: - actual_component_yaml = f.read() - - self.assertEquals(actual_component_yaml, self._expected_component_yaml) diff --git a/sdk/python/kfp/deprecated/containers_tests/test_build_image_api.py b/sdk/python/kfp/deprecated/containers_tests/test_build_image_api.py deleted file mode 100644 index bdfb9d3e2e2..00000000000 --- a/sdk/python/kfp/deprecated/containers_tests/test_build_image_api.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the speci - -import os -import tempfile -import unittest -from pathlib import Path -from typing import Callable - -from kfp.deprecated.containers import build_image_from_working_dir - - -class MockImageBuilder: - - def __init__(self, - dockerfile_text_check: Callable[[str], None] = None, - requirements_text_check: Callable[[str], None] = None, - file_paths_check: Callable[[str], None] = None): - self.dockerfile_text_check = dockerfile_text_check - self.requirements_text_check = requirements_text_check - self.file_paths_check = file_paths_check - - def build(self, local_dir=None, target_image=None, timeout=1000): - if self.dockerfile_text_check: - actual_dockerfile_text = (Path(local_dir) / - 'Dockerfile').read_text() - self.dockerfile_text_check(actual_dockerfile_text) - if self.requirements_text_check: - actual_requirements_text = (Path(local_dir) / - 'requirements.txt').read_text() - self.requirements_text_check(actual_requirements_text) - if self.file_paths_check: - file_paths = set( - os.path.relpath(os.path.join(dirpath, file_name), local_dir) - for dirpath, dirnames, filenames in os.walk(local_dir) - for file_name in filenames) - self.file_paths_check(file_paths) - return target_image - - -class BuildImageApiTests(unittest.TestCase): - - def test_build_image_from_working_dir(self): - expected_dockerfile_text_re = ''' -FROM python:3.9 -WORKDIR /.* -COPY requirements.txt . -RUN python3 -m pip install -r requirements.txt -COPY . . -''' - #mock_builder = - with tempfile.TemporaryDirectory() as context_dir: - requirements_text = 'pandas==1.24' - requirements_txt_relpath = Path('.') / 'requirements.txt' - file1_py_relpath = Path('.') / 'lib' / 'file1.py' - file1_sh_relpath = Path('.') / 'lib' / 'file1.sh' - - context_path = Path(context_dir) - (context_path / - requirements_txt_relpath).write_text(requirements_text) - (context_path / file1_py_relpath).parent.mkdir( - parents=True, exist_ok=True) - (context_path / file1_py_relpath).write_text('#py file') - (context_path / file1_sh_relpath).parent.mkdir( - parents=True, exist_ok=True) - (context_path / file1_sh_relpath).write_text('#sh file') - expected_file_paths = { - 'Dockerfile', - str(requirements_txt_relpath), - str(file1_py_relpath), - } - - def dockerfile_text_check(actual_dockerfile_text): - self.assertRegex(actual_dockerfile_text.strip(), - expected_dockerfile_text_re.strip()) - - def requirements_text_check(actual_requirements_text): - self.assertEqual(actual_requirements_text.strip(), - requirements_text.strip()) - - def file_paths_check(file_paths): - self.assertEqual(file_paths, expected_file_paths) - - builder = MockImageBuilder(dockerfile_text_check, - requirements_text_check, - file_paths_check) - result = build_image_from_working_dir( - working_dir=context_dir, - base_image='python:3.9', - builder=builder) - - def test_image_cache(self): - builder = InvocationCountingDummyImageBuilder() - - from kfp.deprecated.containers._cache import clear_cache - clear_cache('build_image_from_working_dir') - - self.assertEqual(builder.invocations_count, 0) - with prepare_context_dir( - py_content='py1', sh_content='sh1') as context_dir: - build_image_from_working_dir( - working_dir=context_dir, - base_image='python:3.9', - builder=builder) - self.assertEqual(builder.invocations_count, 1) - - # Check that changes to .sh files do not break cache - with prepare_context_dir( - py_content='py1', sh_content='sh2') as context_dir: - build_image_from_working_dir( - working_dir=context_dir, - base_image='python:3.9', - builder=builder) - self.assertEqual(builder.invocations_count, 1) - - # Check that changes to .py files result in new image being built - with prepare_context_dir( - py_content='py2', sh_content='sh1') as context_dir: - build_image_from_working_dir( - working_dir=context_dir, - base_image='python:3.9', - builder=builder) - self.assertEqual(builder.invocations_count, 2) - - -class InvocationCountingDummyImageBuilder: - - def __init__(self): - self.invocations_count = 0 - - def build(self, local_dir=None, target_image=None, timeout=1000): - self.invocations_count = self.invocations_count + 1 - return "image/name@sha256:0123456789abcdef0123456789abcdef" - - -def prepare_context_dir(py_content: str = '#py file', - sh_content: str = '#sh file') -> str: - context_dir = tempfile.TemporaryDirectory() - #Preparing context - requirements_text = 'pandas==1.24' - requirements_txt_relpath = Path('.') / 'requirements.txt' - file1_py_relpath = Path('.') / 'lib' / 'file1.py' - file1_sh_relpath = Path('.') / 'lib' / 'file1.sh' - - context_path = Path(context_dir.name) - (context_path / requirements_txt_relpath).write_text(requirements_text) - (context_path / file1_py_relpath).parent.mkdir(parents=True, exist_ok=True) - (context_path / file1_py_relpath).write_text(py_content) - (context_path / file1_sh_relpath).parent.mkdir(parents=True, exist_ok=True) - (context_path / file1_sh_relpath).write_text(sh_content) - # End preparing context - - return context_dir - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/containers_tests/testdata/__init__.py b/sdk/python/kfp/deprecated/containers_tests/testdata/__init__.py deleted file mode 100644 index e7878caf33e..00000000000 --- a/sdk/python/kfp/deprecated/containers_tests/testdata/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file diff --git a/sdk/python/kfp/deprecated/containers_tests/testdata/executor_output.json b/sdk/python/kfp/deprecated/containers_tests/testdata/executor_output.json deleted file mode 100644 index 48091b09da8..00000000000 --- a/sdk/python/kfp/deprecated/containers_tests/testdata/executor_output.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "artifacts": { - "output": { - "artifacts": [ - { - "name": "test-artifact", - "uri": "gs://root/execution/output", - "type": { - "instanceSchema": "title: kfp.Model\ntype: object\nproperties:\n framework:\n type: string\n framework_version:\n type: string\n" - }, - "metadata": { - "test_property": "test value" - } - } - ] - } - }, - "parameters": { - "int_output": { - "intValue": 42 - }, - "string_output": { - "stringValue": "hello world!" - }, - "float_output": { - "doubleValue": 12.12 - } - } -} \ No newline at end of file diff --git a/sdk/python/kfp/deprecated/containers_tests/testdata/expected_component.yaml b/sdk/python/kfp/deprecated/containers_tests/testdata/expected_component.yaml deleted file mode 100644 index b3a525e8523..00000000000 --- a/sdk/python/kfp/deprecated/containers_tests/testdata/expected_component.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Test function -metadata: - annotations: - pipelines.kubeflow.org/component_v2: "true" -inputs: -- {name: test_param, type: String} -- {name: test_artifact, type: Dataset} -outputs: -- {name: test_output, type: Model} -implementation: - container: - image: gcr.io/my-project/my-image:123456 - command: [python, -m, kfp.containers.entrypoint] - args: - - --executor_input_str - - {executorInput: null} - - --function_name - - test_function - - --output_metadata_path - - {outputMetadata: null} diff --git a/sdk/python/kfp/deprecated/containers_tests/testdata/main.py b/sdk/python/kfp/deprecated/containers_tests/testdata/main.py deleted file mode 100644 index e72028a8a7a..00000000000 --- a/sdk/python/kfp/deprecated/containers_tests/testdata/main.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""User module under test.""" -from typing import NamedTuple -from kfp.deprecated import components -from kfp.deprecated.dsl import artifact -from kfp.deprecated.dsl import ontology_artifacts - - -def test_func( - test_param: str, test_artifact: components.InputArtifact('Dataset'), - test_output1: components.OutputArtifact('Model') -) -> NamedTuple('Outputs', [('test_output2', str)]): - assert test_param == 'hello from producer' - # In the associated test case, input artifact is produced by conventional - # KFP components, thus no concrete artifact type can be determined. - assert isinstance(test_artifact, artifact.Artifact) - assert isinstance(test_output1, ontology_artifacts.Model) - assert test_output1.uri - from collections import namedtuple - - Outputs = namedtuple('Outputs', 'test_output2') - return Outputs('bye world') - - -def test_func2( - test_param: str, test_artifact: components.InputArtifact('Dataset'), - test_output1: components.OutputArtifact('Model') -) -> NamedTuple('Outputs', [('test_output2', str)]): - assert test_param == 'hello from producer' - # In the associated test case, input artifact is produced by a new-styled - # KFP components with metadata, thus it's expected to be deserialized to - # Dataset object. - assert isinstance(test_artifact, ontology_artifacts.Dataset) - assert isinstance(test_output1, ontology_artifacts.Model) - assert test_output1.uri - from collections import namedtuple - - Outputs = namedtuple('Outputs', 'test_output2') - return Outputs('bye world') diff --git a/sdk/python/kfp/deprecated/containers_tests/testdata/pipeline_source.py b/sdk/python/kfp/deprecated/containers_tests/testdata/pipeline_source.py deleted file mode 100644 index e70ca294fda..00000000000 --- a/sdk/python/kfp/deprecated/containers_tests/testdata/pipeline_source.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Python source file under test.""" - - -def test_func(a, b): - return a + b diff --git a/sdk/python/kfp/deprecated/dsl/__init__.py b/sdk/python/kfp/deprecated/dsl/__init__.py deleted file mode 100644 index 8cce20f8a1a..00000000000 --- a/sdk/python/kfp/deprecated/dsl/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2018-2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from ._pipeline_param import PipelineParam, match_serialized_pipelineparam -from ._pipeline import Pipeline, PipelineExecutionMode, pipeline, get_pipeline_conf, PipelineConf -from ._container_op import BaseOp, ContainerOp, InputArgumentPath, UserContainer, Sidecar -from ._resource_op import ResourceOp -from ._volume_op import VolumeOp, VOLUME_MODE_RWO, VOLUME_MODE_RWM, VOLUME_MODE_ROM -from ._pipeline_volume import PipelineVolume -from ._volume_snapshot_op import VolumeSnapshotOp -from ._ops_group import OpsGroup, ExitHandler, Condition, ParallelFor, SubGraph -from ._component import graph_component, component - - -def importer(*args, **kwargs): - import warnings - from kfp.dsl import importer as v2importer - warnings.warn( - '`kfp.dsl.importer` is a deprecated alias and will be removed' - ' in KFP v2.0. Please import from `kfp.dsl` instead.', - category=FutureWarning) - return v2importer(*args, **kwargs) - - -EXECUTION_ID_PLACEHOLDER = '{{workflow.uid}}-{{pod.name}}' -RUN_ID_PLACEHOLDER = '{{workflow.uid}}' - -ROOT_PARAMETER_NAME = 'pipeline-root' diff --git a/sdk/python/kfp/deprecated/dsl/_component.py b/sdk/python/kfp/deprecated/dsl/_component.py deleted file mode 100644 index 0098932fd6f..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_component.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -from ._pipeline_param import PipelineParam -from .types import check_types, InconsistentTypeException -from ._ops_group import Graph -import kfp.deprecated as kfp - -# @deprecated( -# version='0.2.6', -# reason='This decorator does not seem to be used, so we deprecate it. ' -# 'If you need this decorator, please create an issue at ' -# 'https://github.com/kubeflow/pipelines/issues', -# ) -# def python_component(name, -# description=None, -# base_image=None, -# target_component_file: str = None): -# """Decorator for Python component functions. - -# This decorator adds the metadata to the function object itself. - -# Args: -# name: Human-readable name of the component -# description: Optional. Description of the component -# base_image: Optional. Docker container image to use as the base of the -# component. Needs to have Python 3.5+ installed. -# target_component_file: Optional. Local file to store the component -# definition. The file can then be used for sharing. - -# Returns: -# The same function (with some metadata fields set). - -# Example: -# :: - -# @dsl.python_component( -# name='my awesome component', -# description='Come, Let\'s play', -# base_image='tensorflow/tensorflow:1.11.0-py3', -# ) -# def my_component(a: str, b: int) -> str: -# ... -# """ - -# def _python_component(func): -# func._component_human_name = name -# if description: -# func._component_description = description -# if base_image: -# func._component_base_image = base_image -# if target_component_file: -# func._component_target_component_file = target_component_file -# return func - -# return _python_component - - -def component(func): - """Decorator for component functions that returns a ContainerOp. - - This is useful to enable type checking in the DSL compiler. - - Example: - :: - - @dsl.component - def foobar(model: TFModel(), step: MLStep()): - return dsl.ContainerOp() - """ - from functools import wraps - - @wraps(func) - def _component(*args, **kargs): - from ..components._python_op import _extract_component_interface - component_meta = _extract_component_interface(func) - if kfp.TYPE_CHECK: - arg_index = 0 - for arg in args: - if isinstance(arg, PipelineParam) and not check_types( - arg.param_type, component_meta.inputs[arg_index].type): - raise InconsistentTypeException( - 'Component "' + component_meta.name + - '" is expecting ' + - component_meta.inputs[arg_index].name + ' to be type(' + - str(component_meta.inputs[arg_index].type) + - '), but the passed argument is type(' + - str(arg.param_type) + ')') - arg_index += 1 - if kargs is not None: - for key in kargs: - if isinstance(kargs[key], PipelineParam): - for input_spec in component_meta.inputs: - if input_spec.name == key and not check_types( - kargs[key].param_type, input_spec.type): - raise InconsistentTypeException( - 'Component "' + component_meta.name + - '" is expecting ' + input_spec.name + - ' to be type(' + str(input_spec.type) + - '), but the passed argument is type(' + - str(kargs[key].param_type) + ')') - - container_op = func(*args, **kargs) - container_op._set_metadata(component_meta) - return container_op - - return _component - - -#TODO: combine the component and graph_component decorators into one -def graph_component(func): - """Decorator for graph component functions. - - This decorator returns an ops_group. - - Example: - :: - - # Warning: caching is tricky when recursion is involved. Please be careful - # and set proper max_cache_staleness in case of infinite loop. - import kfp.dsl as dsl - @dsl.graph_component - def flip_component(flip_result): - print_flip = PrintOp(flip_result) - flipA = FlipCoinOp().after(print_flip) - flipA.execution_options.caching_strategy.max_cache_staleness = "P0D" - with dsl.Condition(flipA.output == 'heads'): - flip_component(flipA.output) - return {'flip_result': flipA.output} - """ - from functools import wraps - - @wraps(func) - def _graph_component(*args, **kargs): - # We need to make sure that the arguments are correctly mapped to inputs - # regardless of the passing order - signature = inspect.signature(func) - bound_arguments = signature.bind(*args, **kargs) - graph_ops_group = Graph(func.__name__) - graph_ops_group.inputs = list(bound_arguments.arguments.values()) - graph_ops_group.arguments = bound_arguments.arguments - for input in graph_ops_group.inputs: - if not isinstance(input, PipelineParam): - raise ValueError('arguments to ' + func.__name__ + - ' should be PipelineParams.') - - # Entering the Graph Context - with graph_ops_group: - # Call the function - if not graph_ops_group.recursive_ref: - func(*args, **kargs) - - return graph_ops_group - - return _graph_component diff --git a/sdk/python/kfp/deprecated/dsl/_component_bridge.py b/sdk/python/kfp/deprecated/dsl/_component_bridge.py deleted file mode 100644 index 0f10afecbd4..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_component_bridge.py +++ /dev/null @@ -1,691 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import collections -import copy -import json -import pathlib -from typing import Any, Mapping, Optional - -from kfp.deprecated._config import COMPILING_FOR_V2 -from kfp.deprecated.components import _structures -from kfp.deprecated.components import _components -from kfp.deprecated.components import _naming -from kfp.deprecated import dsl -from kfp.deprecated.dsl import _container_op -from kfp.deprecated.dsl import _for_loop -from kfp.deprecated.dsl import _pipeline_param -from kfp.deprecated.dsl import component_spec as dsl_component_spec -from kfp.deprecated.dsl import dsl_utils -from kfp.deprecated.dsl import types -from kfp.deprecated.dsl.types import type_utils -from kfp.pipeline_spec import pipeline_spec_pb2 - -# Placeholder to represent the output directory hosting all the generated URIs. -# Its actual value will be specified during pipeline compilation. -# The format of OUTPUT_DIR_PLACEHOLDER is serialized dsl.PipelineParam, to -# ensure being extracted as a pipeline parameter during compilation. -# Note that we cannot direclty import dsl module here due to circular -# dependencies. -OUTPUT_DIR_PLACEHOLDER = '{{pipelineparam:op=;name=pipeline-root}}' -# Placeholder to represent to UID of the current pipeline at runtime. -# Will be replaced by engine-specific placeholder during compilation. -RUN_ID_PLACEHOLDER = '{{kfp.run_uid}}' -# Format of the Argo parameter used to pass the producer's Pod ID to -# the consumer. -PRODUCER_POD_NAME_PARAMETER = '{}-producer-pod-id-' -# Format of the input output port name placeholder. -INPUT_OUTPUT_NAME_PATTERN = '{{{{kfp.input-output-name.{}}}}}' -# Fixed name for per-task output metadata json file. -OUTPUT_METADATA_JSON = '/tmp/outputs/executor_output.json' -# Executor input placeholder. -_EXECUTOR_INPUT_PLACEHOLDER = '{{$}}' - - -# TODO(chensun): block URI placeholder usage in v1. -def _generate_output_uri_placeholder(port_name: str) -> str: - """Generates the URI placeholder for an output.""" - return "{{{{$.outputs.artifacts['{}'].uri}}}}".format(port_name) - - -def _generate_input_uri_placeholder(port_name: str) -> str: - """Generates the URI placeholder for an input.""" - return "{{{{$.inputs.artifacts['{}'].uri}}}}".format(port_name) - - -def _generate_output_metadata_path() -> str: - """Generates the URI to write the output metadata JSON file.""" - - return OUTPUT_METADATA_JSON - - -def _generate_input_metadata_path(port_name: str) -> str: - """Generates the placeholder for input artifact metadata file.""" - - # Return a placeholder for path to input artifact metadata, which will be - # rewritten during pipeline compilation. - return str( - pathlib.PurePosixPath( - OUTPUT_DIR_PLACEHOLDER, RUN_ID_PLACEHOLDER, - '{{{{inputs.parameters.{input}}}}}'.format( - input=PRODUCER_POD_NAME_PARAMETER.format(port_name)), - OUTPUT_METADATA_JSON)) - - -def _generate_input_output_name(port_name: str) -> str: - """Generates the placeholder for input artifact's output name.""" - - # Return a placeholder for the output port name of the input artifact, which - # will be rewritten during pipeline compilation. - return INPUT_OUTPUT_NAME_PATTERN.format(port_name) - - -def _generate_executor_input() -> str: - """Generates the placeholder for serialized executor input.""" - return _EXECUTOR_INPUT_PLACEHOLDER - - -class ExtraPlaceholderResolver: - - def __init__(self): - self.input_paths = {} - self.input_metadata_paths = {} - self.output_paths = {} - - def resolve_placeholder( - self, - arg, - component_spec: _structures.ComponentSpec, - arguments: dict, - ) -> str: - inputs_dict = { - input_spec.name: input_spec - for input_spec in component_spec.inputs or [] - } - - if isinstance(arg, _structures.InputUriPlaceholder): - input_name = arg.input_name - if input_name in arguments: - input_uri = _generate_input_uri_placeholder(input_name) - self.input_paths[ - input_name] = _components._generate_input_file_name( - input_name) - return input_uri - else: - input_spec = inputs_dict[input_name] - if input_spec.optional: - return None - else: - raise ValueError( - 'No value provided for input {}'.format(input_name)) - - elif isinstance(arg, _structures.OutputUriPlaceholder): - output_name = arg.output_name - output_uri = _generate_output_uri_placeholder(output_name) - self.output_paths[ - output_name] = _components._generate_output_file_name( - output_name) - return output_uri - - elif isinstance(arg, _structures.InputMetadataPlaceholder): - input_name = arg.input_name - if input_name in arguments: - input_metadata_path = _generate_input_metadata_path(input_name) - self.input_metadata_paths[input_name] = input_metadata_path - return input_metadata_path - else: - input_spec = inputs_dict[input_name] - if input_spec.optional: - return None - else: - raise ValueError( - 'No value provided for input {}'.format(input_name)) - - elif isinstance(arg, _structures.InputOutputPortNamePlaceholder): - input_name = arg.input_name - if input_name in arguments: - return _generate_input_output_name(input_name) - else: - input_spec = inputs_dict[input_name] - if input_spec.optional: - return None - else: - raise ValueError( - 'No value provided for input {}'.format(input_name)) - - elif isinstance(arg, _structures.OutputMetadataPlaceholder): - # TODO: Consider making the output metadata per-artifact. - return _generate_output_metadata_path() - elif isinstance(arg, _structures.ExecutorInputPlaceholder): - return _generate_executor_input() - - return None - - -def _create_container_op_from_component_and_arguments( - component_spec: _structures.ComponentSpec, - arguments: Mapping[str, Any], - component_ref: Optional[_structures.ComponentReference] = None, -) -> _container_op.ContainerOp: - """Instantiates ContainerOp object. - - Args: - component_spec: The component spec object. - arguments: The dictionary of component arguments. - component_ref: (only for v1) The component references. - - Returns: - A ContainerOp instance. - """ - # Add component inputs with default value to the arguments dict if they are not - # in the arguments dict already. - arguments = arguments.copy() - for input_spec in component_spec.inputs or []: - if input_spec.name not in arguments and input_spec.default is not None: - default_value = input_spec.default - if input_spec.type == 'Integer': - default_value = int(default_value) - elif input_spec.type == 'Float': - default_value = float(default_value) - elif (type_utils.is_parameter_type(input_spec.type) and - COMPILING_FOR_V2): - parameter_type = type_utils.get_parameter_type(input_spec.type) - default_value = type_utils.deserialize_parameter_value( - value=default_value, parameter_type=parameter_type) - - arguments[input_spec.name] = default_value - - # Check types of the reference arguments and serialize PipelineParams - original_arguments = arguments - arguments = arguments.copy() - for input_name, argument_value in arguments.items(): - if isinstance(argument_value, _pipeline_param.PipelineParam): - input_type = component_spec._inputs_dict[input_name].type - argument_type = argument_value.param_type - types.verify_type_compatibility( - argument_type, input_type, - 'Incompatible argument passed to the input "{}" of component "{}": ' - .format(input_name, component_spec.name)) - - arguments[input_name] = str(argument_value) - if isinstance(argument_value, _container_op.ContainerOp): - raise TypeError( - 'ContainerOp object was passed to component as an input argument. ' - 'Pass a single output instead.') - placeholder_resolver = ExtraPlaceholderResolver() - resolved_cmd = _components._resolve_command_line_and_paths( - component_spec=component_spec, - arguments=arguments, - placeholder_resolver=placeholder_resolver.resolve_placeholder, - ) - - container_spec = component_spec.implementation.container - - old_warn_value = _container_op.ContainerOp._DISABLE_REUSABLE_COMPONENT_WARNING - _container_op.ContainerOp._DISABLE_REUSABLE_COMPONENT_WARNING = True - - output_paths = collections.OrderedDict(resolved_cmd.output_paths or {}) - output_paths.update(placeholder_resolver.output_paths) - input_paths = collections.OrderedDict(resolved_cmd.input_paths or {}) - input_paths.update(placeholder_resolver.input_paths) - - artifact_argument_paths = [ - dsl.InputArgumentPath( - argument=arguments[input_name], - input=input_name, - path=path, - ) for input_name, path in input_paths.items() - ] - - task = _container_op.ContainerOp( - name=component_spec.name or _components._default_component_name, - image=container_spec.image, - command=resolved_cmd.command, - arguments=resolved_cmd.args, - file_outputs=output_paths, - artifact_argument_paths=artifact_argument_paths, - ) - _container_op.ContainerOp._DISABLE_REUSABLE_COMPONENT_WARNING = old_warn_value - - component_meta = copy.copy(component_spec) - task._set_metadata(component_meta, original_arguments) - if component_ref: - component_ref_without_spec = copy.copy(component_ref) - component_ref_without_spec.spec = None - task._component_ref = component_ref_without_spec - - task._parameter_arguments = resolved_cmd.inputs_consumed_by_value - name_to_spec_type = {} - if component_meta.inputs: - name_to_spec_type = { - input.name: { - 'type': input.type, - 'default': input.default, - } for input in component_meta.inputs - } - - if COMPILING_FOR_V2: - for name, spec_type in name_to_spec_type.items(): - if (name in original_arguments and - type_utils.is_parameter_type(spec_type['type'])): - if isinstance(original_arguments[name], (list, dict)): - task._parameter_arguments[name] = json.dumps( - original_arguments[name]) - else: - task._parameter_arguments[name] = str( - original_arguments[name]) - - for name in list(task.artifact_arguments.keys()): - if name in task._parameter_arguments: - del task.artifact_arguments[name] - - for name in list(task.input_artifact_paths.keys()): - if name in task._parameter_arguments: - del task.input_artifact_paths[name] - - # Previously, ContainerOp had strict requirements for the output names, so we - # had to convert all the names before passing them to the ContainerOp - # constructor. - # Outputs with non-pythonic names could not be accessed using their original - # names. Now ContainerOp supports any output names, so we're now using the - # original output names. However to support legacy pipelines, we're also - # adding output references with pythonic names. - # TODO: Add warning when people use the legacy output names. - output_names = [ - output_spec.name for output_spec in component_spec.outputs or [] - ] # Stabilizing the ordering - output_name_to_python = _naming.generate_unique_name_conversion_table( - output_names, _naming._sanitize_python_function_name) - for output_name in output_names: - pythonic_output_name = output_name_to_python[output_name] - # Note: Some component outputs are currently missing from task.outputs - # (e.g. MLPipeline UI Metadata) - if pythonic_output_name not in task.outputs and output_name in task.outputs: - task.outputs[pythonic_output_name] = task.outputs[output_name] - - if container_spec.env: - from kubernetes import client as k8s_client - for name, value in container_spec.env.items(): - task.container.add_env_variable( - k8s_client.V1EnvVar(name=name, value=value)) - - if component_spec.metadata: - annotations = component_spec.metadata.annotations or {} - for key, value in annotations.items(): - task.add_pod_annotation(key, value) - for key, value in (component_spec.metadata.labels or {}).items(): - task.add_pod_label(key, value) - # Disabling the caching for the volatile components by default - if annotations.get('volatile_component', 'false') == 'true': - task.execution_options.caching_strategy.max_cache_staleness = 'P0D' - - _attach_v2_specs(task, component_spec, original_arguments) - - return task - - -def _attach_v2_specs( - task: _container_op.ContainerOp, - component_spec: _structures.ComponentSpec, - arguments: Mapping[str, Any], -) -> None: - """Attaches v2 specs to a ContainerOp object. - - Attach v2_specs to the ContainerOp object regardless whether the pipeline is - being compiled to v1 (Argo yaml) or v2 (IR json). - However, there're different behaviors for the two cases. Namely, resolved - commands and arguments, error handling, etc. - Regarding the difference in error handling, v2 has a stricter requirement on - input type annotation. For instance, an input without any type annotation is - viewed as an artifact, and if it's paired with InputValuePlaceholder, an - error will be thrown at compile time. However, we cannot raise such an error - in v1, as it wouldn't break existing pipelines. - - Args: - task: The ContainerOp object to attach IR specs. - component_spec: The component spec object. - arguments: The dictionary of component arguments. - """ - - def _resolve_commands_and_args_v2( - component_spec: _structures.ComponentSpec, - arguments: Mapping[str, Any], - ) -> _components._ResolvedCommandLineAndPaths: - """Resolves the command line argument placeholders for v2 (IR). - - Args: - component_spec: The component spec object. - arguments: The dictionary of component arguments. - - Returns: - A named tuple: _components._ResolvedCommandLineAndPaths. - """ - inputs_dict = { - input_spec.name: input_spec - for input_spec in component_spec.inputs or [] - } - outputs_dict = { - output_spec.name: output_spec - for output_spec in component_spec.outputs or [] - } - - def _input_artifact_uri_placeholder(input_key: str) -> str: - if COMPILING_FOR_V2 and type_utils.is_parameter_type( - inputs_dict[input_key].type): - raise TypeError( - 'Input "{}" with type "{}" cannot be paired with ' - 'InputUriPlaceholder.'.format(input_key, - inputs_dict[input_key].type)) - else: - return _generate_input_uri_placeholder(input_key) - - def _input_artifact_path_placeholder(input_key: str) -> str: - if COMPILING_FOR_V2 and type_utils.is_parameter_type( - inputs_dict[input_key].type): - raise TypeError( - 'Input "{}" with type "{}" cannot be paired with ' - 'InputPathPlaceholder.'.format(input_key, - inputs_dict[input_key].type)) - else: - return "{{{{$.inputs.artifacts['{}'].path}}}}".format(input_key) - - def _input_parameter_placeholder(input_key: str) -> str: - if COMPILING_FOR_V2 and not type_utils.is_parameter_type( - inputs_dict[input_key].type): - raise TypeError( - 'Input "{}" with type "{}" cannot be paired with ' - 'InputValuePlaceholder.'.format( - input_key, inputs_dict[input_key].type)) - else: - return "{{{{$.inputs.parameters['{}']}}}}".format(input_key) - - def _output_artifact_uri_placeholder(output_key: str) -> str: - if COMPILING_FOR_V2 and type_utils.is_parameter_type( - outputs_dict[output_key].type): - raise TypeError( - 'Output "{}" with type "{}" cannot be paired with ' - 'OutputUriPlaceholder.'.format( - output_key, outputs_dict[output_key].type)) - else: - return _generate_output_uri_placeholder(output_key) - - def _output_artifact_path_placeholder(output_key: str) -> str: - return "{{{{$.outputs.artifacts['{}'].path}}}}".format(output_key) - - def _output_parameter_path_placeholder(output_key: str) -> str: - return "{{{{$.outputs.parameters['{}'].output_file}}}}".format( - output_key) - - def _resolve_output_path_placeholder(output_key: str) -> str: - if type_utils.is_parameter_type(outputs_dict[output_key].type): - return _output_parameter_path_placeholder(output_key) - else: - return _output_artifact_path_placeholder(output_key) - - placeholder_resolver = ExtraPlaceholderResolver() - - def _resolve_ir_placeholders_v2( - arg, - component_spec: _structures.ComponentSpec, - arguments: dict, - ) -> str: - inputs_dict = { - input_spec.name: input_spec - for input_spec in component_spec.inputs or [] - } - if isinstance(arg, _structures.InputValuePlaceholder): - input_name = arg.input_name - input_value = arguments.get(input_name, None) - if input_value is not None: - return _input_parameter_placeholder(input_name) - else: - input_spec = inputs_dict[input_name] - if input_spec.optional: - return None - else: - raise ValueError( - 'No value provided for input {}'.format(input_name)) - - elif isinstance(arg, _structures.InputUriPlaceholder): - input_name = arg.input_name - if input_name in arguments: - input_uri = _input_artifact_uri_placeholder(input_name) - return input_uri - else: - input_spec = inputs_dict[input_name] - if input_spec.optional: - return None - else: - raise ValueError( - 'No value provided for input {}'.format(input_name)) - - elif isinstance(arg, _structures.OutputUriPlaceholder): - output_name = arg.output_name - output_uri = _output_artifact_uri_placeholder(output_name) - return output_uri - - return placeholder_resolver.resolve_placeholder( - arg=arg, - component_spec=component_spec, - arguments=arguments, - ) - - resolved_cmd = _components._resolve_command_line_and_paths( - component_spec=component_spec, - arguments=arguments, - input_path_generator=_input_artifact_path_placeholder, - output_path_generator=_resolve_output_path_placeholder, - placeholder_resolver=_resolve_ir_placeholders_v2, - ) - return resolved_cmd - - pipeline_task_spec = pipeline_spec_pb2.PipelineTaskSpec() - - # Check types of the reference arguments and serialize PipelineParams - arguments = arguments.copy() - - # Preserve input params for ContainerOp.inputs - input_params_set = set([ - param for param in arguments.values() - if isinstance(param, _pipeline_param.PipelineParam) - ]) - - for input_name, argument_value in arguments.items(): - input_type = component_spec._inputs_dict[input_name].type - argument_type = None - - if isinstance(argument_value, _pipeline_param.PipelineParam): - argument_type = argument_value.param_type - - types.verify_type_compatibility( - argument_type, input_type, - 'Incompatible argument passed to the input "{}" of component "{}": ' - .format(input_name, component_spec.name)) - - # Loop arguments defaults to 'String' type if type is unknown. - # This has to be done after the type compatiblity check. - if argument_type is None and isinstance( - argument_value, - (_for_loop.LoopArguments, _for_loop.LoopArgumentVariable)): - argument_type = 'String' - - arguments[input_name] = str(argument_value) - - if type_utils.is_parameter_type(input_type): - if argument_value.op_name: - pipeline_task_spec.inputs.parameters[ - input_name].task_output_parameter.producer_task = ( - dsl_utils.sanitize_task_name( - argument_value.op_name)) - pipeline_task_spec.inputs.parameters[ - input_name].task_output_parameter.output_parameter_key = ( - argument_value.name) - else: - pipeline_task_spec.inputs.parameters[ - input_name].component_input_parameter = argument_value.name - else: - if argument_value.op_name: - pipeline_task_spec.inputs.artifacts[ - input_name].task_output_artifact.producer_task = ( - dsl_utils.sanitize_task_name( - argument_value.op_name)) - pipeline_task_spec.inputs.artifacts[ - input_name].task_output_artifact.output_artifact_key = ( - argument_value.name) - elif isinstance(argument_value, str): - argument_type = 'String' - pipeline_params = _pipeline_param.extract_pipelineparams_from_any( - argument_value) - if pipeline_params and COMPILING_FOR_V2: - # argument_value contains PipelineParam placeholders which needs to be - # replaced. And the input needs to be added to the task spec. - for param in pipeline_params: - # Form the name for the compiler injected input, and make sure it - # doesn't collide with any existing input names. - additional_input_name = ( - dsl_component_spec - .additional_input_name_for_pipelineparam(param)) - for existing_input_name, _ in arguments.items(): - if existing_input_name == additional_input_name: - raise ValueError( - 'Name collision between existing input name ' - '{} and compiler injected input name {}'.format( - existing_input_name, additional_input_name)) - - # Add the additional param to the input params set. Otherwise, it will - # not be included when the params set is not empty. - input_params_set.add(param) - additional_input_placeholder = ( - "{{{{$.inputs.parameters['{}']}}}}".format( - additional_input_name)) - argument_value = argument_value.replace( - param.pattern, additional_input_placeholder) - - # The output references are subject to change -- the producer task may - # not be whitin the same DAG. - if param.op_name: - pipeline_task_spec.inputs.parameters[ - additional_input_name].task_output_parameter.producer_task = ( - dsl_utils.sanitize_task_name(param.op_name)) - pipeline_task_spec.inputs.parameters[ - additional_input_name].task_output_parameter.output_parameter_key = param.name - else: - pipeline_task_spec.inputs.parameters[ - additional_input_name].component_input_parameter = param.full_name - - input_type = component_spec._inputs_dict[input_name].type - if type_utils.is_parameter_type(input_type): - pipeline_task_spec.inputs.parameters[ - input_name].runtime_value.constant.string_value = argument_value - elif isinstance(argument_value, int): - argument_type = 'Integer' - pipeline_task_spec.inputs.parameters[ - input_name].runtime_value.constant.number_value = argument_value - elif isinstance(argument_value, float): - argument_type = 'Float' - pipeline_task_spec.inputs.parameters[ - input_name].runtime_value.constant.number_value = argument_value - elif isinstance(argument_value, bool): - argument_type = 'Bool' - pipeline_task_spec.inputs.parameters[ - input_name].runtime_value.constant.bool_value = argument_value - elif isinstance(argument_value, list): - argument_type = 'List' - - # Convert any PipelineParams to strings. - argument_value = map( - lambda x: str(x) - if isinstance(x, dsl.PipelineParam) else x, argument_value) - - pipeline_task_spec.inputs.parameters[ - input_name].runtime_value.constant.list_value.extend( - argument_value) - elif isinstance(argument_value, dict): - argument_type = 'Dict' - pipeline_task_spec.inputs.parameters[ - input_name].runtime_value.constant.struct_value.update( - argument_value) - elif isinstance(argument_value, _container_op.ContainerOp): - raise TypeError( - f'ContainerOp object {input_name} was passed to component as an ' - 'input argument. Pass a single output instead.') - else: - if COMPILING_FOR_V2: - raise NotImplementedError( - 'Input argument supports only the following types: ' - 'PipelineParam, str, int, float, bool, dict, and list. Got: ' - f'"{argument_value}".') - - argument_is_parameter_type = type_utils.is_parameter_type(argument_type) - input_is_parameter_type = type_utils.is_parameter_type(input_type) - if COMPILING_FOR_V2 and (argument_is_parameter_type != - input_is_parameter_type): - if isinstance(argument_value, dsl.PipelineParam): - param_or_value_msg = 'PipelineParam "{}"'.format( - argument_value.full_name) - else: - param_or_value_msg = 'value "{}"'.format(argument_value) - - raise TypeError( - 'Passing ' - '{param_or_value} with type "{arg_type}" (as "{arg_category}") to ' - 'component input ' - '"{input_name}" with type "{input_type}" (as "{input_category}") is ' - 'incompatible. Please fix the type of the component input.' - .format( - param_or_value=param_or_value_msg, - arg_type=argument_type, - arg_category='Parameter' - if argument_is_parameter_type else 'Artifact', - input_name=input_name, - input_type=input_type, - input_category='Parameter' - if input_is_parameter_type else 'Artifact', - )) - - if not component_spec.name: - component_spec.name = _components._default_component_name - - resolved_cmd = _resolve_commands_and_args_v2( - component_spec=component_spec, arguments=arguments) - - task.container_spec = ( - pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec( - image=component_spec.implementation.container.image, - command=resolved_cmd.command, - args=resolved_cmd.args, - env=[ - pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec - .EnvVar(name=name, value=value) - for name, value in task.container.env_dict.items() - ], - )) - - # TODO(chensun): dedupe IR component_spec and contaienr_spec - pipeline_task_spec.component_ref.name = ( - dsl_utils.sanitize_component_name(task.name)) - executor_label = dsl_utils.sanitize_executor_label(task.name) - - task.component_spec = dsl_component_spec.build_component_spec_from_structure( - component_spec, executor_label, arguments.keys()) - - task.task_spec = pipeline_task_spec - - # Override command and arguments if compiling to v2. - if COMPILING_FOR_V2: - task.command = resolved_cmd.command - task.arguments = resolved_cmd.args - - # limit this to v2 compiling only to avoid possible behavior change in v1. - task.inputs = list(input_params_set) diff --git a/sdk/python/kfp/deprecated/dsl/_container_op.py b/sdk/python/kfp/deprecated/dsl/_container_op.py deleted file mode 100644 index f78a850f16d..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_container_op.py +++ /dev/null @@ -1,1616 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import warnings -from typing import (Any, Callable, Dict, List, Optional, Sequence, Tuple, - TypeVar, Union) - -from kfp.deprecated._config import COMPILING_FOR_V2 -from kfp.deprecated.components import _components, _structures -from kfp.deprecated.dsl import _pipeline_param -from kfp.pipeline_spec import pipeline_spec_pb2 -from kubernetes.client import V1Affinity, V1Toleration -from kubernetes.client.models import (V1Container, V1ContainerPort, - V1EnvFromSource, V1EnvVar, V1Lifecycle, - V1Probe, V1ResourceRequirements, - V1SecurityContext, V1Volume, - V1VolumeDevice, V1VolumeMount) - -# generics -T = TypeVar('T') -# type alias: either a string or a list of string -StringOrStringList = Union[str, List[str]] -ContainerOpArgument = Union[str, int, float, bool, - _pipeline_param.PipelineParam] -ArgumentOrArguments = Union[ContainerOpArgument, List] - -ALLOWED_RETRY_POLICIES = ( - 'Always', - 'OnError', - 'OnFailure', - 'OnTransientError', -) - -# Shorthand for PipelineContainerSpec -_PipelineContainerSpec = pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec - -# Unit constants for k8s size string. -_E = 10**18 # Exa -_EI = 1 << 60 # Exa: power-of-two approximate -_P = 10**15 # Peta -_PI = 1 << 50 # Peta: power-of-two approximate -# noinspection PyShadowingBuiltins -_T = 10**12 # Tera -_TI = 1 << 40 # Tera: power-of-two approximate -_G = 10**9 # Giga -_GI = 1 << 30 # Giga: power-of-two approximate -_M = 10**6 # Mega -_MI = 1 << 20 # Mega: power-of-two approximate -_K = 10**3 # Kilo -_KI = 1 << 10 # Kilo: power-of-two approximate - -_GKE_ACCELERATOR_LABEL = 'cloud.google.com/gke-accelerator' - -_DEFAULT_CUSTOM_JOB_MACHINE_TYPE = 'n1-standard-4' - - -# util functions -def deprecation_warning(func: Callable, op_name: str, - container_name: str) -> Callable: - """Decorator function to give a pending deprecation warning.""" - - def _wrapped(*args, **kwargs): - warnings.warn( - '`dsl.ContainerOp.%s` will be removed in future releases. ' - 'Use `dsl.ContainerOp.container.%s` instead.' % - (op_name, container_name), PendingDeprecationWarning) - return func(*args, **kwargs) - - return _wrapped - - -def _create_getter_setter(prop): - """Create a tuple of getter and setter methods for a property in - `Container`.""" - - def _getter(self): - return getattr(self._container, prop) - - def _setter(self, value): - return setattr(self._container, prop, value) - - return _getter, _setter - - -def _proxy_container_op_props(cls: 'ContainerOp'): - """Takes the `ContainerOp` class and proxy the PendingDeprecation - properties in `ContainerOp` to the `Container` instance.""" - # properties mapping to proxy: ContainerOps. => Container. - prop_map = dict(image='image', env_variables='env') - # itera and create class props - for op_prop, container_prop in prop_map.items(): - # create getter and setter - _getter, _setter = _create_getter_setter(container_prop) - # decorate with deprecation warning - getter = deprecation_warning(_getter, op_prop, container_prop) - setter = deprecation_warning(_setter, op_prop, container_prop) - # update attribites with properties - setattr(cls, op_prop, property(getter, setter)) - return cls - - -def as_string_list( - list_or_str: Optional[Union[Any, Sequence[Any]]]) -> List[str]: - """Convert any value except None to a list if not already a list.""" - if list_or_str is None: - return None - if isinstance(list_or_str, Sequence) and not isinstance(list_or_str, str): - list_value = list_or_str - else: - list_value = [list_or_str] - return [str(item) for item in list_value] - - -def create_and_append(current_list: Union[List[T], None], item: T) -> List[T]: - """Create a list (if needed) and appends an item to it.""" - current_list = current_list or [] - current_list.append(item) - return current_list - - -class Container(V1Container): - """A wrapper over k8s container definition object - (io.k8s.api.core.v1.Container), which is used to represent the `container` - property in argo's workflow template - (io.argoproj.workflow.v1alpha1.Template). - - `Container` class also comes with utility functions to set and update the - the various properties for a k8s container definition. - - NOTE: A notable difference is that `name` is not required and will not be - processed for `Container` (in contrast to `V1Container` where `name` is a - required property). - - See: - * - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_container.py - * https://github.com/argoproj/argo-workflows/blob/master/api/openapi-spec/swagger.json - - Example:: - - from kfp.dsl import ContainerOp - from kubernetes.client.models import V1EnvVar - - - # creates a operation - op = ContainerOp(name='bash-ops', - image='busybox:latest', - command=['echo'], - arguments=['$MSG']) - - # returns a `Container` object from `ContainerOp` - # and add an environment variable to `Container` - op.container.add_env_variable(V1EnvVar(name='MSG', value='hello world')) - - Attributes: - attribute_map (dict): The key is attribute name and the value is json key - in definition. - """ - # remove `name` from attribute_map, swagger_types and openapi_types so `name` is not generated in the JSON - - if hasattr(V1Container, 'swagger_types'): - swagger_types = { - key: value - for key, value in V1Container.swagger_types.items() - if key != 'name' - } - if hasattr(V1Container, 'openapi_types'): - openapi_types = { - key: value - for key, value in V1Container.openapi_types.items() - if key != 'name' - } - attribute_map = { - key: value - for key, value in V1Container.attribute_map.items() - if key != 'name' - } - - def __init__(self, image: str, command: List[str], args: List[str], - **kwargs): - """Creates a new instance of `Container`. - - Args: - image {str}: image to use, e.g. busybox:latest - command {List[str]}: entrypoint array. Not executed within a shell. - args {List[str]}: arguments to entrypoint. - **kwargs: keyword arguments for `V1Container` - """ - # set name to '' if name is not provided - # k8s container MUST have a name - # argo workflow template does not need a name for container def - if not kwargs.get('name'): - kwargs['name'] = '' - - # v2 container_spec - self._container_spec = None - - self.env_dict = {} - - super(Container, self).__init__( - image=image, command=command, args=args, **kwargs) - - def _validate_size_string(self, size_string): - """Validate a given string is valid for memory/ephemeral-storage - request or limit.""" - - if isinstance(size_string, _pipeline_param.PipelineParam): - if size_string.value: - size_string = size_string.value - else: - return - - if re.match(r'^[0-9]+(E|Ei|P|Pi|T|Ti|G|Gi|M|Mi|K|Ki){0,1}$', - size_string) is None: - raise ValueError( - 'Invalid memory string. Should be an integer, or integer followed ' - 'by one of "E|Ei|P|Pi|T|Ti|G|Gi|M|Mi|K|Ki"') - - def _validate_cpu_string(self, cpu_string): - """Validate a given string is valid for cpu request or limit.""" - - if isinstance(cpu_string, _pipeline_param.PipelineParam): - if cpu_string.value: - cpu_string = cpu_string.value - else: - return - - if re.match(r'^[0-9]+m$', cpu_string) is not None: - return - - try: - float(cpu_string) - except ValueError: - raise ValueError( - 'Invalid cpu string. Should be float or integer, or integer followed ' - 'by "m".') - - def _validate_positive_number(self, str_value, param_name): - """Validate a given string is in positive integer format.""" - - if isinstance(str_value, _pipeline_param.PipelineParam): - if str_value.value: - str_value = str_value.value - else: - return - - try: - int_value = int(str_value) - except ValueError: - raise ValueError( - 'Invalid {}. Should be integer.'.format(param_name)) - - if int_value <= 0: - raise ValueError('{} must be positive integer.'.format(param_name)) - - def add_resource_limit(self, resource_name, value) -> 'Container': - """Add the resource limit of the container. - - Args: - resource_name: The name of the resource. It can be cpu, memory, etc. - value: The string value of the limit. - """ - - self.resources = self.resources or V1ResourceRequirements() - self.resources.limits = self.resources.limits or {} - self.resources.limits.update({resource_name: value}) - return self - - def add_resource_request(self, resource_name, value) -> 'Container': - """Add the resource request of the container. - - Args: - resource_name: The name of the resource. It can be cpu, memory, etc. - value: The string value of the request. - """ - - self.resources = self.resources or V1ResourceRequirements() - self.resources.requests = self.resources.requests or {} - self.resources.requests.update({resource_name: value}) - return self - - def get_resource_limit(self, resource_name: str) -> Optional[str]: - """Get the resource limit of the container. - - Args: - resource_name: The name of the resource. It can be cpu, memory, etc. - """ - - if not self.resources or not self.resources.limits: - return None - return self.resources.limits.get(resource_name) - - def get_resource_request(self, resource_name: str) -> Optional[str]: - """Get the resource request of the container. - - Args: - resource_name: The name of the resource. It can be cpu, memory, etc. - """ - - if not self.resources or not self.resources.requests: - return None - return self.resources.requests.get(resource_name) - - def set_memory_request( - self, memory: Union[str, - _pipeline_param.PipelineParam]) -> 'Container': - """Set memory request (minimum) for this operator. - - Args: - memory(Union[str, PipelineParam]): a string which can be a number or a number followed by one of - "E", "P", "T", "G", "M", "K". - """ - - if not isinstance(memory, _pipeline_param.PipelineParam): - self._validate_size_string(memory) - return self.add_resource_request('memory', memory) - - def set_memory_limit( - self, memory: Union[str, - _pipeline_param.PipelineParam]) -> 'Container': - """Set memory limit (maximum) for this operator. - - Args: - memory(Union[str, PipelineParam]): a string which can be a number or a number followed by one of - "E", "P", "T", "G", "M", "K". - """ - if not isinstance(memory, _pipeline_param.PipelineParam): - self._validate_size_string(memory) - if self._container_spec: - self._container_spec.resources.memory_limit = _get_resource_number( - memory) - return self.add_resource_limit('memory', memory) - - def set_ephemeral_storage_request(self, size) -> 'Container': - """Set ephemeral-storage request (minimum) for this operator. - - Args: - size: a string which can be a number or a number followed by one of - "E", "P", "T", "G", "M", "K". - """ - self._validate_size_string(size) - return self.add_resource_request('ephemeral-storage', size) - - def set_ephemeral_storage_limit(self, size) -> 'Container': - """Set ephemeral-storage request (maximum) for this operator. - - Args: - size: a string which can be a number or a number followed by one of - "E", "P", "T", "G", "M", "K". - """ - self._validate_size_string(size) - return self.add_resource_limit('ephemeral-storage', size) - - def set_cpu_request( - self, cpu: Union[str, - _pipeline_param.PipelineParam]) -> 'Container': - """Set cpu request (minimum) for this operator. - - Args: - cpu(Union[str, PipelineParam]): A string which can be a number or a number followed by "m", which - means 1/1000. - """ - if not isinstance(cpu, _pipeline_param.PipelineParam): - self._validate_cpu_string(cpu) - return self.add_resource_request('cpu', cpu) - - def set_cpu_limit( - self, cpu: Union[str, - _pipeline_param.PipelineParam]) -> 'Container': - """Set cpu limit (maximum) for this operator. - - Args: - cpu(Union[str, PipelineParam]): A string which can be a number or a number followed by "m", which - means 1/1000. - """ - - if not isinstance(cpu, _pipeline_param.PipelineParam): - self._validate_cpu_string(cpu) - if self._container_spec: - self._container_spec.resources.cpu_limit = _get_cpu_number(cpu) - return self.add_resource_limit('cpu', cpu) - - def set_gpu_limit( - self, - gpu: Union[str, _pipeline_param.PipelineParam], - vendor: Union[str, _pipeline_param.PipelineParam] = 'nvidia' - ) -> 'Container': - """Set gpu limit for the operator. - - This function add '.com/gpu' into resource limit. - Note that there is no need to add GPU request. GPUs are only supposed to - be specified in the limits section. See - https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/. - - Args: - gpu(Union[str, PipelineParam]): A string which must be a positive number. - vendor(Union[str, PipelineParam]): Optional. A string which is the vendor of the requested gpu. - The supported values are: 'nvidia' (default), and 'amd'. The value is - ignored in v2. - """ - - if not isinstance(gpu, _pipeline_param.PipelineParam): - self._validate_positive_number(gpu, 'gpu') - - if self._container_spec: - # For backforward compatibiliy, allow `gpu` to be a string. - self._container_spec.resources.accelerator.count = int(gpu) - - if vendor != 'nvidia' and vendor != 'amd': - raise ValueError('vendor can only be nvidia or amd.') - - return self.add_resource_limit('%s.com/gpu' % vendor, gpu) - - return self.add_resource_limit(vendor, gpu) - - def add_volume_mount(self, volume_mount) -> 'Container': - """Add volume to the container. - - Args: - volume_mount: Kubernetes volume mount For detailed spec, check volume - mount definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_volume_mount.py - """ - - if not isinstance(volume_mount, V1VolumeMount): - raise ValueError( - 'invalid argument. Must be of instance `V1VolumeMount`.') - - self.volume_mounts = create_and_append(self.volume_mounts, volume_mount) - return self - - def add_volume_devices(self, volume_device) -> 'Container': - """Add a block device to be used by the container. - - Args: - volume_device: Kubernetes volume device For detailed spec, volume - device definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_volume_device.py - """ - - if not isinstance(volume_device, V1VolumeDevice): - raise ValueError( - 'invalid argument. Must be of instance `V1VolumeDevice`.') - - self.volume_devices = create_and_append(self.volume_devices, - volume_device) - return self - - def set_env_variable(self, name: str, value: str) -> 'Container': - """Sets environment variable to the container (v2 only). - - Args: - name: The name of the environment variable. - value: The value of the environment variable. - """ - - if not COMPILING_FOR_V2: - raise ValueError( - 'set_env_variable is v2 only. Use add_env_variable for v1.') - - # Merge with any existing environment varaibles - self.env_dict = { - env.name: env.value for env in self._container_spec.env or [] - } - self.env_dict[name] = value - - del self._container_spec.env[:] - self._container_spec.env.extend([ - _PipelineContainerSpec.EnvVar(name=name, value=value) - for name, value in self.env_dict.items() - ]) - return self - - def add_env_variable(self, env_variable) -> 'Container': - """Add environment variable to the container. - - Args: - env_variable: Kubernetes environment variable For detailed spec, check - environment variable definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_env_var.py - """ - - if not isinstance(env_variable, V1EnvVar): - raise ValueError( - 'invalid argument. Must be of instance `V1EnvVar`.') - - self.env = create_and_append(self.env, env_variable) - return self - - def add_env_from(self, env_from) -> 'Container': - """Add a source to populate environment variables int the container. - - Args: - env_from: Kubernetes environment from source For detailed spec, check - environment from source definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_env_var_source.py - """ - - if not isinstance(env_from, V1EnvFromSource): - raise ValueError( - 'invalid argument. Must be of instance `V1EnvFromSource`.') - - self.env_from = create_and_append(self.env_from, env_from) - return self - - def set_image_pull_policy(self, image_pull_policy) -> 'Container': - """Set image pull policy for the container. - - Args: - image_pull_policy: One of `Always`, `Never`, `IfNotPresent`. - """ - if image_pull_policy not in ['Always', 'Never', 'IfNotPresent']: - raise ValueError( - 'Invalid imagePullPolicy. Must be one of `Always`, `Never`, `IfNotPresent`.' - ) - - self.image_pull_policy = image_pull_policy - return self - - def add_port(self, container_port) -> 'Container': - """Add a container port to the container. - - Args: - container_port: Kubernetes container port For detailed spec, check - container port definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_container_port.py - """ - - if not isinstance(container_port, V1ContainerPort): - raise ValueError( - 'invalid argument. Must be of instance `V1ContainerPort`.') - - self.ports = create_and_append(self.ports, container_port) - return self - - def set_security_context(self, security_context) -> 'Container': - """Set security configuration to be applied on the container. - - Args: - security_context: Kubernetes security context For detailed spec, check - security context definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_security_context.py - """ - - if not isinstance(security_context, V1SecurityContext): - raise ValueError( - 'invalid argument. Must be of instance `V1SecurityContext`.') - - self.security_context = security_context - return self - - def set_stdin(self, stdin=True) -> 'Container': - """Whether this container should allocate a buffer for stdin in the - container runtime. If this is not set, reads from stdin in the - container will always result in EOF. - - Args: - stdin: boolean flag - """ - - self.stdin = stdin - return self - - def set_stdin_once(self, stdin_once=True) -> 'Container': - """Whether the container runtime should close the stdin channel after - it has been opened by a single attach. When stdin is true the stdin - stream will remain open across multiple attach sessions. If stdinOnce - is set to true, stdin is opened on container start, is empty until the - first client attaches to stdin, and then remains open and accepts data - until the client disconnects, at which time stdin is closed and remains - closed until the container is restarted. If this flag is false, a - container processes that reads from stdin will never receive an EOF. - - Args: - stdin_once: boolean flag - """ - - self.stdin_once = stdin_once - return self - - def set_termination_message_path(self, - termination_message_path) -> 'Container': - """Path at which the file to which the container's termination message - will be written is mounted into the container's filesystem. Message - written is intended to be brief final status, such as an assertion - failure message. Will be truncated by the node if greater than 4096 - bytes. The total message length across all containers will be limited - to 12kb. - - Args: - termination_message_path: path for the termination message - """ - self.termination_message_path = termination_message_path - return self - - def set_termination_message_policy( - self, termination_message_policy) -> 'Container': - """Indicate how the termination message should be populated. File will - use the contents of terminationMessagePath to populate the container - status message on both success and failure. FallbackToLogsOnError will - use the last chunk of container log output if the termination message - file is empty and the container exited with an error. The log output is - limited to 2048 bytes or 80 lines, whichever is smaller. - - Args: - termination_message_policy: `File` or `FallbackToLogsOnError` - """ - if termination_message_policy not in ['File', 'FallbackToLogsOnError']: - raise ValueError( - 'terminationMessagePolicy must be `File` or `FallbackToLogsOnError`' - ) - self.termination_message_policy = termination_message_policy - return self - - def set_tty(self, tty: bool = True) -> 'Container': - """Whether this container should allocate a TTY for itself, also - requires 'stdin' to be true. - - Args: - tty: boolean flag - """ - - self.tty = tty - return self - - def set_readiness_probe(self, readiness_probe) -> 'Container': - """Set a readiness probe for the container. - - Args: - readiness_probe: Kubernetes readiness probe For detailed spec, check - probe definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_probe.py - """ - - if not isinstance(readiness_probe, V1Probe): - raise ValueError('invalid argument. Must be of instance `V1Probe`.') - - self.readiness_probe = readiness_probe - return self - - def set_liveness_probe(self, liveness_probe) -> 'Container': - """Set a liveness probe for the container. - - Args: - liveness_probe: Kubernetes liveness probe For detailed spec, check - probe definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_probe.py - """ - - if not isinstance(liveness_probe, V1Probe): - raise ValueError('invalid argument. Must be of instance `V1Probe`.') - - self.liveness_probe = liveness_probe - return self - - def set_lifecycle(self, lifecycle) -> 'Container': - """Setup a lifecycle config for the container. - - Args: - lifecycle: Kubernetes lifecycle For detailed spec, lifecycle - definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_lifecycle.py - """ - - if not isinstance(lifecycle, V1Lifecycle): - raise ValueError( - 'invalid argument. Must be of instance `V1Lifecycle`.') - - self.lifecycle = lifecycle - return self - - -class UserContainer(Container): - """Represents an argo workflow UserContainer - (io.argoproj.workflow.v1alpha1.UserContainer) to be used in `UserContainer` - property in argo's workflow template - (io.argoproj.workflow.v1alpha1.Template). - - `UserContainer` inherits from `Container` class with an addition of - `mirror_volume_mounts` - attribute (`mirrorVolumeMounts` property). - - See - https://github.com/argoproj/argo-workflows/blob/master/api/openapi-spec/swagger.json - - Args: - name: unique name for the user container - image: image to use for the user container, e.g. redis:alpine - command: entrypoint array. Not executed within a shell. - args: arguments to the entrypoint. - mirror_volume_mounts: MirrorVolumeMounts will mount the same volumes - specified in the main container to the container (including - artifacts), at the same mountPaths. This enables dind daemon to - partially see the same filesystem as the main container in order to - use features such as docker volume binding - **kwargs: keyword arguments available for `Container` - - Attributes: - swagger_types (dict): The key is attribute name and the value is attribute - type. - - Example :: - - from kfp.dsl import ContainerOp, UserContainer - # creates a `ContainerOp` and adds a redis init container - op = (ContainerOp(name='foo-op', image='busybox:latest') - .add_initContainer(UserContainer(name='redis', image='redis:alpine'))) - """ - # adds `mirror_volume_mounts` to `UserContainer` swagger definition - # NOTE inherits definition from `V1Container` rather than `Container` - # because `Container` has no `name` property. - if hasattr(V1Container, 'swagger_types'): - swagger_types = dict( - **V1Container.swagger_types, mirror_volume_mounts='bool') - if hasattr(V1Container, 'openapi_types'): - openapi_types = dict( - **V1Container.openapi_types, mirror_volume_mounts='bool') - attribute_map = dict( - **V1Container.attribute_map, mirror_volume_mounts='mirrorVolumeMounts') - - def __init__(self, - name: str, - image: str, - command: StringOrStringList = None, - args: StringOrStringList = None, - mirror_volume_mounts: bool = None, - **kwargs): - super().__init__( - name=name, - image=image, - command=as_string_list(command), - args=as_string_list(args), - **kwargs) - - self.mirror_volume_mounts = mirror_volume_mounts - - def set_mirror_volume_mounts(self, mirror_volume_mounts=True): - """Setting mirrorVolumeMounts to true will mount the same volumes - specified in the main container to the container (including artifacts), - at the same mountPaths. This enables dind daemon to partially see the - same filesystem as the main container in order to use features such as - docker volume binding. - - Args: - mirror_volume_mounts: boolean flag - """ - - self.mirror_volume_mounts = mirror_volume_mounts - return self - - @property - def inputs(self): - """A list of PipelineParam found in the UserContainer object.""" - return _pipeline_param.extract_pipelineparams_from_any(self) - - -class Sidecar(UserContainer): - """Creates a new instance of `Sidecar`. - - Args: - name: unique name for the sidecar container - image: image to use for the sidecar container, e.g. redis:alpine - command: entrypoint array. Not executed within a shell. - args: arguments to the entrypoint. - mirror_volume_mounts: MirrorVolumeMounts will mount the same volumes - specified in the main container to the sidecar (including artifacts), - at the same mountPaths. This enables dind daemon to partially see the - same filesystem as the main container in order to use features such as - docker volume binding - **kwargs: keyword arguments available for `Container` - """ - - def __init__(self, - name: str, - image: str, - command: StringOrStringList = None, - args: StringOrStringList = None, - mirror_volume_mounts: bool = None, - **kwargs): - super().__init__( - name=name, - image=image, - command=command, - args=args, - mirror_volume_mounts=mirror_volume_mounts, - **kwargs) - - -def _make_hash_based_id_for_op(op): - # Generating a unique ID for Op. For class instances, the hash is the object's memory address which is unique. - return op.human_name + ' ' + hex(2**63 + hash(op))[2:] - - -# Pointer to a function that generates a unique ID for the Op instance (Possibly by registering the Op instance in some system). -_register_op_handler = _make_hash_based_id_for_op - - -class BaseOp(object): - """Base operator. - - Args: - name: the name of the op. It does not have to be unique within a - pipeline because the pipeline will generates a unique new name in case - of conflicts. - init_containers: the list of `UserContainer` objects describing the - InitContainer to deploy before the `main` container. - sidecars: the list of `Sidecar` objects describing the sidecar - containers to deploy together with the `main` container. - is_exit_handler: Deprecated. - """ - - # list of attributes that might have pipeline params - used to generate - # the input parameters during compilation. - # Excludes `file_outputs` and `outputs` as they are handled separately - # in the compilation process to generate the DAGs and task io parameters. - attrs_with_pipelineparams = [ - 'node_selector', 'volumes', 'pod_annotations', 'pod_labels', - 'num_retries', 'init_containers', 'sidecars', 'tolerations' - ] - - def __init__(self, - name: str, - init_containers: List[UserContainer] = None, - sidecars: List[Sidecar] = None, - is_exit_handler: bool = False): - - if is_exit_handler: - warnings.warn('is_exit_handler=True is no longer needed.', - DeprecationWarning) - - self.is_exit_handler = is_exit_handler - - # human_name must exist to construct operator's name - self.human_name = name - self.display_name = None #TODO Set display_name to human_name - # ID of the current Op. Ideally, it should be generated by the compiler that sees the bigger context. - # However, the ID is used in the task output references (PipelineParams) which can be serialized to strings. - # Because of this we must obtain a unique ID right now. - self.name = _register_op_handler(self) - - # TODO: proper k8s definitions so that `convert_k8s_obj_to_json` can be used? - # `io.argoproj.workflow.v1alpha1.Template` properties - self.node_selector = {} - self.volumes = [] - self.tolerations = [] - self.affinity = {} - self.pod_annotations = {} - self.pod_labels = {} - - # Retry strategy - self.num_retries = 0 - self.retry_policy = None - self.backoff_factor = None - self.backoff_duration = None - self.backoff_max_duration = None - - self.timeout = 0 - self.init_containers = init_containers or [] - self.sidecars = sidecars or [] - - # used to mark this op with loop arguments - self.loop_args = None - - # Placeholder for inputs when adding ComponentSpec metadata to this - # ContainerOp. This holds inputs defined in ComponentSpec that have - # a corresponding PipelineParam. - self._component_spec_inputs_with_pipeline_params = [] - - # attributes specific to `BaseOp` - self._inputs = [] - self.dependent_names = [] - - # Caching option, default to True - self.enable_caching = True - - @property - def inputs(self): - """List of PipelineParams that will be converted into input parameters - (io.argoproj.workflow.v1alpha1.Inputs) for the argo workflow.""" - # Iterate through and extract all the `PipelineParam` in Op when - # called the 1st time (because there are in-place updates to `PipelineParam` - # during compilation - remove in-place updates for easier debugging?) - if not self._inputs: - self._inputs = self._component_spec_inputs_with_pipeline_params or [] - # TODO replace with proper k8s obj? - for key in self.attrs_with_pipelineparams: - self._inputs += _pipeline_param.extract_pipelineparams_from_any( - getattr(self, key)) - # keep only unique - self._inputs = list(set(self._inputs)) - return self._inputs - - @inputs.setter - def inputs(self, value): - # to support in-place updates - self._inputs = value - - def apply(self, mod_func): - """Applies a modifier function to self. - - The function should return the passed object. - This is needed to chain "extention methods" to this class. - - Example:: - - from kfp.gcp import use_gcp_secret - task = ( - train_op(...) - .set_memory_request('1G') - .apply(use_gcp_secret('user-gcp-sa')) - .set_memory_limit('2G') - ) - """ - return mod_func(self) or self - - def after(self, *ops): - """Specify explicit dependency on other ops.""" - for op in ops: - self.dependent_names.append(op.name) - return self - - def add_volume(self, volume): - """Add K8s volume to the container. - - Args: - volume: Kubernetes volumes For detailed spec, check volume definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_volume.py - """ - self.volumes.append(volume) - return self - - def add_toleration(self, tolerations: V1Toleration): - """Add K8s tolerations. - - Args: - tolerations: Kubernetes toleration For detailed spec, check toleration - definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_toleration.py - """ - self.tolerations.append(tolerations) - return self - - def add_affinity(self, affinity: V1Affinity): - """Add K8s Affinity. - - Args: - affinity: Kubernetes affinity For detailed spec, check affinity - definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/client/models/v1_affinity.py - - Example:: - - V1Affinity( - node_affinity=V1NodeAffinity( - required_during_scheduling_ignored_during_execution=V1NodeSelector( - node_selector_terms=[V1NodeSelectorTerm( - match_expressions=[V1NodeSelectorRequirement( - key='beta.kubernetes.io/instance-type', - operator='In', - values=['p2.xlarge'])])]))) - """ - self.affinity = affinity - return self - - def add_node_selector_constraint( - self, label_name: Union[str, _pipeline_param.PipelineParam], - value: Union[str, _pipeline_param.PipelineParam]): - """Add a constraint for nodeSelector. - - Each constraint is a key-value pair label. - For the container to be eligible to run on a node, the node must have each - of the constraints appeared as labels. - - Args: - label_name(Union[str, PipelineParam]): The name of the constraint label. - value(Union[str, PipelineParam]): The value of the constraint label. - """ - - self.node_selector[label_name] = value - return self - - def add_pod_annotation(self, name: str, value: str): - """Adds a pod's metadata annotation. - - Args: - name: The name of the annotation. - value: The value of the annotation. - """ - - self.pod_annotations[name] = value - return self - - def add_pod_label(self, name: str, value: str): - """Adds a pod's metadata label. - - Args: - name: The name of the label. - value: The value of the label. - """ - - self.pod_labels[name] = value - return self - - def set_retry(self, - num_retries: int, - policy: Optional[str] = None, - backoff_duration: Optional[str] = None, - backoff_factor: Optional[float] = None, - backoff_max_duration: Optional[str] = None): - """Sets the number of times the task is retried until it's declared - failed. - - Args: - num_retries: Number of times to retry on failures. - policy: Retry policy name. - backoff_duration: The time interval between retries. Defaults to an - immediate retry. In case you specify a simple number, the unit - defaults to seconds. You can also specify a different unit, for - instance, 2m (2 minutes), 1h (1 hour). - backoff_factor: The exponential backoff factor applied to - backoff_duration. For example, if backoff_duration="60" - (60 seconds) and backoff_factor=2, the first retry will happen - after 60 seconds, then after 120, 240, and so on. - backoff_max_duration: The maximum interval that can be reached with - the backoff strategy. - """ - if policy is not None and policy not in ALLOWED_RETRY_POLICIES: - raise ValueError('policy must be one of: %r' % - (ALLOWED_RETRY_POLICIES,)) - - self.num_retries = num_retries - self.retry_policy = policy - self.backoff_factor = backoff_factor - self.backoff_duration = backoff_duration - self.backoff_max_duration = backoff_max_duration - return self - - def set_timeout(self, seconds: int): - """Sets the timeout for the task in seconds. - - Args: - seconds: Number of seconds. - """ - - self.timeout = seconds - return self - - def add_init_container(self, init_container: UserContainer): - """Add a init container to the Op. - - Args: - init_container: UserContainer object. - """ - - self.init_containers.append(init_container) - return self - - def add_sidecar(self, sidecar: Sidecar): - """Add a sidecar to the Op. - - Args: - sidecar: SideCar object. - """ - - self.sidecars.append(sidecar) - return self - - def set_display_name(self, name: str): - self.display_name = name - return self - - def set_caching_options(self, enable_caching: bool) -> 'BaseOp': - """Sets caching options for the Op. - - Args: - enable_caching: Whether or not to enable caching for this task. - - Returns: - Self return to allow chained setting calls. - """ - self.enable_caching = enable_caching - return self - - def __repr__(self): - return str({self.__class__.__name__: self.__dict__}) - - -from ._pipeline_volume import \ - PipelineVolume # The import is here to prevent circular reference problems. - - -class InputArgumentPath: - - def __init__(self, argument, input=None, path=None): - self.argument = argument - self.input = input - self.path = path - - -def _is_legacy_output_name(output_name: str) -> Tuple[bool, str]: - normalized_output_name = re.sub('[^a-zA-Z0-9]', '-', output_name.lower()) - if normalized_output_name in [ - 'mlpipeline-ui-metadata', - 'mlpipeline-metrics', - ]: - return True, normalized_output_name - return False, output_name - - -class ContainerOp(BaseOp): - """Represents an op implemented by a container image. - - Args: - name: the name of the op. It does not have to be unique within a - pipeline because the pipeline will generates a unique new name in case - of conflicts. - image: the container image name, such as 'python:3.5-jessie' - command: the command to run in the container. If None, uses default CMD - in defined in container. - arguments: the arguments of the command. The command can include "%s" - and supply a PipelineParam as the string replacement. For example, - ('echo %s' % input_param). At container run time the argument will be - 'echo param_value'. - init_containers: the list of `UserContainer` objects describing the - InitContainer to deploy before the `main` container. - sidecars: the list of `Sidecar` objects describing the sidecar - containers to deploy together with the `main` container. - container_kwargs: the dict of additional keyword arguments to pass to - the op's `Container` definition. - artifact_argument_paths: Optional. Maps input artifact arguments (values - or references) to the local file paths where they'll be placed. At - pipeline run time, the value of the artifact argument is saved to a - local file with specified path. This parameter is only needed when the - input file paths are hard-coded in the program. Otherwise it's better - to pass input artifact placement paths by including artifact arguments - in the command-line using the InputArgumentPath class instances. - file_outputs: Maps output names to container local output file paths. - The system will take the data from those files and will make it - available for passing to downstream tasks. For each output in the - file_outputs map there will be a corresponding output reference - available in the task.outputs dictionary. These output references can - be passed to the other tasks as arguments. The following output names - are handled specially by the frontend and - backend: "mlpipeline-ui-metadata" and "mlpipeline-metrics". - output_artifact_paths: Deprecated. Maps output artifact labels to local - artifact file paths. Deprecated: Use file_outputs instead. It now - supports big data outputs. - is_exit_handler: Deprecated. This is no longer needed. - pvolumes: Dictionary for the user to match a path on the op's fs with a - V1Volume or it inherited type. - E.g {"/my/path": vol, "/mnt": other_op.pvolumes["/output"]}. - - Example:: - - from kfp import dsl - from kubernetes.client.models import V1EnvVar, V1SecretKeySelector - @dsl.pipeline( - name='foo', - description='hello world') - def foo_pipeline(tag: str, pull_image_policy: str): - # any attributes can be parameterized (both serialized string or actual PipelineParam) - op = dsl.ContainerOp(name='foo', - image='busybox:%s' % tag, - # pass in init_container list - init_containers=[dsl.UserContainer('print', 'busybox:latest', command='echo "hello"')], - # pass in sidecars list - sidecars=[dsl.Sidecar('print', 'busybox:latest', command='echo "hello"')], - # pass in k8s container kwargs - container_kwargs={'env': [V1EnvVar('foo', 'bar')]}, - ) - # set `imagePullPolicy` property for `container` with `PipelineParam` - op.container.set_image_pull_policy(pull_image_policy) - # add sidecar with parameterized image tag - # sidecar follows the argo sidecar swagger spec - op.add_sidecar(dsl.Sidecar('redis', 'redis:%s' % tag).set_image_pull_policy('Always')) - """ - - # list of attributes that might have pipeline params - used to generate - # the input parameters during compilation. - # Excludes `file_outputs` and `outputs` as they are handled separately - # in the compilation process to generate the DAGs and task io parameters. - - _DISABLE_REUSABLE_COMPONENT_WARNING = False - - def __init__( - self, - name: str, - image: str, - command: Optional[StringOrStringList] = None, - arguments: Optional[ArgumentOrArguments] = None, - init_containers: Optional[List[UserContainer]] = None, - sidecars: Optional[List[Sidecar]] = None, - container_kwargs: Optional[Dict] = None, - artifact_argument_paths: Optional[List[InputArgumentPath]] = None, - file_outputs: Optional[Dict[str, str]] = None, - output_artifact_paths: Optional[Dict[str, str]] = None, - is_exit_handler: bool = False, - pvolumes: Optional[Dict[str, V1Volume]] = None, - ): - super().__init__( - name=name, - init_containers=init_containers, - sidecars=sidecars, - is_exit_handler=is_exit_handler) - - self.attrs_with_pipelineparams = BaseOp.attrs_with_pipelineparams + [ - '_container', 'artifact_arguments', '_parameter_arguments' - ] #Copying the BaseOp class variable! - - input_artifact_paths = {} - artifact_arguments = {} - file_outputs = dict(file_outputs or {}) # Making a copy - output_artifact_paths = dict(output_artifact_paths or - {}) # Making a copy - - self._is_v2 = False - - def resolve_artifact_argument(artarg): - from ..components._components import _generate_input_file_name - if not isinstance(artarg, InputArgumentPath): - return artarg - input_name = getattr( - artarg.input, 'name', - artarg.input) or ('input-' + str(len(artifact_arguments))) - input_path = artarg.path or _generate_input_file_name(input_name) - input_artifact_paths[input_name] = input_path - artifact_arguments[input_name] = str(artarg.argument) - return input_path - - for artarg in artifact_argument_paths or []: - resolve_artifact_argument(artarg) - - if isinstance(command, Sequence) and not isinstance(command, str): - command = list(map(resolve_artifact_argument, command)) - if isinstance(arguments, Sequence) and not isinstance(arguments, str): - arguments = list(map(resolve_artifact_argument, arguments)) - - # convert to list if not a list - command = as_string_list(command) - arguments = as_string_list(arguments) - - if (not ContainerOp._DISABLE_REUSABLE_COMPONENT_WARNING) and ( - '--component_launcher_class_path' not in (arguments or [])): - # The warning is suppressed for pipelines created using the TFX SDK. - warnings.warn( - 'Please create reusable components instead of constructing ContainerOp instances directly.' - ' Reusable components are shareable, portable and have compatibility and support guarantees.' - ' Please see the documentation: https://www.kubeflow.org/docs/pipelines/sdk/component-development/#writing-your-component-definition-file' - ' The components can be created manually (or, in case of python, using kfp.components.create_component_from_func or func_to_container_op)' - ' and then loaded using kfp.components.load_component_from_file, load_component_from_uri or load_component_from_text: ' - 'https://kubeflow-pipelines.readthedocs.io/en/stable/source/kfp.components.html#kfp.components.load_component_from_file', - category=FutureWarning, - ) - if COMPILING_FOR_V2: - raise RuntimeError( - 'Constructing ContainerOp instances directly is deprecated and not ' - 'supported when compiling to v2 (using v2 compiler or v1 compiler ' - 'with V2_COMPATIBLE or V2_ENGINE mode).') - - # `container` prop in `io.argoproj.workflow.v1alpha1.Template` - container_kwargs = container_kwargs or {} - self._container = Container( - image=image, args=arguments, command=command, **container_kwargs) - - # NOTE for backward compatibility (remove in future?) - # proxy old ContainerOp callables to Container - - # attributes to NOT proxy - ignore_set = frozenset(['to_dict', 'to_str']) - - # decorator func to proxy a method in `Container` into `ContainerOp` - def _proxy(proxy_attr): - """Decorator func to proxy to ContainerOp.container.""" - - def _decorated(*args, **kwargs): - # execute method - ret = getattr(self._container, proxy_attr)(*args, **kwargs) - if ret == self._container: - return self - return ret - - return deprecation_warning(_decorated, proxy_attr, proxy_attr) - - # iter thru container and attach a proxy func to the container method - for attr_to_proxy in dir(self._container): - func = getattr(self._container, attr_to_proxy) - # ignore private methods, and bypass method overrided by subclasses. - if (not hasattr(self, attr_to_proxy) and - hasattr(func, '__call__') and (attr_to_proxy[0] != '_') and - (attr_to_proxy not in ignore_set)): - # only proxy public callables - setattr(self, attr_to_proxy, _proxy(attr_to_proxy)) - - if output_artifact_paths: - warnings.warn( - 'The output_artifact_paths parameter is deprecated since SDK v0.1.32. ' - 'Use the file_outputs parameter instead. file_outputs now supports ' - 'outputting big data.', DeprecationWarning) - - # Skip the special handling that is unnecessary in v2. - if not COMPILING_FOR_V2: - # Special handling for the mlpipeline-ui-metadata and mlpipeline-metrics - # outputs that should always be saved as artifacts - # TODO: Remove when outputs are always saved as artifacts - for output_name, path in dict(file_outputs).items(): - is_legacy_name, normalized_name = _is_legacy_output_name( - output_name) - if is_legacy_name: - output_artifact_paths[normalized_name] = path - del file_outputs[output_name] - - # attributes specific to `ContainerOp` - self.input_artifact_paths = input_artifact_paths - self.artifact_arguments = artifact_arguments - self.file_outputs = file_outputs - self.output_artifact_paths = output_artifact_paths or {} - - self._metadata = None - self._parameter_arguments = None - - self.execution_options = _structures.ExecutionOptionsSpec( - caching_strategy=_structures.CachingStrategySpec(),) - - self.outputs = {} - if file_outputs: - self.outputs = { - name: _pipeline_param.PipelineParam(name, op_name=self.name) - for name in file_outputs.keys() - } - - self._set_single_output_attribute() - - self.pvolumes = {} - self.add_pvolumes(pvolumes) - - def _set_single_output_attribute(self): - # Syntactic sugar: Add task.output attribute if the component has a single - # output. - # TODO: Currently the "MLPipeline UI Metadata" output is removed from - # outputs to preserve backwards compatibility. Maybe stop excluding it from - # outputs, but rather exclude it from unique_outputs. - unique_outputs = set(self.outputs.values()) - if len(unique_outputs) == 1: - self.output = list(unique_outputs)[0] - else: - self.output = _MultipleOutputsError() - - @property - def is_v2(self): - return self._is_v2 - - @is_v2.setter - def is_v2(self, is_v2: bool): - self._is_v2 = is_v2 - - # v2 container spec - @property - def container_spec(self): - return self._container._container_spec - - @container_spec.setter - def container_spec(self, spec: _PipelineContainerSpec): - if not isinstance(spec, _PipelineContainerSpec): - raise TypeError('container_spec can only be PipelineContainerSpec. ' - 'Got: {}'.format(spec)) - self._container._container_spec = spec - - @property - def command(self): - return self._container.command - - @command.setter - def command(self, value): - self._container.command = as_string_list(value) - - @property - def arguments(self): - return self._container.args - - @arguments.setter - def arguments(self, value): - self._container.args = as_string_list(value) - - @property - def container(self): - """`Container` object that represents the `container` property in - `io.argoproj.workflow.v1alpha1.Template`. Can be used to update the - container configurations. - - Example:: - - import kfp.dsl as dsl - from kubernetes.client.models import V1EnvVar - - @dsl.pipeline(name='example_pipeline') - def immediate_value_pipeline(): - op1 = (dsl.ContainerOp(name='example', image='nginx:alpine') - .container - .add_env_variable(V1EnvVar(name='HOST', - value='foo.bar')) - .add_env_variable(V1EnvVar(name='PORT', value='80')) - .parent # return the parent `ContainerOp` - ) - """ - return self._container - - def _set_metadata(self, - metadata, - arguments: Optional[Dict[str, Any]] = None): - """Passes the ContainerOp the metadata information and configures the - right output. - - Args: - metadata (ComponentSpec): component metadata - arguments: Dictionary of input arguments to the component. - """ - if not isinstance(metadata, _structures.ComponentSpec): - raise ValueError('_set_metadata is expecting ComponentSpec.') - - self._metadata = metadata - - if self._metadata.outputs: - declared_outputs = { - output.name: - _pipeline_param.PipelineParam(output.name, op_name=self.name) - for output in self._metadata.outputs - } - self.outputs.update(declared_outputs) - - for output in self._metadata.outputs: - if output.name in self.file_outputs: - continue - is_legacy_name, normalized_name = _is_legacy_output_name( - output.name) - if is_legacy_name and normalized_name in self.output_artifact_paths: - output_filename = self.output_artifact_paths[ - normalized_name] - else: - output_filename = _components._generate_output_file_name( - output.name) - self.file_outputs[output.name] = output_filename - - if not COMPILING_FOR_V2: - for output_name, path in dict(self.file_outputs).items(): - is_legacy_name, normalized_name = _is_legacy_output_name( - output_name) - if is_legacy_name: - self.output_artifact_paths[normalized_name] = path - del self.file_outputs[output_name] - del self.outputs[output_name] - - if arguments is not None: - for input_name, value in arguments.items(): - self.artifact_arguments[input_name] = str(value) - if (isinstance(value, _pipeline_param.PipelineParam)): - self._component_spec_inputs_with_pipeline_params.append( - value) - - if input_name not in self.input_artifact_paths: - input_artifact_path = _components._generate_input_file_name( - input_name) - self.input_artifact_paths[input_name] = input_artifact_path - - if self.file_outputs: - for output in self.file_outputs.keys(): - output_type = self.outputs[output].param_type - for output_meta in self._metadata.outputs: - if output_meta.name == output: - output_type = output_meta.type - self.outputs[output].param_type = output_type - - self._set_single_output_attribute() - - def add_pvolumes(self, pvolumes: Dict[str, V1Volume] = None): - """Updates the existing pvolumes dict, extends volumes and - volume_mounts and redefines the pvolume attribute. - - Args: - pvolumes: Dictionary. Keys are mount paths, values are Kubernetes - volumes or inherited types (e.g. PipelineVolumes). - """ - if pvolumes: - for mount_path, pvolume in pvolumes.items(): - if hasattr(pvolume, 'dependent_names'): - self.dependent_names.extend(pvolume.dependent_names) - else: - pvolume = PipelineVolume(volume=pvolume) - pvolume = pvolume.after(self) - self.pvolumes[mount_path] = pvolume - self.add_volume(pvolume) - self._container.add_volume_mount( - V1VolumeMount(name=pvolume.name, mount_path=mount_path)) - - self.pvolume = None - if len(self.pvolumes) == 1: - self.pvolume = list(self.pvolumes.values())[0] - return self - - def add_node_selector_constraint( - self, label_name: Union[str, _pipeline_param.PipelineParam], - value: Union[str, _pipeline_param.PipelineParam]) -> 'ContainerOp': - """Sets accelerator type requirement for this task. - - When compiling for v2, this function can be optionally used with - set_gpu_limit to set the number of accelerator required. Otherwise, by - default the number requested will be 1. - - Args: - label_name: The name of the constraint label. - For v2, only 'cloud.google.com/gke-accelerator' is supported now. - value: The name of the accelerator. - For v2, available values include 'nvidia-tesla-k80', 'tpu-v3'. - - Returns: - self return to allow chained call with other resource specification. - """ - if self.container_spec and not ( - isinstance(label_name, _pipeline_param.PipelineParam) or - isinstance(value, _pipeline_param.PipelineParam)): - accelerator_cnt = 1 - if self.container_spec.resources.accelerator.count > 1: - # Reserve the number if already set. - accelerator_cnt = self.container_spec.resources.accelerator.count - - accelerator_config = _PipelineContainerSpec.ResourceSpec.AcceleratorConfig( - type=_sanitize_gpu_type(value), count=accelerator_cnt) - self.container_spec.resources.accelerator.CopyFrom( - accelerator_config) - - super(ContainerOp, self).add_node_selector_constraint(label_name, value) - return self - - -# proxy old ContainerOp properties to ContainerOp.container -# with PendingDeprecationWarning. -ContainerOp = _proxy_container_op_props(ContainerOp) - - -class _MultipleOutputsError: - - @staticmethod - def raise_error(): - raise RuntimeError( - 'This task has multiple outputs. Use `task.outputs[]` ' - 'dictionary to refer to the one you need.') - - def __getattribute__(self, name): - _MultipleOutputsError.raise_error() - - def __str__(self): - _MultipleOutputsError.raise_error() - - -def _get_cpu_number(cpu_string: str) -> float: - """Converts the cpu string to number of vCPU core.""" - # dsl.ContainerOp._validate_cpu_string guaranteed that cpu_string is either - # 1) a string can be converted to a float; or - # 2) a string followed by 'm', and it can be converted to a float. - if cpu_string.endswith('m'): - return float(cpu_string[:-1]) / 1000 - else: - return float(cpu_string) - - -def _get_resource_number(resource_string: str) -> float: - """Converts the resource string to number of resource in GB.""" - # dsl.ContainerOp._validate_size_string guaranteed that memory_string - # represents an integer, optionally followed by one of (E, Ei, P, Pi, T, Ti, - # G, Gi, M, Mi, K, Ki). - # See the meaning of different suffix at - # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory - # Also, ResourceSpec in pipeline IR expects a number in GB. - if resource_string.endswith('E'): - return float(resource_string[:-1]) * _E / _G - elif resource_string.endswith('Ei'): - return float(resource_string[:-2]) * _EI / _G - elif resource_string.endswith('P'): - return float(resource_string[:-1]) * _P / _G - elif resource_string.endswith('Pi'): - return float(resource_string[:-2]) * _PI / _G - elif resource_string.endswith('T'): - return float(resource_string[:-1]) * _T / _G - elif resource_string.endswith('Ti'): - return float(resource_string[:-2]) * _TI / _G - elif resource_string.endswith('G'): - return float(resource_string[:-1]) - elif resource_string.endswith('Gi'): - return float(resource_string[:-2]) * _GI / _G - elif resource_string.endswith('M'): - return float(resource_string[:-1]) * _M / _G - elif resource_string.endswith('Mi'): - return float(resource_string[:-2]) * _MI / _G - elif resource_string.endswith('K'): - return float(resource_string[:-1]) * _K / _G - elif resource_string.endswith('Ki'): - return float(resource_string[:-2]) * _KI / _G - else: - # By default interpret as a plain integer, in the unit of Bytes. - return float(resource_string) / _G - - -def _sanitize_gpu_type(gpu_type: str) -> str: - """Converts the GPU type to conform the enum style.""" - return gpu_type.replace('-', '_').upper() diff --git a/sdk/python/kfp/deprecated/dsl/_container_op_test.py b/sdk/python/kfp/deprecated/dsl/_container_op_test.py deleted file mode 100644 index 00f2e9d8803..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_container_op_test.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for kfp.dsl.container_op.""" -import unittest - -from kfp.deprecated.dsl import _container_op -from kfp.pipeline_spec import pipeline_spec_pb2 - -from google.protobuf import text_format -from google.protobuf import json_format - -_EXPECTED_CONTAINER_WITH_RESOURCE = """ -resources { - cpu_limit: 1.0 - memory_limit: 1.0 - accelerator { - type: 'NVIDIA_TESLA_K80' - count: 1 - } -} -""" - -# Shorthand for PipelineContainerSpec -_PipelineContainerSpec = pipeline_spec_pb2.PipelineDeploymentConfig.PipelineContainerSpec - - -class ContainerOpTest(unittest.TestCase): - - def test_chained_call_resource_setter(self): - task = _container_op.ContainerOp(name='test_task', image='python:3.9') - task.container_spec = _PipelineContainerSpec() - (task.set_cpu_limit('1').set_memory_limit( - '1G').add_node_selector_constraint( - 'cloud.google.com/gke-accelerator', - 'nvidia-tesla-k80').set_gpu_limit(1)) - - expected_container_spec = text_format.Parse( - _EXPECTED_CONTAINER_WITH_RESOURCE, _PipelineContainerSpec()) - - self.assertDictEqual( - json_format.MessageToDict(task.container_spec), - json_format.MessageToDict(expected_container_spec)) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/dsl/_for_loop.py b/sdk/python/kfp/deprecated/dsl/_for_loop.py deleted file mode 100644 index 2650409e55f..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_for_loop.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re -from typing import Any, Dict, List, Optional, Tuple, Union - -from kfp.deprecated import dsl -from kfp.deprecated.dsl import _pipeline_param - -ItemList = List[Union[int, float, str, Dict[str, Any]]] - - -class LoopArguments(dsl.PipelineParam): - """Class representing the arguments that are looped over in a ParallelFor - loop in the KFP DSL. - - This doesn't need to be instantiated by the end user, rather it will - be automatically created by a ParallelFor ops group. - """ - LOOP_ITEM_NAME_BASE = 'loop-item' - LOOP_ITEM_PARAM_NAME_BASE = 'loop-item-param' - # number of characters in the code which is passed to the constructor - NUM_CODE_CHARS = 8 - LEGAL_SUBVAR_NAME_REGEX = re.compile(r'^[a-zA-Z_][0-9a-zA-Z_]*$') - - @classmethod - def _subvar_name_is_legal(cls, proposed_variable_name: str): - return re.match(cls.LEGAL_SUBVAR_NAME_REGEX, - proposed_variable_name) is not None - - def __init__(self, - items: Union[ItemList, dsl.PipelineParam], - code: str, - name_override: Optional[str] = None, - op_name: Optional[str] = None, - *args, - **kwargs): - """LoopArguments represent the set of items to loop over in a - ParallelFor loop. - - This class shouldn't be instantiated by the user but rather is created by - _ops_group.ParallelFor. - - Args: - items: List of items to loop over. If a list of dicts then, all - dicts must have the same keys and every key must be a legal Python - variable name. - code: A unique code used to identify these loop arguments. Should - match the code for the ParallelFor ops_group which created these - LoopArguments. This prevents parameter name collisions. - """ - if name_override is None: - super().__init__(name=self._make_name(code), *args, **kwargs) - else: - super().__init__( - name=name_override, op_name=op_name, *args, **kwargs) - - if not isinstance(items, (list, tuple, dsl.PipelineParam)): - raise TypeError( - 'Expected list, tuple, or PipelineParam, got {}.'.format( - type(items))) - - if isinstance(items, tuple): - items = list(items) - - if isinstance(items, list) and isinstance(items[0], dict): - subvar_names = set(items[0].keys()) - for item in items: - if not set(item.keys()) == subvar_names: - raise ValueError( - 'If you input a list of dicts then all dicts should have the same keys. ' - 'Got: {}.'.format(items)) - - # then this block creates loop_args.variable_a and loop_args.variable_b - for subvar_name in subvar_names: - if not self._subvar_name_is_legal(subvar_name): - raise ValueError( - "Tried to create subvariable named {} but that's not a legal Python variable " - 'name.'.format(subvar_name)) - setattr( - self, subvar_name, - LoopArgumentVariable( - loop_args_name=self.name, - this_variable_name=subvar_name, - loop_args_op_name=self.op_name, - loop_args=self, - )) - - self.items_or_pipeline_param = items - self.referenced_subvar_names = [] - - @classmethod - def from_pipeline_param(cls, param: dsl.PipelineParam) -> 'LoopArguments': - return LoopArguments( - items=param, - code=None, - name_override=param.name + '-' + cls.LOOP_ITEM_NAME_BASE, - op_name=param.op_name, - value=param.value, - ) - - def __getattr__(self, item): - # this is being overridden so that we can access subvariables of the - # LoopArguments (i.e.: item.a) without knowing the subvariable names ahead - # of time - self.referenced_subvar_names.append(item) - return LoopArgumentVariable( - loop_args_name=self.name, - this_variable_name=item, - loop_args_op_name=self.op_name, - loop_args=self, - ) - - def to_list_for_task_yaml(self): - if isinstance(self.items_or_pipeline_param, (list, tuple)): - return self.items_or_pipeline_param - else: - raise ValueError( - 'You should only call this method on loop args which have list items, ' - 'not pipeline param items.') - - @classmethod - def _make_name(cls, code: str): - """Make a name for this parameter. - - Code is a - """ - return '{}-{}'.format(cls.LOOP_ITEM_PARAM_NAME_BASE, code) - - @classmethod - def name_is_loop_argument(cls, param_name: str) -> bool: - """Return True if the given parameter name looks like a loop argument. - - Either it came from a withItems loop item or withParams loop - item. - """ - return cls.name_is_withitems_loop_argument(param_name) \ - or cls.name_is_withparams_loop_argument(param_name) - - @classmethod - def name_is_withitems_loop_argument(cls, param_name: str) -> bool: - """Return True if the given parameter name looks like it came from a - loop arguments parameter.""" - return (cls.LOOP_ITEM_PARAM_NAME_BASE + '-') in param_name - - @classmethod - def name_is_withparams_loop_argument(cls, param_name: str) -> bool: - """Return True if the given parameter name looks like it came from a - withParams loop item.""" - return ('-' + cls.LOOP_ITEM_NAME_BASE) in param_name - - @classmethod - def remove_loop_item_base_name(cls, param_name: str) -> str: - """Removes the last LOOP_ITEM_NAME_BASE from the end of param name.""" - if ('-' + cls.LOOP_ITEM_NAME_BASE) in param_name: - # Split from the right, so that it handles multi-level nested args. - return param_name.rsplit('-' + cls.LOOP_ITEM_NAME_BASE, 1)[0] - return param_name - - -class LoopArgumentVariable(dsl.PipelineParam): - """Represents a subvariable for loop arguments. - - This is used for cases where we're looping over maps, each of - which contains several variables. - """ - SUBVAR_NAME_DELIMITER = '-subvar-' - - def __init__( - self, - loop_args_name: str, - this_variable_name: str, - loop_args_op_name: Optional[str], - # For backward compatible, add loop_args as an optional argument. - # Ideally, this should replace loop_args_name and loop_args_op_name. - loop_args: Optional[LoopArguments] = None, - ): - """ - If the user ran: - with dsl.ParallelFor([{'a': 1, 'b': 2}, {'a': 3, 'b': 4}]) as item: - ... - Then there's be one LoopArgumentsVariable for 'a' and another for 'b'. - - Args: - loop_args_name: The name of the LoopArguments object that this is - a subvariable to. - this_variable_name: The name of this subvariable, which is the name - of the dict key that spawned this subvariable. - loop_args_op_name: The name of the op that produced the loop arguments. - loop_args: Optional; The LoopArguments object this subvariable is based on. - """ - super().__init__( - name=self.get_name( - loop_args_name=loop_args_name, - this_variable_name=this_variable_name), - op_name=loop_args_op_name, - ) - self.loop_args = loop_args - - @property - def items_or_pipeline_param( - self) -> Union[ItemList, _pipeline_param.PipelineParam]: - return self.loop_args.items_or_pipeline_param - - @classmethod - def get_name(cls, loop_args_name: str, this_variable_name: str) -> str: - """Get the name. - - Args: - loop_args_name: the name of the loop args parameter that this - LoopArgsVariable is attached to. - this_variable_name: the name of this LoopArgumentsVariable subvar. - - Returns: - The name of this loop args variable. - """ - return '{}{}{}'.format(loop_args_name, cls.SUBVAR_NAME_DELIMITER, - this_variable_name) - - @classmethod - def name_is_loop_arguments_variable(cls, param_name: str) -> bool: - """Return True if the given parameter name looks like it came from a - LoopArgumentsVariable.""" - return re.match('.+%s.+' % cls.SUBVAR_NAME_DELIMITER, - param_name) is not None - - @classmethod - def parse_loop_args_name_and_this_var_name(cls, t: str) -> Tuple[str, str]: - """Get the loop arguments param name and this subvariable name from the - given parameter name.""" - m = re.match( - '(?P.*){}(?P.*)'.format( - cls.SUBVAR_NAME_DELIMITER), t) - if m is None: - return None - else: - return m.groupdict()['loop_args_name'], m.groupdict( - )['this_var_name'] - - @classmethod - def get_subvar_name(cls, t: str) -> str: - """Get the subvariable name from a given LoopArgumentsVariable - parameter name.""" - out = cls.parse_loop_args_name_and_this_var_name(t) - if out is None: - raise ValueError("Couldn't parse variable name: {}".format(t)) - return out[1] diff --git a/sdk/python/kfp/deprecated/dsl/_metadata.py b/sdk/python/kfp/deprecated/dsl/_metadata.py deleted file mode 100644 index 4cf1f7e2057..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_metadata.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import warnings -from .types import BaseType, _check_valid_type_dict - - -def _annotation_to_typemeta(annotation): - """_annotation_to_type_meta converts an annotation to a type structure. - - Args: - annotation(BaseType/str/dict): input/output annotations - BaseType: registered in kfp.dsl.types - str: either a string of a dict serialization or a string of the type name - dict: type name and properties. note that the properties values can be - dict. - - Returns: - dict or string representing the type - """ - if isinstance(annotation, BaseType): - arg_type = annotation.to_dict() - elif isinstance(annotation, str): - arg_type = annotation - elif isinstance(annotation, dict): - if not _check_valid_type_dict(annotation): - raise ValueError('Annotation ' + str(annotation) + - ' is not a valid type dictionary.') - arg_type = annotation - else: - return None - return arg_type - - -def _extract_pipeline_metadata(func): - """Creates pipeline metadata structure instance based on the function - signature.""" - - # Most of this code is only needed for verifying the default values against - # "openapi_schema_validator" type properties. - # TODO: Move the value verification code to some other place - - from ._pipeline_param import PipelineParam - - import inspect - fullargspec = inspect.getfullargspec(func) - args = fullargspec.args - annotations = fullargspec.annotations - - # defaults - arg_defaults = {} - if fullargspec.defaults: - for arg, default in zip( - reversed(fullargspec.args), reversed(fullargspec.defaults)): - arg_defaults[arg] = default - - for arg in args: - arg_type = None - arg_default = arg_defaults[arg] if arg in arg_defaults else None - if isinstance(arg_default, PipelineParam): - warnings.warn( - 'Explicit creation of `kfp.dsl.PipelineParam`s by the users is ' - 'deprecated. The users should define the parameter type and default ' - 'values using standard pythonic constructs: ' - 'def my_func(a: int = 1, b: str = "default"):') - arg_default = arg_default.value - if arg in annotations: - arg_type = _annotation_to_typemeta(annotations[arg]) - arg_type_properties = list(arg_type.values())[0] if isinstance( - arg_type, dict) else {} - if 'openapi_schema_validator' in arg_type_properties and (arg_default - is not None): - from jsonschema import validate - import json - schema_object = arg_type_properties['openapi_schema_validator'] - if isinstance(schema_object, str): - # In case the property value for the schema validator is a string - # instead of a dict. - schema_object = json.loads(schema_object) - # Only validating non-serialized values - validate(instance=arg_default, schema=schema_object) - - from kfp.deprecated.components._python_op import _extract_component_interface - component_spec = _extract_component_interface(func) - return component_spec diff --git a/sdk/python/kfp/deprecated/dsl/_ops_group.py b/sdk/python/kfp/deprecated/dsl/_ops_group.py deleted file mode 100644 index 65fa90c4338..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_ops_group.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright 2018-2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from typing import Union - -from kfp.deprecated.dsl import _for_loop, _pipeline_param - -from . import _container_op -from . import _pipeline - - -class OpsGroup(object): - """Represents a logical group of ops and group of OpsGroups. - - This class is the base class for groups of ops, such as ops sharing - an exit handler, a condition branch, or a loop. This class is not - supposed to be used by pipeline authors. It is useful for - implementing a compiler. - """ - - def __init__(self, - group_type: str, - name: str = None, - parallelism: int = None): - """Create a new instance of OpsGroup. - - Args: - group_type (str): one of 'pipeline', 'exit_handler', 'condition', - 'for_loop', and 'graph'. - name (str): name of the opsgroup - parallelism (int): parallelism for the sub-DAG:s - """ - #TODO: declare the group_type to be strongly typed - self.type = group_type - self.ops = list() - self.groups = list() - self.name = name - self.dependencies = [] - self.parallelism = parallelism - # recursive_ref points to the opsgroups with the same name if exists. - self.recursive_ref = None - - self.loop_args = None - - @staticmethod - def _get_matching_opsgroup_already_in_pipeline(group_type, name): - """Retrieves the opsgroup when the pipeline already contains it. - - the opsgroup might be already in the pipeline in case of recursive calls. - - Args: - group_type (str): one of 'pipeline', 'exit_handler', 'condition', and - 'graph'. - name (str): the name before conversion. - """ - if not _pipeline.Pipeline.get_default_pipeline(): - raise ValueError('Default pipeline not defined.') - if name is None: - return None - name_pattern = '^' + (group_type + '-' + name + '-').replace( - '_', '-') + r'[\d]+$' - for ops_group_already_in_pipeline in _pipeline.Pipeline.get_default_pipeline( - ).groups: - import re - if ops_group_already_in_pipeline.type == group_type \ - and re.match(name_pattern ,ops_group_already_in_pipeline.name): - return ops_group_already_in_pipeline - return None - - def _make_name_unique(self): - """Generate a unique opsgroup name in the pipeline.""" - if not _pipeline.Pipeline.get_default_pipeline(): - raise ValueError('Default pipeline not defined.') - - self.name = ( - self.type + '-' + ('' if self.name is None else self.name + '-') + - str(_pipeline.Pipeline.get_default_pipeline().get_next_group_id())) - self.name = self.name.replace('_', '-') - - def __enter__(self): - if not _pipeline.Pipeline.get_default_pipeline(): - raise ValueError('Default pipeline not defined.') - - self.recursive_ref = self._get_matching_opsgroup_already_in_pipeline( - self.type, self.name) - if not self.recursive_ref: - self._make_name_unique() - - _pipeline.Pipeline.get_default_pipeline().push_ops_group(self) - return self - - def __exit__(self, *args): - _pipeline.Pipeline.get_default_pipeline().pop_ops_group() - - def after(self, *ops): - """Specify explicit dependency on other ops.""" - for op in ops: - self.dependencies.append(op) - return self - - def remove_op_recursive(self, op): - if self.ops and op in self.ops: - self.ops.remove(op) - for sub_group in self.groups or []: - sub_group.remove_op_recursive(op) - - -class SubGraph(OpsGroup): - TYPE_NAME = 'subgraph' - - def __init__(self, parallelism: int): - if parallelism < 1: - raise ValueError( - 'SubGraph parallism set to < 1, allowed values are > 0') - super(SubGraph, self).__init__(self.TYPE_NAME, parallelism=parallelism) - - -class ExitHandler(OpsGroup): - """Represents an exit handler that is invoked upon exiting a group of ops. - - Args: - exit_op: An operator invoked at exiting a group of ops. - - Raises: - ValueError: Raised if the exit_op is invalid. - - Example: - :: - - exit_op = ContainerOp(...) - with ExitHandler(exit_op): - op1 = ContainerOp(...) - op2 = ContainerOp(...) - """ - - def __init__(self, exit_op: _container_op.ContainerOp): - super(ExitHandler, self).__init__('exit_handler') - if exit_op.dependent_names: - raise ValueError('exit_op cannot depend on any other ops.') - - # Removing exit_op form any group - _pipeline.Pipeline.get_default_pipeline().remove_op_from_groups(exit_op) - - # Setting is_exit_handler since the compiler might be using this attribute. - # TODO: Check that it's needed - exit_op.is_exit_handler = True - - self.exit_op = exit_op - - -class Condition(OpsGroup): - """Represents an condition group with a condition. - - Args: - condition (ConditionOperator): the condition. - name (str): name of the condition - Example: :: - with Condition(param1=='pizza', '[param1 is pizza]'): op1 = - ContainerOp(...) op2 = ContainerOp(...) - """ - - def __init__(self, condition, name=None): - super(Condition, self).__init__('condition', name) - self.condition = condition - - -class Graph(OpsGroup): - """Graph DAG with inputs, recursive_inputs, and outputs. - - This is not used directly by the users but auto generated when the - graph_component decoration exists - - Args: - name: Name of the graph. - """ - - def __init__(self, name): - super(Graph, self).__init__(group_type='graph', name=name) - self.inputs = [] - self.outputs = {} - self.dependencies = [] - - -class ParallelFor(OpsGroup): - """Represents a parallel for loop over a static set of items. - - Example: - In this case :code:`op1` would be executed twice, once with case - :code:`args=['echo 1']` and once with case :code:`args=['echo 2']`:: - - with dsl.ParallelFor([{'a': 1, 'b': 10}, {'a': 2, 'b': 20}]) as item: - op1 = ContainerOp(..., args=['echo {}'.format(item.a)]) - op2 = ContainerOp(..., args=['echo {}'.format(item.b]) - """ - TYPE_NAME = 'for_loop' - - def __init__(self, - loop_args: Union[_for_loop.ItemList, - _pipeline_param.PipelineParam], - parallelism: int = None): - if parallelism and parallelism < 1: - raise ValueError( - 'ParallelFor parallism set to < 1, allowed values are > 0') - - self.items_is_pipeline_param = isinstance(loop_args, - _pipeline_param.PipelineParam) - - super().__init__(self.TYPE_NAME, parallelism=parallelism) - - if self.items_is_pipeline_param: - loop_args = _for_loop.LoopArguments.from_pipeline_param(loop_args) - elif not self.items_is_pipeline_param and not isinstance( - loop_args, _for_loop.LoopArguments): - # we were passed a raw list, wrap it in loop args - loop_args = _for_loop.LoopArguments( - loop_args, - code=str(_pipeline.Pipeline.get_default_pipeline() - .get_next_group_id()), - ) - - self.loop_args = loop_args - - def __enter__(self) -> _for_loop.LoopArguments: - _ = super().__enter__() - return self.loop_args diff --git a/sdk/python/kfp/deprecated/dsl/_pipeline.py b/sdk/python/kfp/deprecated/dsl/_pipeline.py deleted file mode 100644 index d69490a24e0..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_pipeline.py +++ /dev/null @@ -1,372 +0,0 @@ -# Copyright 2018-2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import enum -from typing import Callable, Optional, Union - -from kubernetes.client.models import V1PodDNSConfig -from kfp.deprecated.dsl import _container_op -from kfp.deprecated.dsl import _ops_group -from kfp.deprecated.dsl import _component_bridge -from kfp.deprecated.components import _components -from kfp.deprecated.components import _naming - -# This handler is called whenever the @pipeline decorator is applied. -# It can be used by command-line DSL compiler to inject code that runs for every -# pipeline definition. -_pipeline_decorator_handler = None - - -class PipelineExecutionMode(enum.Enum): - # Compile to Argo YAML without support for metadata-enabled components. - V1_LEGACY = 1 - # Compiles to Argo YAML with support for metadata-enabled components. - # Pipelines compiled using this mode aim to be compatible with v2 semantics. - V2_COMPATIBLE = 2 - # Compiles to KFP v2 IR for execution using the v2 engine. - # This option is unsupported right now. - V2_ENGINE = 3 - - -def pipeline(name: Optional[str] = None, - description: Optional[str] = None, - pipeline_root: Optional[str] = None): - """Decorator of pipeline functions. - - Example - :: - - @pipeline( - name='my-pipeline', - description='My ML Pipeline.' - pipeline_root='gs://my-bucket/my-output-path' - ) - def my_pipeline(a: PipelineParam, b: PipelineParam): - ... - - Args: - name: The pipeline name. Default to a sanitized version of the function - name. - description: Optionally, a human-readable description of the pipeline. - pipeline_root: The root directory to generate input/output URI under this - pipeline. This is required if input/output URI placeholder is used in this - pipeline. - """ - - def _pipeline(func: Callable): - if name: - func._component_human_name = name - if description: - func._component_description = description - if pipeline_root: - func.pipeline_root = pipeline_root - - if _pipeline_decorator_handler: - return _pipeline_decorator_handler(func) or func - else: - return func - - return _pipeline - - -class PipelineConf(): - """PipelineConf contains pipeline level settings.""" - - def __init__(self): - self.image_pull_secrets = [] - self.timeout = 0 - self.ttl_seconds_after_finished = -1 - self._pod_disruption_budget_min_available = None - self.op_transformers = [] - self.default_pod_node_selector = {} - self.image_pull_policy = None - self.parallelism = None - self._data_passing_method = None - self.dns_config = None - - def set_image_pull_secrets(self, image_pull_secrets): - """Configures the pipeline level imagepullsecret. - - Args: - image_pull_secrets: a list of Kubernetes V1LocalObjectReference For - detailed description, check Kubernetes V1LocalObjectReference definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1LocalObjectReference.md - """ - self.image_pull_secrets = image_pull_secrets - return self - - def set_timeout(self, seconds: int): - """Configures the pipeline level timeout. - - Args: - seconds: number of seconds for timeout - """ - self.timeout = seconds - return self - - def set_parallelism(self, max_num_pods: int): - """Configures the max number of total parallel pods that can execute at - the same time in a workflow. - - Args: - max_num_pods: max number of total parallel pods. - """ - if max_num_pods < 1: - raise ValueError( - 'Pipeline max_num_pods set to < 1, allowed values are > 0') - - self.parallelism = max_num_pods - return self - - def set_ttl_seconds_after_finished(self, seconds: int): - """Configures the ttl after the pipeline has finished. - - Args: - seconds: number of seconds for the workflow to be garbage collected after - it is finished. - """ - self.ttl_seconds_after_finished = seconds - return self - - def set_pod_disruption_budget(self, min_available: Union[int, str]): - """PodDisruptionBudget holds the number of concurrent disruptions that - you allow for pipeline Pods. - - Args: - min_available (Union[int, str]): An eviction is allowed if at least - "minAvailable" pods selected by "selector" will still be available after - the eviction, i.e. even in the absence of the evicted pod. So for - example you can prevent all voluntary evictions by specifying "100%". - "minAvailable" can be either an absolute number or a percentage. - """ - self._pod_disruption_budget_min_available = min_available - return self - - def set_default_pod_node_selector(self, label_name: str, value: str): - """Add a constraint for nodeSelector for a pipeline. - - Each constraint is a key-value pair label. - - For the container to be eligible to run on a node, the node must have each - of the constraints appeared as labels. - - Args: - label_name: The name of the constraint label. - value: The value of the constraint label. - """ - self.default_pod_node_selector[label_name] = value - return self - - def set_image_pull_policy(self, policy: str): - """Configures the default image pull policy. - - Args: - policy: the pull policy, has to be one of: Always, Never, IfNotPresent. - For more info: - https://github.com/kubernetes-client/python/blob/10a7f95435c0b94a6d949ba98375f8cc85a70e5a/kubernetes/docs/V1Container.md - """ - self.image_pull_policy = policy - return self - - def add_op_transformer(self, transformer): - """Configures the op_transformers which will be applied to all ops in - the pipeline. The ops can be ResourceOp, VolumeOp, or ContainerOp. - - Args: - transformer: A function that takes a kfp Op as input and returns a kfp Op - """ - self.op_transformers.append(transformer) - - def set_dns_config(self, dns_config: V1PodDNSConfig): - """Set the dnsConfig to be given to each pod. - - Args: - dns_config: Kubernetes V1PodDNSConfig For detailed description, check - Kubernetes V1PodDNSConfig definition - https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1PodDNSConfig.md - - Example: - :: - - import kfp - from kubernetes.client.models import V1PodDNSConfig, V1PodDNSConfigOption - pipeline_conf = kfp.dsl.PipelineConf() - pipeline_conf.set_dns_config(dns_config=V1PodDNSConfig( - nameservers=["1.2.3.4"], - options=[V1PodDNSConfigOption(name="ndots", value="2")], - )) - """ - self.dns_config = dns_config - - @property - def data_passing_method(self): - return self._data_passing_method - - @data_passing_method.setter - def data_passing_method(self, value): - """Sets the object representing the method used for intermediate data - passing. - - Example: - :: - - from kfp.dsl import PipelineConf, data_passing_methods - from kubernetes.client.models import V1Volume, V1PersistentVolumeClaimVolumeSource - pipeline_conf = PipelineConf() - pipeline_conf.data_passing_method = - data_passing_methods.KubernetesVolume( - volume=V1Volume( - name='data', - persistent_volume_claim=V1PersistentVolumeClaimVolumeSource('data-volume'), - ), - path_prefix='artifact_data/', - ) - """ - self._data_passing_method = value - - -def get_pipeline_conf(): - """Configure the pipeline level setting to the current pipeline - Note: call the function inside the user defined pipeline function. - """ - return Pipeline.get_default_pipeline().conf - - -# TODO: Pipeline is in fact an opsgroup, refactor the code. -class Pipeline(): - """A pipeline contains a list of operators. - - This class is not supposed to be used by pipeline authors since pipeline - authors can use pipeline functions (decorated with @pipeline) to reference - their pipelines. - This class is useful for implementing a compiler. For example, the compiler - can use the following to get the pipeline object and its ops: - - Example: - :: - - with Pipeline() as p: - pipeline_func(*args_list) - - traverse(p.ops) - """ - - # _default_pipeline is set when it (usually a compiler) runs "with Pipeline()" - _default_pipeline = None - - @staticmethod - def get_default_pipeline(): - """Get default pipeline.""" - return Pipeline._default_pipeline - - @staticmethod - def add_pipeline(name, description, func): - """Add a pipeline function with the specified name and description.""" - # Applying the @pipeline decorator to the pipeline function - func = pipeline(name=name, description=description)(func) - - def __init__(self, name: str): - """Create a new instance of Pipeline. - - Args: - name: the name of the pipeline. Once deployed, the name will show up in - Pipeline System UI. - """ - self.name = name - self.ops = {} - # Add the root group. - self.groups = [_ops_group.OpsGroup('pipeline', name=name)] - self.group_id = 0 - self.conf = PipelineConf() - self._metadata = None - - def __enter__(self): - if Pipeline._default_pipeline: - raise Exception('Nested pipelines are not allowed.') - - Pipeline._default_pipeline = self - self._old_container_task_constructor = ( - _components._container_task_constructor) - _components._container_task_constructor = ( - _component_bridge._create_container_op_from_component_and_arguments) - - def register_op_and_generate_id(op): - return self.add_op(op, op.is_exit_handler) - - self._old__register_op_handler = _container_op._register_op_handler - _container_op._register_op_handler = register_op_and_generate_id - return self - - def __exit__(self, *args): - Pipeline._default_pipeline = None - _container_op._register_op_handler = self._old__register_op_handler - _components._container_task_constructor = ( - self._old_container_task_constructor) - - def add_op(self, op: _container_op.BaseOp, define_only: bool): - """Add a new operator. - - Args: - op: An operator of ContainerOp, ResourceOp or their inherited types. - Returns - op_name: a unique op name. - """ - # Sanitizing the op name. - # Technically this could be delayed to the compilation stage, but string - # serialization of PipelineParams make unsanitized names problematic. - op_name = _naming._sanitize_python_function_name(op.human_name).replace( - '_', '-') - #If there is an existing op with this name then generate a new name. - op_name = _naming._make_name_unique_by_adding_index( - op_name, list(self.ops.keys()), '-') - if op_name == '': - op_name = _naming._make_name_unique_by_adding_index( - 'task', list(self.ops.keys()), '-') - - self.ops[op_name] = op - if not define_only: - self.groups[-1].ops.append(op) - - return op_name - - def push_ops_group(self, group: _ops_group.OpsGroup): - """Push an OpsGroup into the stack. - - Args: - group: An OpsGroup. Typically it is one of ExitHandler, Branch, and Loop. - """ - self.groups[-1].groups.append(group) - self.groups.append(group) - - def pop_ops_group(self): - """Remove the current OpsGroup from the stack.""" - del self.groups[-1] - - def remove_op_from_groups(self, op): - for group in self.groups: - group.remove_op_recursive(op) - - def get_next_group_id(self): - """Get next id for a new group.""" - - self.group_id += 1 - return self.group_id - - def _set_metadata(self, metadata): - """_set_metadata passes the containerop the metadata information. - - Args: - metadata (ComponentMeta): component metadata - """ - self._metadata = metadata diff --git a/sdk/python/kfp/deprecated/dsl/_pipeline_param.py b/sdk/python/kfp/deprecated/dsl/_pipeline_param.py deleted file mode 100644 index cdec0cae885..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_pipeline_param.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import re -from collections import namedtuple -from typing import Dict, List, Optional, Union - -# TODO: Move this to a separate class -# For now, this identifies a condition with only "==" operator supported. -ConditionOperator = namedtuple('ConditionOperator', - 'operator operand1 operand2') -PipelineParamTuple = namedtuple('PipelineParamTuple', 'name op pattern') - - -def sanitize_k8s_name(name, allow_capital_underscore=False): - """Cleans and converts the names in the workflow. - - Args: - name: original name, - allow_capital_underscore: whether to allow capital letter and underscore in - this name. - - Returns: - A sanitized name. - """ - if allow_capital_underscore: - return re.sub('-+', '-', re.sub('[^-_0-9A-Za-z]+', '-', - name)).lstrip('-').rstrip('-') - else: - return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', - name.lower())).lstrip('-').rstrip('-') - - -def match_serialized_pipelineparam(payload: str) -> List[PipelineParamTuple]: - """Matches the supplied serialized pipelineparam. - - Args: - payloads: The search space for the serialized pipelineparams. - - Returns: - The matched pipeline params we found in the supplied payload. - """ - matches = re.findall(r'{{pipelineparam:op=([\w\s_-]*);name=([\w\s_-]+)}}', - payload) - param_tuples = [] - for match in matches: - pattern = '{{pipelineparam:op=%s;name=%s}}' % (match[0], match[1]) - param_tuples.append( - PipelineParamTuple( - name=sanitize_k8s_name(match[1], True), - op=sanitize_k8s_name(match[0]), - pattern=pattern)) - return param_tuples - - -def _extract_pipelineparams( - payloads: Union[str, List[str]]) -> List['PipelineParam']: - """Extracts a list of PipelineParam instances from the payload string. - - Note: this function removes all duplicate matches. - - Args: - payload: a string/a list of strings that contains serialized pipelineparams - - Return: List[] - """ - if isinstance(payloads, str): - payloads = [payloads] - param_tuples = [] - for payload in payloads: - param_tuples += match_serialized_pipelineparam(payload) - pipeline_params = [] - for param_tuple in list(set(param_tuples)): - pipeline_params.append( - PipelineParam( - param_tuple.name, param_tuple.op, pattern=param_tuple.pattern)) - return pipeline_params - - -def extract_pipelineparams_from_any( - payload: Union['PipelineParam', str, list, tuple, dict] -) -> List['PipelineParam']: - """Recursively extract PipelineParam instances or serialized string from - any object or list of objects. - - Args: - payload (str or k8_obj or list[str or k8_obj]): a string/a list of strings - that contains serialized pipelineparams or a k8 definition object. - - Return: List[PipelineParam] - """ - if not payload: - return [] - - # PipelineParam - if isinstance(payload, PipelineParam): - return [payload] - - # str - if isinstance(payload, str): - return list(set(_extract_pipelineparams(payload))) - - # list or tuple - if isinstance(payload, list) or isinstance(payload, tuple): - pipeline_params = [] - for item in payload: - pipeline_params += extract_pipelineparams_from_any(item) - return list(set(pipeline_params)) - - # dict - if isinstance(payload, dict): - pipeline_params = [] - for key, value in payload.items(): - pipeline_params += extract_pipelineparams_from_any(key) - pipeline_params += extract_pipelineparams_from_any(value) - return list(set(pipeline_params)) - - # k8s OpenAPI object - if hasattr(payload, 'attribute_map') and isinstance(payload.attribute_map, - dict): - pipeline_params = [] - for key in payload.attribute_map: - pipeline_params += extract_pipelineparams_from_any( - getattr(payload, key)) - - return list(set(pipeline_params)) - - # return empty list - return [] - - -class PipelineParam(object): - """Representing a future value that is passed between pipeline components. - - A PipelineParam object can be used as a pipeline function argument so that it - will be a pipeline parameter that shows up in ML Pipelines system UI. It can - also represent an intermediate value passed between components. - - Args: - name: name of the pipeline parameter. - op_name: the name of the operation that produces the PipelineParam. None - means it is not produced by any operator, so if None, either user - constructs it directly (for providing an immediate value), or it is a - pipeline function argument. - value: The actual value of the PipelineParam. If provided, the PipelineParam - is "resolved" immediately. For now, we support string only. - param_type: the type of the PipelineParam. - pattern: the serialized string regex pattern this pipeline parameter created - from. - - Raises: ValueError in name or op_name contains invalid characters, or both - op_name and value are set. - """ - - def __init__(self, - name: str, - op_name: Optional[str] = None, - value: Optional[str] = None, - param_type: Optional[Union[str, Dict]] = None, - pattern: Optional[str] = None): - valid_name_regex = r'^[A-Za-z][A-Za-z0-9\s_-]*$' - if not re.match(valid_name_regex, name): - raise ValueError( - 'Only letters, numbers, spaces, "_", and "-" are allowed in name. ' - 'Must begin with a letter. Got name: {}'.format(name)) - - if op_name and value: - raise ValueError('op_name and value cannot be both set.') - - self.name = name - # ensure value is None even if empty string or empty list - # so that serialization and unserialization remain consistent - # (i.e. None => '' => None) - self.op_name = op_name if op_name else None - self.value = value - self.param_type = param_type - self.pattern = pattern or str(self) - - @property - def full_name(self): - """Unique name in the argo yaml for the PipelineParam.""" - if self.op_name: - return self.op_name + '-' + self.name - return self.name - - def __str__(self): - """String representation. - - The string representation is a string identifier so we can mix - the PipelineParam inline with other strings such as arguments. - For example, we can support: ['echo %s' % param] as the - container command and later a compiler can replace the - placeholder "{{pipelineparam:op=%s;name=%s}}" with its own - parameter identifier. - """ - - #This is deleted because if users specify default values to PipelineParam, - # The compiler can not detect it as the value is not NULL. - #if self.value: - # return str(self.value) - - op_name = self.op_name if self.op_name else '' - return '{{pipelineparam:op=%s;name=%s}}' % (op_name, self.name) - - def __repr__(self): - # return str({self.__class__.__name__: self.__dict__}) - # We make repr return the placeholder string so that if someone uses - # str()-based serialization of complex objects containing `PipelineParam`, - # it works properly. - # (e.g. str([1, 2, 3, kfp.dsl.PipelineParam("aaa"), 4, 5, 6,])) - return str(self) - - def to_struct(self): - # Used by the json serializer. Outputs a JSON-serializable representation of - # the object - return str(self) - - def __eq__(self, other): - return ConditionOperator('==', self, other) - - def __ne__(self, other): - return ConditionOperator('!=', self, other) - - def __lt__(self, other): - return ConditionOperator('<', self, other) - - def __le__(self, other): - return ConditionOperator('<=', self, other) - - def __gt__(self, other): - return ConditionOperator('>', self, other) - - def __ge__(self, other): - return ConditionOperator('>=', self, other) - - def __hash__(self): - return hash((self.op_name, self.name)) - - def ignore_type(self): - """ignore_type ignores the type information such that type checking - would also pass.""" - self.param_type = None - return self diff --git a/sdk/python/kfp/deprecated/dsl/_pipeline_volume.py b/sdk/python/kfp/deprecated/dsl/_pipeline_volume.py deleted file mode 100644 index accb79037e4..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_pipeline_volume.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import hashlib -import json - -from kubernetes.client.models import (V1Volume, - V1PersistentVolumeClaimVolumeSource) - -from . import _pipeline - - -def prune_none_dict_values(d: dict) -> dict: - return { - k: prune_none_dict_values(v) if isinstance(v, dict) else v - for k, v in d.items() - if v is not None - } - - -class PipelineVolume(V1Volume): - """Representing a volume that is passed between pipeline operators and is. - - to be mounted by a ContainerOp or its inherited type. - - A PipelineVolume object can be used as an extention of the pipeline - function's filesystem. It may then be passed between ContainerOps, - exposing dependencies. - - TODO(https://github.com/kubeflow/pipelines/issues/4822): Determine the - stability level of this feature. - - Args: - pvc: The name of an existing PVC - volume: Create a deep copy out of a V1Volume or PipelineVolume with no - deps - - Raises: - ValueError: If volume is not None and kwargs is not None - If pvc is not None and kwargs.pop("name") is not None - """ - - def __init__(self, pvc: str = None, volume: V1Volume = None, **kwargs): - if volume and kwargs: - raise ValueError("You can't pass a volume along with other " - "kwargs.") - - name_provided = True - init_volume = {} - if volume: - init_volume = { - attr: getattr(volume, attr) - for attr in self.attribute_map.keys() - } - else: - if "name" in kwargs: - if len(kwargs["name"]) > 63: - raise ValueError("PipelineVolume name must be no more than" - " 63 characters") - init_volume = {"name": kwargs.pop("name")} - else: - name_provided = False - init_volume = {"name": "pvolume-placeholder"} - if pvc and kwargs: - raise ValueError("You can only pass 'name' along with 'pvc'.") - elif pvc and not kwargs: - pvc_volume_source = V1PersistentVolumeClaimVolumeSource( - claim_name=str(pvc)) - init_volume["persistent_volume_claim"] = pvc_volume_source - super().__init__(**init_volume, **kwargs) - if not name_provided: - volume_dict = prune_none_dict_values(self.to_dict()) - hash_value = hashlib.sha256( - bytes(json.dumps(volume_dict, sort_keys=True), - "utf-8")).hexdigest() - name = "pvolume-{}".format(hash_value) - self.name = name[0:63] if len(name) > 63 else name - self.dependent_names = [] - - def after(self, *ops): - """Creates a duplicate of self with the required dependecies excluding - the redundant dependenices. - - Args: - *ops: Pipeline operators to add as dependencies - """ - - def implies(newdep, olddep): - if newdep.name == olddep: - return True - for parentdep_name in newdep.dependent_names: - if parentdep_name == olddep: - return True - else: - parentdep = _pipeline.Pipeline.get_default_pipeline( - ).ops[parentdep_name] - if parentdep: - if implies(parentdep, olddep): - return True - return False - - ret = self.__class__(volume=self) - ret.dependent_names = [op.name for op in ops] - - for olddep in self.dependent_names: - implied = False - for newdep in ops: - implied = implies(newdep, olddep) - if implied: - break - if not implied: - ret.dependent_names.append(olddep) - - return ret diff --git a/sdk/python/kfp/deprecated/dsl/_resource_op.py b/sdk/python/kfp/deprecated/dsl/_resource_op.py deleted file mode 100644 index af4c59dae4f..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_resource_op.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict, List, Optional -import warnings - -from ._container_op import BaseOp, ContainerOp -from . import _pipeline_param - - -class Resource(object): - """A wrapper over Argo ResourceTemplate definition object - (io.argoproj.workflow.v1alpha1.ResourceTemplate) which is used to represent - the `resource` property in argo's workflow template - (io.argoproj.workflow.v1alpha1.Template).""" - swagger_types = { - "action": "str", - "merge_strategy": "str", - "success_condition": "str", - "failure_condition": "str", - "set_owner_reference": "bool", - "manifest": "str", - "flags": "list[str]" - } - openapi_types = { - "action": "str", - "merge_strategy": "str", - "success_condition": "str", - "failure_condition": "str", - "set_owner_reference": "bool", - "manifest": "str", - "flags": "list[str]" - } - attribute_map = { - "action": "action", - "merge_strategy": "mergeStrategy", - "success_condition": "successCondition", - "failure_condition": "failureCondition", - "set_owner_reference": "setOwnerReference", - "manifest": "manifest", - "flags": "flags" - } - - def __init__(self, - action: str = None, - merge_strategy: str = None, - success_condition: str = None, - failure_condition: str = None, - set_owner_reference: bool = None, - manifest: str = None, - flags: Optional[List[str]] = None): - """Create a new instance of Resource.""" - self.action = action - self.merge_strategy = merge_strategy - self.success_condition = success_condition - self.failure_condition = failure_condition - self.set_owner_reference = set_owner_reference - self.manifest = manifest - self.flags = flags - - -class ResourceOp(BaseOp): - """Represents an op which will be translated into a resource template. - - TODO(https://github.com/kubeflow/pipelines/issues/4822): Determine the - stability level of this feature. - - Args: - k8s_resource: A k8s resource which will be submitted to the cluster - action: One of "create"/"delete"/"apply"/"patch" (default is "create") - merge_strategy: The merge strategy for the "apply" action - success_condition: The successCondition of the template - failure_condition: The failureCondition of the template - For more info see: - https://github.com/argoproj/argo-workflows/blob/master/examples/k8s-jobs.yaml - attribute_outputs: Maps output labels to resource's json paths, - similarly to file_outputs of ContainerOp - kwargs: name, sidecars. See BaseOp definition - - Raises: - ValueError: if not inside a pipeline - if the name is an invalid string - if no k8s_resource is provided - if merge_strategy is set without "apply" action - """ - - def __init__(self, - k8s_resource=None, - action: str = "create", - merge_strategy: str = None, - success_condition: str = None, - failure_condition: str = None, - set_owner_reference: bool = None, - attribute_outputs: Optional[Dict[str, str]] = None, - flags: Optional[List[str]] = None, - **kwargs): - - super().__init__(**kwargs) - self.attrs_with_pipelineparams = list(self.attrs_with_pipelineparams) - self.attrs_with_pipelineparams.extend( - ["_resource", "k8s_resource", "attribute_outputs"]) - - if k8s_resource is None: - raise ValueError("You need to provide a k8s_resource.") - - if action == "delete": - warnings.warn( - 'Please use `kubernetes_resource_delete_op` instead of ' - '`ResourceOp(action="delete")`', DeprecationWarning) - - if merge_strategy and action != "apply": - raise ValueError( - "You can't set merge_strategy when action != 'apply'") - - # if action is delete, there should not be any outputs, success_condition, - # and failure_condition - if action == "delete" and (success_condition or failure_condition or - attribute_outputs): - raise ValueError( - "You can't set success_condition, failure_condition, or " - "attribute_outputs when action == 'delete'") - - if action == "delete" and flags is None: - flags = ["--wait=false"] - init_resource = { - "action": action, - "merge_strategy": merge_strategy, - "success_condition": success_condition, - "failure_condition": failure_condition, - "set_owner_reference": set_owner_reference, - "flags": flags - } - # `resource` prop in `io.argoproj.workflow.v1alpha1.Template` - self._resource = Resource(**init_resource) - - self.k8s_resource = k8s_resource - - # if action is delete, there should not be any outputs, success_condition, - # and failure_condition - if action == "delete": - self.attribute_outputs = {} - self.outputs = {} - self.output = None - return - - # Set attribute_outputs - extra_attribute_outputs = \ - attribute_outputs if attribute_outputs else {} - self.attribute_outputs = \ - self.attribute_outputs if hasattr(self, "attribute_outputs") \ - else {} - self.attribute_outputs.update(extra_attribute_outputs) - # Add name and manifest if not specified by the user - if "name" not in self.attribute_outputs: - self.attribute_outputs["name"] = "{.metadata.name}" - if "manifest" not in self.attribute_outputs: - self.attribute_outputs["manifest"] = "{}" - - # Set outputs - self.outputs = { - name: _pipeline_param.PipelineParam(name, op_name=self.name) - for name in self.attribute_outputs.keys() - } - # If user set a single attribute_output, set self.output as that - # parameter, else set it as the resource name - self.output = self.outputs["name"] - if len(extra_attribute_outputs) == 1: - self.output = self.outputs[list(extra_attribute_outputs)[0]] - - @property - def resource(self): - """`Resource` object that represents the `resource` property in - `io.argoproj.workflow.v1alpha1.Template`.""" - return self._resource - - def delete(self, flags: Optional[List[str]] = None): - """Returns a ResourceOp which deletes the resource.""" - if self.resource.action == "delete": - raise ValueError("This operation is already a resource deletion.") - - if isinstance(self.k8s_resource, dict): - kind = self.k8s_resource["kind"] - else: - kind = self.k8s_resource.kind - - return kubernetes_resource_delete_op( - name=self.outputs["name"], - kind=kind, - flags=flags or ["--wait=false"]) - - -def kubernetes_resource_delete_op( - name: str, - kind: str, - namespace: str = None, - flags: Optional[List[str]] = None, -) -> ContainerOp: - """Operation that deletes a Kubernetes resource. - - Outputs: - name: The name of the deleted resource - """ - - command = [ - "kubectl", "delete", - str(kind), - str(name), "--ignore-not-found", "--output", "name" - ] - if namespace: - command.extend(["--namespace", str(namespace)]) - if flags: - command.extend(flags) - - result = ContainerOp( - name="kubernetes_resource_delete", - image="gcr.io/cloud-builders/kubectl", - command=command, - ) - return result diff --git a/sdk/python/kfp/deprecated/dsl/_volume_op.py b/sdk/python/kfp/deprecated/dsl/_volume_op.py deleted file mode 100644 index a69b4197318..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_volume_op.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -from typing import List, Dict -from kubernetes.client.models import (V1ObjectMeta, V1ResourceRequirements, - V1PersistentVolumeClaimSpec, - V1PersistentVolumeClaim, - V1TypedLocalObjectReference) - -from ._resource_op import ResourceOp -from ._pipeline_param import (PipelineParam, match_serialized_pipelineparam, - sanitize_k8s_name) -from ._pipeline_volume import PipelineVolume - -VOLUME_MODE_RWO = ["ReadWriteOnce"] -VOLUME_MODE_RWM = ["ReadWriteMany"] -VOLUME_MODE_ROM = ["ReadOnlyMany"] - - -class VolumeOp(ResourceOp): - """Represents an op which will be translated into a resource template which - will be creating a PVC. - - TODO(https://github.com/kubeflow/pipelines/issues/4822): Determine the - stability level of this feature. - - Args: - resource_name: A desired name for the PVC which will be created - size: The size of the PVC which will be created - storage_class: The storage class to use for the dynamically created PVC - modes: The access modes for the PVC - annotations: Annotations to be patched in the PVC - data_source: May be a V1TypedLocalObjectReference, and then it is used - in the data_source field of the PVC as is. Can also be a - string/PipelineParam, and in that case it will be used as a - VolumeSnapshot name (Alpha feature) - volume_name: VolumeName is the binding reference to the PersistentVolume - backing this claim. - generate_unique_name: Generate unique name for the PVC - kwargs: See :py:class:`kfp.dsl.ResourceOp` - - Raises: - ValueError: if k8s_resource is provided along with other arguments - if k8s_resource is not a V1PersistentVolumeClaim - if size is None - if size is an invalid memory string (when not a - PipelineParam) - if data_source is not one of (str, PipelineParam, - V1TypedLocalObjectReference) - """ - - def __init__(self, - resource_name: str = None, - size: str = None, - storage_class: str = None, - modes: List[str] = None, - annotations: Dict[str, str] = None, - data_source=None, - volume_name=None, - generate_unique_name: bool = True, - **kwargs): - # Add size to attribute outputs - self.attribute_outputs = {"size": "{.status.capacity.storage}"} - - if "k8s_resource" in kwargs: - if resource_name or size or storage_class or modes or annotations: - raise ValueError("You cannot provide k8s_resource along with " - "other arguments.") - if not isinstance(kwargs["k8s_resource"], V1PersistentVolumeClaim): - raise ValueError("k8s_resource in VolumeOp must be an instance" - " of V1PersistentVolumeClaim") - super().__init__(**kwargs) - self.volume = PipelineVolume( - name=sanitize_k8s_name(self.name), pvc=self.outputs["name"]) - return - - if not size: - raise ValueError("Please provide size") - elif not match_serialized_pipelineparam(str(size)): - self._validate_memory_string(size) - - if data_source and not isinstance( - data_source, (str, PipelineParam, V1TypedLocalObjectReference)): - raise ValueError("data_source can be one of (str, PipelineParam, " - "V1TypedLocalObjectReference).") - if data_source and isinstance(data_source, (str, PipelineParam)): - data_source = V1TypedLocalObjectReference( - api_group="snapshot.storage.k8s.io", - kind="VolumeSnapshot", - name=data_source) - - # Set the k8s_resource - if not match_serialized_pipelineparam(str(resource_name)): - resource_name = sanitize_k8s_name(resource_name) - pvc_metadata = V1ObjectMeta( - name="{{workflow.name}}-%s" % - resource_name if generate_unique_name else resource_name, - annotations=annotations) - requested_resources = V1ResourceRequirements(requests={"storage": size}) - pvc_spec = V1PersistentVolumeClaimSpec( - access_modes=modes or VOLUME_MODE_RWM, - resources=requested_resources, - storage_class_name=storage_class, - data_source=data_source, - volume_name=volume_name) - k8s_resource = V1PersistentVolumeClaim( - api_version="v1", - kind="PersistentVolumeClaim", - metadata=pvc_metadata, - spec=pvc_spec) - - super().__init__( - k8s_resource=k8s_resource, - **kwargs, - ) - self.volume = PipelineVolume( - name=sanitize_k8s_name(self.name), pvc=self.outputs["name"]) - - def _validate_memory_string(self, memory_string): - """Validate a given string is valid for memory request or limit.""" - if re.match(r"^[0-9]+(E|Ei|P|Pi|T|Ti|G|Gi|M|Mi|K|Ki){0,1}$", - memory_string) is None: - raise ValueError("Invalid memory string. Should be an integer, " + - "or integer followed by one of " + - '"E|Ei|P|Pi|T|Ti|G|Gi|M|Mi|K|Ki"') diff --git a/sdk/python/kfp/deprecated/dsl/_volume_snapshot_op.py b/sdk/python/kfp/deprecated/dsl/_volume_snapshot_op.py deleted file mode 100644 index ae3be172ffb..00000000000 --- a/sdk/python/kfp/deprecated/dsl/_volume_snapshot_op.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Dict -from kubernetes.client.models import (V1Volume, V1TypedLocalObjectReference, - V1ObjectMeta) - -from ._resource_op import ResourceOp -from ._pipeline_param import match_serialized_pipelineparam, sanitize_k8s_name - - -class VolumeSnapshotOp(ResourceOp): - """Represents an op which will be translated into a resource template which - will be creating a VolumeSnapshot. - - TODO(https://github.com/kubeflow/pipelines/issues/4822): Determine the - stability level of this feature. - - Args: - resource_name: A desired name for the VolumeSnapshot which will be - created - pvc: The name of the PVC which will be snapshotted - snapshot_class: The snapshot class to use for the dynamically created - VolumeSnapshot - annotations: Annotations to be patched in the VolumeSnapshot - volume: An instance of V1Volume - kwargs: See :py:class:`kfp.dsl.ResourceOp` - - Raises: - ValueError: if k8s_resource is provided along with other arguments - if k8s_resource is not a VolumeSnapshot - if pvc and volume are None - if pvc and volume are not None - if volume does not reference a PVC - """ - - def __init__(self, - resource_name: str = None, - pvc: str = None, - snapshot_class: str = None, - annotations: Dict[str, str] = None, - volume: V1Volume = None, - api_version: str = "snapshot.storage.k8s.io/v1alpha1", - **kwargs): - # Add size to output params - self.attribute_outputs = {"size": "{.status.restoreSize}"} - # Add default success_condition if None provided - if "success_condition" not in kwargs: - kwargs["success_condition"] = "status.readyToUse == true" - - if "k8s_resource" in kwargs: - if resource_name or pvc or snapshot_class or annotations or volume: - raise ValueError("You cannot provide k8s_resource along with " - "other arguments.") - # TODO: Check if is VolumeSnapshot - super().__init__(**kwargs) - self.snapshot = V1TypedLocalObjectReference( - api_group="snapshot.storage.k8s.io", - kind="VolumeSnapshot", - name=self.outputs["name"]) - return - - if not (pvc or volume): - raise ValueError("You must provide a pvc or a volume.") - elif pvc and volume: - raise ValueError("You can't provide both pvc and volume.") - - source = None - deps = [] - if pvc: - source = V1TypedLocalObjectReference( - kind="PersistentVolumeClaim", name=pvc) - else: - if not hasattr(volume, "persistent_volume_claim"): - raise ValueError("The volume must be referencing a PVC.") - if hasattr(volume, - "dependent_names"): #TODO: Replace with type check - deps = list(volume.dependent_names) - source = V1TypedLocalObjectReference( - kind="PersistentVolumeClaim", - name=volume.persistent_volume_claim.claim_name) - - # Set the k8s_resource - # TODO: Use VolumeSnapshot - if not match_serialized_pipelineparam(str(resource_name)): - resource_name = sanitize_k8s_name(resource_name) - snapshot_metadata = V1ObjectMeta( - name="{{workflow.name}}-%s" % resource_name, - annotations=annotations) - k8s_resource = { - "apiVersion": api_version, - "kind": "VolumeSnapshot", - "metadata": snapshot_metadata, - "spec": { - "source": source - } - } - if snapshot_class: - k8s_resource["spec"]["snapshotClassName"] = snapshot_class - - super().__init__(k8s_resource=k8s_resource, **kwargs) - self.dependent_names.extend(deps) - self.snapshot = V1TypedLocalObjectReference( - api_group="snapshot.storage.k8s.io", - kind="VolumeSnapshot", - name=self.outputs["name"]) diff --git a/sdk/python/kfp/deprecated/dsl/artifact.py b/sdk/python/kfp/deprecated/dsl/artifact.py deleted file mode 100644 index f4867b15653..00000000000 --- a/sdk/python/kfp/deprecated/dsl/artifact.py +++ /dev/null @@ -1,215 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Base class for MLMD artifact in KFP SDK.""" - -from typing import Any, Optional - -from absl import logging -import importlib -import yaml - -from google.protobuf import json_format -from google.protobuf import struct_pb2 -from kfp.pipeline_spec import pipeline_spec_pb2 -from kfp.deprecated.dsl import serialization_utils -from kfp.deprecated.dsl import artifact_utils - -KFP_ARTIFACT_ONTOLOGY_MODULE = 'kfp.dsl.ontology_artifacts' -DEFAULT_ARTIFACT_SCHEMA = 'title: kfp.Artifact\ntype: object\nproperties:\n' - - -class Artifact(object): - """KFP Artifact Python class. - - Artifact Python class/object mainly serves following purposes in different - period of its lifecycle. - - 1. During compile time, users can use Artifact class to annotate I/O types of - their components. - 2. At runtime, Artifact objects provide helper function/utilities to access - the underlying RuntimeArtifact pb message, and provide additional layers - of validation to ensure type compatibility for fields specified in the - instance schema. - """ - - TYPE_NAME = "kfp.Artifact" - - # Initialization flag to support setattr / getattr behavior. - _initialized = False - - def __init__(self, instance_schema: Optional[str] = None): - """Constructs an instance of Artifact. - - Setups up self._metadata_fields to perform type checking and - initialize RuntimeArtifact. - """ - if self.__class__ == Artifact: - if not instance_schema: - raise ValueError( - 'The "instance_schema" argument must be set for Artifact.') - self._instance_schema = instance_schema - else: - if instance_schema: - raise ValueError( - 'The "instance_schema" argument must not be passed for Artifact \ - subclass: {}'.format(self.__class__)) - - # setup self._metadata_fields - self.TYPE_NAME, self._metadata_fields = artifact_utils.parse_schema( - self._instance_schema) - - # Instantiate a RuntimeArtifact pb message as the POD data structure. - self._artifact = pipeline_spec_pb2.RuntimeArtifact() - - # Stores the metadata for the Artifact. - self.metadata = {} - - self._artifact.type.CopyFrom( - pipeline_spec_pb2.ArtifactTypeSchema( - instance_schema=self._instance_schema)) - - self._initialized = True - - @property - def type_schema(self) -> str: - """Gets the instance_schema for this Artifact object.""" - return self._instance_schema - - def __getattr__(self, name: str) -> Any: - """Custom __getattr__ to allow access to artifact metadata.""" - - if name not in self._metadata_fields: - raise AttributeError( - 'No metadata field: {} in artifact.'.format(name)) - - return self.metadata[name] - - def __setattr__(self, name: str, value: Any): - """Custom __setattr__ to allow access to artifact metadata.""" - - if not self._initialized: - object.__setattr__(self, name, value) - return - - metadata_fields = {} - if self._metadata_fields: - metadata_fields = self._metadata_fields - - if name not in self._metadata_fields: - if (name in self.__dict__ or - any(name in c.__dict__ for c in self.__class__.mro())): - # Use any provided getter / setter if available. - object.__setattr__(self, name, value) - return - # In the case where we do not handle this via an explicit getter / - # setter, we assume that the user implied an artifact attribute store, - # and we raise an exception since such an attribute was not explicitly - # defined in the Artifact PROPERTIES dictionary. - raise AttributeError('Cannot set an unspecified metadata field:{} \ - on artifact. Only fields specified in instance schema can be \ - set.'.format(name)) - - # Type checking to be performed during serialization. - self.metadata[name] = value - - def _update_runtime_artifact(self): - """Verifies metadata is well-formed and updates artifact instance.""" - - artifact_utils.verify_schema_instance(self._instance_schema, - self.metadata) - - if len(self.metadata) != 0: - metadata_protobuf_struct = struct_pb2.Struct() - metadata_protobuf_struct.update(self.metadata) - self._artifact.metadata.CopyFrom(metadata_protobuf_struct) - - @property - def type(self): - return self.__class__ - - @property - def type_name(self): - return self.TYPE_NAME - - @property - def uri(self) -> str: - return self._artifact.uri - - @uri.setter - def uri(self, uri: str) -> None: - self._artifact.uri = uri - - @property - def name(self) -> str: - return self._artifact.name - - @name.setter - def name(self, name: str) -> None: - self._artifact.name = name - - @property - def runtime_artifact(self) -> pipeline_spec_pb2.RuntimeArtifact: - self._update_runtime_artifact() - return self._artifact - - @runtime_artifact.setter - def runtime_artifact(self, artifact: pipeline_spec_pb2.RuntimeArtifact): - self._artifact = artifact - - def serialize(self) -> str: - """Serializes an Artifact to JSON dict format.""" - self._update_runtime_artifact() - return json_format.MessageToJson(self._artifact, sort_keys=True) - - @classmethod - def get_artifact_type(cls) -> str: - """Gets the instance_schema according to the Python schema spec.""" - result_map = {'title': cls.TYPE_NAME, 'type': 'object'} - - return serialization_utils.yaml_dump(result_map) - - @classmethod - def get_ir_type(cls) -> pipeline_spec_pb2.ArtifactTypeSchema: - return pipeline_spec_pb2.ArtifactTypeSchema( - instance_schema=cls.get_artifact_type()) - - @classmethod - def get_from_runtime_artifact( - cls, artifact: pipeline_spec_pb2.RuntimeArtifact) -> Any: - """Deserializes an Artifact object from RuntimeArtifact message.""" - instance_schema = yaml.safe_load(artifact.type.instance_schema) - type_name = instance_schema['title'][len('kfp.'):] - result = None - try: - artifact_cls = getattr( - importlib.import_module(KFP_ARTIFACT_ONTOLOGY_MODULE), - type_name) - result = artifact_cls() - except (AttributeError, ImportError, ValueError) as err: - logging.warning('Failed to instantiate Ontology Artifact:{} \ - instance'.format(type_name)) - - if not result: - # Otherwise generate a generic Artifact object. - result = Artifact(instance_schema=artifact.type.instance_schema) - result.runtime_artifact = artifact - result.metadata = json_format.MessageToDict(artifact.metadata) - return result - - @classmethod - def deserialize(cls, data: str) -> Any: - """Deserializes an Artifact object from JSON dict.""" - artifact = pipeline_spec_pb2.RuntimeArtifact() - json_format.Parse(data, artifact, ignore_unknown_fields=True) - return cls.get_from_runtime_artifact(artifact) diff --git a/sdk/python/kfp/deprecated/dsl/artifact_utils.py b/sdk/python/kfp/deprecated/dsl/artifact_utils.py deleted file mode 100644 index 9433b722493..00000000000 --- a/sdk/python/kfp/deprecated/dsl/artifact_utils.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Helper utils used in artifact and ontology_artifact classes.""" - -from typing import Any, Dict, Tuple - -import enum -import jsonschema -import os -import yaml - - -class SchemaFieldType(enum.Enum): - """Supported Schema field types.""" - NUMBER = 'number' - INTEGER = 'integer' - STRING = 'string' - BOOL = 'bool' - OBJECT = 'object' - ARRAY = 'array' - - -def parse_schema(yaml_schema: str) -> Tuple[str, Dict[str, SchemaFieldType]]: - """Parses yaml schema. - - Ensures that schema is well-formed and returns dictionary of properties and - its type for type-checking. - - Args: - yaml_schema: Yaml schema to be parsed. - - Returns: - str: Title set in the schema. - Dict: Property name to SchemaFieldType enum. - - Raises: - ValueError if title field is not set in schema or an - unsupported(i.e. not defined in SchemaFieldType) - type is specified for the field. - """ - - schema = yaml.full_load(yaml_schema) - if 'title' not in schema.keys(): - raise ValueError('Invalid _schema, title must be set. \ - Got: {}'.format(yaml_schema)) - - title = schema['title'] - properties = {} - if 'properties' in schema.keys(): - schema_properties = schema['properties'] or {} - for property_name, property_def in schema_properties.items(): - try: - properties[property_name] = SchemaFieldType( - property_def['type']) - except ValueError: - raise ValueError('Unsupported type:{} specified for field: {} \ - in schema'.format(property_def['type'], property_name)) - - return title, properties - - -def verify_schema_instance(schema: str, instance: Dict[str, Any]): - """Verifies instnace is well-formed against the schema. - - Args: - schema: Schema to use for verification. - instance: Object represented as Dict to be verified. - - Raises: - RuntimeError if schema is not well-formed or instance is invalid against - the schema. - """ - - if len(instance) == 0: - return - - try: - jsonschema.validate(instance=instance, schema=yaml.full_load(schema)) - except jsonschema.exceptions.SchemaError: - raise RuntimeError('Invalid schema schema: {} used for \ - verification'.format(schema)) - except jsonschema.exceptions.ValidationError: - raise RuntimeError('Invalid values set: {} in object for schema: \ - {}'.format(instance, schema)) - - -def read_schema_file(schema_file: str) -> str: - """Reads yamls schema from type_scheams folder. - - Args: - schema_file: Name of the file to read schema from. - - Returns: - Read schema from the schema file. - """ - schema_file_path = os.path.join( - os.path.dirname(__file__), 'type_schemas', schema_file) - - with open(schema_file_path) as schema_file: - return schema_file.read() diff --git a/sdk/python/kfp/deprecated/dsl/component_spec.py b/sdk/python/kfp/deprecated/dsl/component_spec.py deleted file mode 100644 index 9ef580ecb95..00000000000 --- a/sdk/python/kfp/deprecated/dsl/component_spec.py +++ /dev/null @@ -1,431 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Functions for creating IR ComponentSpec instance.""" - -from typing import List, Optional, Tuple, Union - -from kfp.deprecated.components import _structures as structures -from kfp.deprecated.dsl import _for_loop, _pipeline_param, dsl_utils -from kfp.pipeline_spec import pipeline_spec_pb2 -from kfp.deprecated.dsl import type_utils - - -def additional_input_name_for_pipelineparam( - param_or_name: Union[_pipeline_param.PipelineParam, str]) -> str: - """Gets the name for an additional (compiler-injected) input.""" - - # Adding a prefix to avoid (reduce chance of) name collision between the - # original component inputs and the injected input. - return 'pipelineparam--' + ( - param_or_name.full_name if isinstance( - param_or_name, _pipeline_param.PipelineParam) else param_or_name) - - -def _exclude_loop_arguments_variables( - param_or_name: Union[_pipeline_param.PipelineParam, str] -) -> Tuple[str, Optional[str]]: - """Gets the pipeline param name excluding loop argument variables. - - Args: - param: The pipeline param object which may or may not be a loop argument. - - Returns: - A tuple of the name of the pipeline param without loop arguments subvar name - and the subvar name is found. - """ - if isinstance(param_or_name, _pipeline_param.PipelineParam): - param_name = param_or_name.full_name - else: - param_name = param_or_name - - subvar_name = None - # Special handling for loop arguments. - # In case of looping over a list of maps, each subvar (a key in the map) - # referencing yields a pipeline param. For example: - # - some-param-loop-item: referencing the whole map - # - some-param-loop-item-subvar-key1: referencing key1 in the map. - # Because of the way IR is designed to support looping over a subvar (using - # `parameter_expression_selector`), we don't create inputs for subvariables. - # So if we see a pipeline param named 'some-param-loop-item-subvar-key1', - # we build the component_spec/task_spec inputs using 'some-param-loop-item' - # (without the subvar suffix). - if _for_loop.LoopArguments.name_is_loop_argument(param_name): - # Subvar pipeline params may not have types, defaults to string type. - if isinstance(param_or_name, _pipeline_param.PipelineParam): - param_or_name.param_type = param_or_name.param_type or 'String' - loop_args_name_and_var_name = ( - _for_loop.LoopArgumentVariable - .parse_loop_args_name_and_this_var_name(param_name)) - if loop_args_name_and_var_name: - param_name = loop_args_name_and_var_name[0] - subvar_name = loop_args_name_and_var_name[1] - return (param_name, subvar_name) - - -def build_component_spec_from_structure( - component_spec: structures.ComponentSpec, - executor_label: str, - actual_inputs: List[str], -) -> pipeline_spec_pb2.ComponentSpec: - """Builds an IR ComponentSpec instance from structures.ComponentSpec. - - Args: - component_spec: The structure component spec. - executor_label: The executor label. - actual_inputs: The actual arugments passed to the task. This is used as a - short term workaround to support optional inputs in component spec IR. - - Returns: - An instance of IR ComponentSpec. - """ - result = pipeline_spec_pb2.ComponentSpec() - result.executor_label = executor_label - - for input_spec in component_spec.inputs or []: - # skip inputs not present - if input_spec.name not in actual_inputs: - continue - if type_utils.is_parameter_type(input_spec.type): - result.input_definitions.parameters[ - input_spec.name].parameter_type = type_utils.get_parameter_type( - input_spec.type) - else: - result.input_definitions.artifacts[ - input_spec.name].artifact_type.CopyFrom( - type_utils.get_artifact_type_schema(input_spec.type)) - - for output_spec in component_spec.outputs or []: - if type_utils.is_parameter_type(output_spec.type): - result.output_definitions.parameters[ - output_spec - .name].parameter_type = type_utils.get_parameter_type( - output_spec.type) - else: - result.output_definitions.artifacts[ - output_spec.name].artifact_type.CopyFrom( - type_utils.get_artifact_type_schema(output_spec.type)) - - return result - - -def build_component_inputs_spec( - component_spec: pipeline_spec_pb2.ComponentSpec, - pipeline_params: List[_pipeline_param.PipelineParam], - is_root_component: bool, -) -> None: - """Builds component inputs spec from pipeline params. - - Args: - component_spec: The component spec to fill in its inputs spec. - pipeline_params: The list of pipeline params. - is_root_component: Whether the component is the root. - """ - for param in pipeline_params: - param_name = param.full_name - if _for_loop.LoopArguments.name_is_loop_argument(param_name): - param.param_type = param.param_type or 'String' - - input_name = ( - param_name if is_root_component else - additional_input_name_for_pipelineparam(param_name)) - - if type_utils.is_parameter_type(param.param_type): - component_spec.input_definitions.parameters[ - input_name].parameter_type = type_utils.get_parameter_type( - param.param_type) - elif input_name not in getattr(component_spec.input_definitions, - 'parameters', []): - component_spec.input_definitions.artifacts[ - input_name].artifact_type.CopyFrom( - type_utils.get_artifact_type_schema(param.param_type)) - - -def build_component_outputs_spec( - component_spec: pipeline_spec_pb2.ComponentSpec, - pipeline_params: List[_pipeline_param.PipelineParam], -) -> None: - """Builds component outputs spec from pipeline params. - - Args: - component_spec: The component spec to fill in its outputs spec. - pipeline_params: The list of pipeline params. - """ - for param in pipeline_params or []: - output_name = param.full_name - if type_utils.is_parameter_type(param.param_type): - component_spec.output_definitions.parameters[ - output_name].parameter_type = type_utils.get_parameter_type( - param.param_type) - elif output_name not in getattr(component_spec.output_definitions, - 'parameters', []): - component_spec.output_definitions.artifacts[ - output_name].artifact_type.CopyFrom( - type_utils.get_artifact_type_schema(param.param_type)) - - -def build_task_inputs_spec( - task_spec: pipeline_spec_pb2.PipelineTaskSpec, - pipeline_params: List[_pipeline_param.PipelineParam], - tasks_in_current_dag: List[str], - is_parent_component_root: bool, -) -> None: - """Builds task inputs spec from pipeline params. - - Args: - task_spec: The task spec to fill in its inputs spec. - pipeline_params: The list of pipeline params. - tasks_in_current_dag: The list of tasks names for tasks in the same dag. - is_parent_component_root: Whether the task is in the root component. - """ - for param in pipeline_params or []: - - param_full_name, subvar_name = _exclude_loop_arguments_variables(param) - input_name = additional_input_name_for_pipelineparam(param.full_name) - - param_name = param.name - if subvar_name: - task_spec.inputs.parameters[ - input_name].parameter_expression_selector = ( - 'parseJson(string_value)["{}"]'.format(subvar_name)) - param_name = _for_loop.LoopArguments.remove_loop_item_base_name( - _exclude_loop_arguments_variables(param_name)[0]) - - if type_utils.is_parameter_type(param.param_type): - if param.op_name and dsl_utils.sanitize_task_name( - param.op_name) in tasks_in_current_dag: - task_spec.inputs.parameters[ - input_name].task_output_parameter.producer_task = ( - dsl_utils.sanitize_task_name(param.op_name)) - task_spec.inputs.parameters[ - input_name].task_output_parameter.output_parameter_key = ( - param_name) - else: - task_spec.inputs.parameters[ - input_name].component_input_parameter = ( - param_full_name if is_parent_component_root else - additional_input_name_for_pipelineparam(param_full_name) - ) - else: - if param.op_name and dsl_utils.sanitize_task_name( - param.op_name) in tasks_in_current_dag: - task_spec.inputs.artifacts[ - input_name].task_output_artifact.producer_task = ( - dsl_utils.sanitize_task_name(param.op_name)) - task_spec.inputs.artifacts[ - input_name].task_output_artifact.output_artifact_key = ( - param_name) - else: - task_spec.inputs.artifacts[ - input_name].component_input_artifact = ( - param_full_name - if is_parent_component_root else input_name) - - -def update_task_inputs_spec( - task_spec: pipeline_spec_pb2.PipelineTaskSpec, - parent_component_inputs: pipeline_spec_pb2.ComponentInputsSpec, - pipeline_params: List[_pipeline_param.PipelineParam], - tasks_in_current_dag: List[str], - input_parameters_in_current_dag: List[str], - input_artifacts_in_current_dag: List[str], -) -> None: - """Updates task inputs spec. - - A task input may reference an output outside its immediate DAG. - For instance:: - - random_num = random_num_op(...) - with dsl.Condition(random_num.output > 5): - print_op('%s > 5' % random_num.output) - - In this example, `dsl.Condition` forms a sub-DAG with one task from `print_op` - inside the sub-DAG. The task of `print_op` references output from `random_num` - task, which is outside the sub-DAG. When compiling to IR, such cross DAG - reference is disallowed. So we need to "punch a hole" in the sub-DAG to make - the input available in the sub-DAG component inputs if it's not already there, - Next, we can call this method to fix the tasks inside the sub-DAG to make them - reference the component inputs instead of directly referencing the original - producer task. - - Args: - task_spec: The task spec to fill in its inputs spec. - parent_component_inputs: The input spec of the task's parent component. - pipeline_params: The list of pipeline params. - tasks_in_current_dag: The list of tasks names for tasks in the same dag. - input_parameters_in_current_dag: The list of input parameters in the DAG - component. - input_artifacts_in_current_dag: The list of input artifacts in the DAG - component. - """ - if not hasattr(task_spec, 'inputs'): - return - - for input_name in getattr(task_spec.inputs, 'parameters', []): - - if task_spec.inputs.parameters[input_name].WhichOneof( - 'kind') == 'task_output_parameter' and ( - task_spec.inputs.parameters[input_name] - .task_output_parameter.producer_task - not in tasks_in_current_dag): - - param = _pipeline_param.PipelineParam( - name=task_spec.inputs.parameters[input_name] - .task_output_parameter.output_parameter_key, - op_name=task_spec.inputs.parameters[input_name] - .task_output_parameter.producer_task) - - component_input_parameter = ( - additional_input_name_for_pipelineparam(param.full_name)) - - if component_input_parameter in parent_component_inputs.parameters: - task_spec.inputs.parameters[ - input_name].component_input_parameter = component_input_parameter - continue - - # The input not found in parent's component input definitions - # This could happen because of loop arguments variables - param_name, subvar_name = _exclude_loop_arguments_variables(param) - if subvar_name: - task_spec.inputs.parameters[ - input_name].parameter_expression_selector = ( - 'parseJson(string_value)["{}"]'.format(subvar_name)) - - component_input_parameter = ( - additional_input_name_for_pipelineparam(param_name)) - - assert component_input_parameter in parent_component_inputs.parameters, \ - 'component_input_parameter: {} not found. All inputs: {}'.format( - component_input_parameter, parent_component_inputs) - - task_spec.inputs.parameters[ - input_name].component_input_parameter = component_input_parameter - - elif task_spec.inputs.parameters[input_name].WhichOneof( - 'kind') == 'component_input_parameter': - - component_input_parameter = ( - task_spec.inputs.parameters[input_name] - .component_input_parameter) - - if component_input_parameter in parent_component_inputs.parameters: - continue - - if additional_input_name_for_pipelineparam( - component_input_parameter - ) in parent_component_inputs.parameters: - task_spec.inputs.parameters[ - input_name].component_input_parameter = ( - additional_input_name_for_pipelineparam( - component_input_parameter)) - continue - - # The input not found in parent's component input definitions - # This could happen because of loop arguments variables - component_input_parameter, subvar_name = _exclude_loop_arguments_variables( - component_input_parameter) - - if subvar_name: - task_spec.inputs.parameters[ - input_name].parameter_expression_selector = ( - 'parseJson(string_value)["{}"]'.format(subvar_name)) - - if component_input_parameter not in input_parameters_in_current_dag: - component_input_parameter = ( - additional_input_name_for_pipelineparam( - component_input_parameter)) - - if component_input_parameter not in parent_component_inputs.parameters: - component_input_parameter = ( - additional_input_name_for_pipelineparam( - component_input_parameter)) - assert component_input_parameter in parent_component_inputs.parameters, \ - 'component_input_parameter: {} not found. All inputs: {}'.format( - component_input_parameter, parent_component_inputs) - - task_spec.inputs.parameters[ - input_name].component_input_parameter = component_input_parameter - - for input_name in getattr(task_spec.inputs, 'artifacts', []): - - if task_spec.inputs.artifacts[input_name].WhichOneof( - 'kind') == 'task_output_artifact' and ( - task_spec.inputs.artifacts[input_name].task_output_artifact - .producer_task not in tasks_in_current_dag): - - param = _pipeline_param.PipelineParam( - name=task_spec.inputs.artifacts[input_name].task_output_artifact - .output_artifact_key, - op_name=task_spec.inputs.artifacts[input_name] - .task_output_artifact.producer_task) - component_input_artifact = ( - additional_input_name_for_pipelineparam(param)) - assert component_input_artifact in parent_component_inputs.artifacts, \ - 'component_input_artifact: {} not found. All inputs: {}'.format( - component_input_artifact, parent_component_inputs) - - task_spec.inputs.artifacts[ - input_name].component_input_artifact = component_input_artifact - - elif task_spec.inputs.artifacts[input_name].WhichOneof( - 'kind') == 'component_input_artifact': - - component_input_artifact = ( - task_spec.inputs.artifacts[input_name].component_input_artifact) - - if component_input_artifact not in input_artifacts_in_current_dag: - component_input_artifact = ( - additional_input_name_for_pipelineparam( - task_spec.inputs.artifacts[input_name] - .component_input_artifact)) - assert component_input_artifact in parent_component_inputs.artifacts, \ - 'component_input_artifact: {} not found. All inputs: {}'.format( - component_input_artifact, parent_component_inputs) - - task_spec.inputs.artifacts[ - input_name].component_input_artifact = component_input_artifact - - -def pop_input_from_component_spec( - component_spec: pipeline_spec_pb2.ComponentSpec, - input_name: str, -) -> None: - """Removes an input from component spec input_definitions. - - Args: - component_spec: The component spec to update in place. - input_name: The name of the input, which could be an artifact or paremeter. - """ - component_spec.input_definitions.artifacts.pop(input_name) - component_spec.input_definitions.parameters.pop(input_name) - - if component_spec.input_definitions == pipeline_spec_pb2.ComponentInputsSpec( - ): - component_spec.ClearField('input_definitions') - - -def pop_input_from_task_spec( - task_spec: pipeline_spec_pb2.PipelineTaskSpec, - input_name: str, -) -> None: - """Removes an input from task spec inputs. - - Args: - task_spec: The pipeline task spec to update in place. - input_name: The name of the input, which could be an artifact or paremeter. - """ - task_spec.inputs.artifacts.pop(input_name) - task_spec.inputs.parameters.pop(input_name) - - if task_spec.inputs == pipeline_spec_pb2.TaskInputsSpec(): - task_spec.ClearField('inputs') diff --git a/sdk/python/kfp/deprecated/dsl/component_spec_test.py b/sdk/python/kfp/deprecated/dsl/component_spec_test.py deleted file mode 100644 index 856615c3202..00000000000 --- a/sdk/python/kfp/deprecated/dsl/component_spec_test.py +++ /dev/null @@ -1,631 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for kfp.dsl.component_spec.""" - -from absl.testing import parameterized - -from kfp.deprecated.components import _structures as structures -from kfp.deprecated.dsl import _pipeline_param -from kfp.deprecated.dsl import component_spec as dsl_component_spec -from kfp.pipeline_spec import pipeline_spec_pb2 - -from google.protobuf import json_format - - -class ComponentSpecTest(parameterized.TestCase): - - TEST_PIPELINE_PARAMS = [ - _pipeline_param.PipelineParam( - name='output1', param_type='Dataset', op_name='op-1'), - _pipeline_param.PipelineParam( - name='output2', param_type='Integer', op_name='op-2'), - _pipeline_param.PipelineParam( - name='output3', param_type='Model', op_name='op-3'), - _pipeline_param.PipelineParam( - name='output4', param_type='Double', op_name='op-4'), - _pipeline_param.PipelineParam( - name='arg_input', param_type='String', op_name=None), - ] - - def setUp(self): - self.maxDiff = None - - def test_build_component_spec_from_structure(self): - structure_component_spec = structures.ComponentSpec( - name='component1', - description='component1 desc', - inputs=[ - structures.InputSpec( - name='input1', description='input1 desc', type='Dataset'), - structures.InputSpec( - name='input2', description='input2 desc', type='String'), - structures.InputSpec( - name='input3', description='input3 desc', type='Integer'), - structures.InputSpec( - name='input4', description='optional inputs', - optional=True), - ], - outputs=[ - structures.OutputSpec( - name='output1', description='output1 desc', type='Model') - ]) - expected_dict = { - 'inputDefinitions': { - 'artifacts': { - 'input1': { - 'artifactType': { - 'schemaTitle': 'system.Dataset', - 'schemaVersion': '0.0.1' - } - } - }, - 'parameters': { - 'input2': { - 'parameterType': 'STRING' - }, - 'input3': { - 'parameterType': 'NUMBER_INTEGER' - } - } - }, - 'outputDefinitions': { - 'artifacts': { - 'output1': { - 'artifactType': { - 'schemaTitle': 'system.Model', - 'schemaVersion': '0.0.1' - } - } - } - }, - 'executorLabel': 'exec-component1' - } - expected_spec = pipeline_spec_pb2.ComponentSpec() - json_format.ParseDict(expected_dict, expected_spec) - - component_spec = ( - dsl_component_spec.build_component_spec_from_structure( - component_spec=structure_component_spec, - executor_label='exec-component1', - actual_inputs=['input1', 'input2', 'input3'], - )) - - self.assertEqual(expected_spec, component_spec) - - @parameterized.parameters( - { - 'is_root_component': True, - 'expected_result': { - 'inputDefinitions': { - 'artifacts': { - 'input1': { - 'artifactType': { - 'schemaTitle': 'system.Dataset', - 'schemaVersion': '0.0.1' - } - } - }, - 'parameters': { - 'input2': { - 'parameterType': 'NUMBER_INTEGER' - }, - 'input3': { - 'parameterType': 'STRING' - }, - 'input4': { - 'parameterType': 'NUMBER_DOUBLE' - } - } - } - } - }, - { - 'is_root_component': False, - 'expected_result': { - 'inputDefinitions': { - 'artifacts': { - 'pipelineparam--input1': { - 'artifactType': { - 'schemaTitle': 'system.Dataset', - 'schemaVersion': '0.0.1' - } - } - }, - 'parameters': { - 'pipelineparam--input2': { - 'parameterType': 'NUMBER_INTEGER' - }, - 'pipelineparam--input3': { - 'parameterType': 'STRING' - }, - 'pipelineparam--input4': { - 'parameterType': 'NUMBER_DOUBLE' - } - } - } - } - }, - ) - def test_build_component_inputs_spec(self, is_root_component, - expected_result): - pipeline_params = [ - _pipeline_param.PipelineParam(name='input1', param_type='Dataset'), - _pipeline_param.PipelineParam(name='input2', param_type='Integer'), - _pipeline_param.PipelineParam(name='input3', param_type='String'), - _pipeline_param.PipelineParam(name='input4', param_type='Float'), - ] - expected_spec = pipeline_spec_pb2.ComponentSpec() - json_format.ParseDict(expected_result, expected_spec) - - component_spec = pipeline_spec_pb2.ComponentSpec() - dsl_component_spec.build_component_inputs_spec(component_spec, - pipeline_params, - is_root_component) - - self.assertEqual(expected_spec, component_spec) - - def test_build_component_outputs_spec(self): - pipeline_params = [ - _pipeline_param.PipelineParam(name='output1', param_type='Dataset'), - _pipeline_param.PipelineParam(name='output2', param_type='Integer'), - _pipeline_param.PipelineParam(name='output3', param_type='String'), - _pipeline_param.PipelineParam(name='output4', param_type='Float'), - ] - expected_dict = { - 'outputDefinitions': { - 'artifacts': { - 'output1': { - 'artifactType': { - 'schemaTitle': 'system.Dataset', - 'schemaVersion': '0.0.1' - } - } - }, - 'parameters': { - 'output2': { - 'parameterType': 'NUMBER_INTEGER' - }, - 'output3': { - 'parameterType': 'STRING' - }, - 'output4': { - 'parameterType': 'NUMBER_DOUBLE' - } - } - } - } - expected_spec = pipeline_spec_pb2.ComponentSpec() - json_format.ParseDict(expected_dict, expected_spec) - - component_spec = pipeline_spec_pb2.ComponentSpec() - dsl_component_spec.build_component_outputs_spec(component_spec, - pipeline_params) - - self.assertEqual(expected_spec, component_spec) - - @parameterized.parameters( - { - 'is_parent_component_root': True, - 'expected_result': { - 'inputs': { - 'artifacts': { - 'pipelineparam--op-1-output1': { - 'taskOutputArtifact': { - 'producerTask': 'op-1', - 'outputArtifactKey': 'output1' - } - }, - 'pipelineparam--op-3-output3': { - 'componentInputArtifact': 'op-3-output3' - } - }, - 'parameters': { - 'pipelineparam--op-2-output2': { - 'taskOutputParameter': { - 'producerTask': 'op-2', - 'outputParameterKey': 'output2' - } - }, - 'pipelineparam--op-4-output4': { - 'componentInputParameter': 'op-4-output4' - }, - 'pipelineparam--arg_input': { - 'componentInputParameter': 'arg_input' - } - } - } - } - }, - { - 'is_parent_component_root': False, - 'expected_result': { - 'inputs': { - 'artifacts': { - 'pipelineparam--op-1-output1': { - 'taskOutputArtifact': { - 'producerTask': 'op-1', - 'outputArtifactKey': 'output1' - } - }, - 'pipelineparam--op-3-output3': { - 'componentInputArtifact': - 'pipelineparam--op-3-output3' - } - }, - 'parameters': { - 'pipelineparam--op-2-output2': { - 'taskOutputParameter': { - 'producerTask': 'op-2', - 'outputParameterKey': 'output2' - } - }, - 'pipelineparam--op-4-output4': { - 'componentInputParameter': - 'pipelineparam--op-4-output4' - }, - 'pipelineparam--arg_input': { - 'componentInputParameter': - 'pipelineparam--arg_input' - } - } - } - } - }, - ) - def test_build_task_inputs_spec(self, is_parent_component_root, - expected_result): - pipeline_params = self.TEST_PIPELINE_PARAMS - tasks_in_current_dag = ['op-1', 'op-2'] - expected_spec = pipeline_spec_pb2.PipelineTaskSpec() - json_format.ParseDict(expected_result, expected_spec) - - task_spec = pipeline_spec_pb2.PipelineTaskSpec() - dsl_component_spec.build_task_inputs_spec(task_spec, pipeline_params, - tasks_in_current_dag, - is_parent_component_root) - - self.assertEqual(expected_spec, task_spec) - - @parameterized.parameters( - { - 'original_task_spec': {}, - 'parent_component_inputs': {}, - 'tasks_in_current_dag': [], - 'input_parameters_in_current_dag': [], - 'input_artifacts_in_current_dag': [], - 'expected_result': {}, - }, - { # Depending on tasks & inputs within the current DAG. - 'original_task_spec': { - 'inputs': { - 'artifacts': { - 'pipelineparam--op-1-output1': { - 'taskOutputArtifact': { - 'producerTask': 'op-1', - 'outputArtifactKey': 'output1' - } - }, - 'artifact1': { - 'componentInputArtifact': 'artifact1' - }, - }, - 'parameters': { - 'pipelineparam--op-2-output2': { - 'taskOutputParameter': { - 'producerTask': 'op-2', - 'outputParameterKey': 'output2' - } - }, - 'param1': { - 'componentInputParameter': 'param1' - }, - } - } - }, - 'parent_component_inputs': { - 'artifacts': { - 'artifact1': { - 'artifactType': { - 'instanceSchema': 'dummy_schema' - } - }, - }, - 'parameters': { - 'param1': { - 'parameterType': 'STRING' - }, - } - }, - 'tasks_in_current_dag': ['op-1', 'op-2'], - 'input_parameters_in_current_dag': ['param1'], - 'input_artifacts_in_current_dag': ['artifact1'], - 'expected_result': { - 'inputs': { - 'artifacts': { - 'pipelineparam--op-1-output1': { - 'taskOutputArtifact': { - 'producerTask': 'op-1', - 'outputArtifactKey': 'output1' - } - }, - 'artifact1': { - 'componentInputArtifact': 'artifact1' - }, - }, - 'parameters': { - 'pipelineparam--op-2-output2': { - 'taskOutputParameter': { - 'producerTask': 'op-2', - 'outputParameterKey': 'output2' - } - }, - 'param1': { - 'componentInputParameter': 'param1' - }, - } - } - }, - }, - { # Depending on tasks and inputs not available in the current DAG. - 'original_task_spec': { - 'inputs': { - 'artifacts': { - 'pipelineparam--op-1-output1': { - 'taskOutputArtifact': { - 'producerTask': 'op-1', - 'outputArtifactKey': 'output1' - } - }, - 'artifact1': { - 'componentInputArtifact': 'artifact1' - }, - }, - 'parameters': { - 'pipelineparam--op-2-output2': { - 'taskOutputParameter': { - 'producerTask': 'op-2', - 'outputParameterKey': 'output2' - } - }, - 'param1': { - 'componentInputParameter': 'param1' - }, - } - } - }, - 'parent_component_inputs': { - 'artifacts': { - 'pipelineparam--op-1-output1': { - 'artifactType': { - 'instanceSchema': 'dummy_schema' - } - }, - 'pipelineparam--artifact1': { - 'artifactType': { - 'instanceSchema': 'dummy_schema' - } - }, - }, - 'parameters': { - 'pipelineparam--op-2-output2' : { - 'parameterType': 'NUMBER_INTEGER' - }, - 'pipelineparam--param1': { - 'parameterType': 'STRING' - }, - } - }, - 'tasks_in_current_dag': ['op-3'], - 'input_parameters_in_current_dag': ['pipelineparam--op-2-output2', 'pipelineparam--param1'], - 'input_artifacts_in_current_dag': ['pipelineparam--op-1-output1', 'pipelineparam--artifact1'], - 'expected_result': { - 'inputs': { - 'artifacts': { - 'pipelineparam--op-1-output1': { - 'componentInputArtifact': - 'pipelineparam--op-1-output1' - }, - 'artifact1': { - 'componentInputArtifact': 'pipelineparam--artifact1' - }, - }, - 'parameters': { - 'pipelineparam--op-2-output2': { - 'componentInputParameter': - 'pipelineparam--op-2-output2' - }, - 'param1': { - 'componentInputParameter': 'pipelineparam--param1' - }, - } - } - }, - }, - ) - def test_update_task_inputs_spec(self, original_task_spec, - parent_component_inputs, - tasks_in_current_dag, - input_parameters_in_current_dag, - input_artifacts_in_current_dag, - expected_result): - pipeline_params = self.TEST_PIPELINE_PARAMS - - expected_spec = pipeline_spec_pb2.PipelineTaskSpec() - json_format.ParseDict(expected_result, expected_spec) - - task_spec = pipeline_spec_pb2.PipelineTaskSpec() - json_format.ParseDict(original_task_spec, task_spec) - parent_component_inputs_spec = pipeline_spec_pb2.ComponentInputsSpec() - json_format.ParseDict(parent_component_inputs, - parent_component_inputs_spec) - dsl_component_spec.update_task_inputs_spec( - task_spec, parent_component_inputs_spec, pipeline_params, - tasks_in_current_dag, input_parameters_in_current_dag, - input_artifacts_in_current_dag) - - self.assertEqual(expected_spec, task_spec) - - def test_pop_input_from_component_spec(self): - component_spec = pipeline_spec_pb2.ComponentSpec( - executor_label='exec-component1') - - component_spec.input_definitions.artifacts[ - 'input1'].artifact_type.schema_title = 'system.Dataset' - component_spec.input_definitions.parameters[ - 'input2'].parameter_type = pipeline_spec_pb2.ParameterType.STRING - component_spec.input_definitions.parameters[ - 'input3'].parameter_type = pipeline_spec_pb2.ParameterType.NUMBER_DOUBLE - - # pop an artifact, and there're other inputs left - dsl_component_spec.pop_input_from_component_spec( - component_spec, 'input1') - expected_dict = { - 'inputDefinitions': { - 'parameters': { - 'input2': { - 'parameterType': 'STRING' - }, - 'input3': { - 'parameterType': 'NUMBER_DOUBLE' - } - } - }, - 'executorLabel': 'exec-component1' - } - expected_spec = pipeline_spec_pb2.ComponentSpec() - json_format.ParseDict(expected_dict, expected_spec) - self.assertEqual(expected_spec, component_spec) - - # pop an parameter, and there're other inputs left - dsl_component_spec.pop_input_from_component_spec( - component_spec, 'input2') - expected_dict = { - 'inputDefinitions': { - 'parameters': { - 'input3': { - 'parameterType': 'NUMBER_DOUBLE' - } - } - }, - 'executorLabel': 'exec-component1' - } - expected_spec = pipeline_spec_pb2.ComponentSpec() - json_format.ParseDict(expected_dict, expected_spec) - self.assertEqual(expected_spec, component_spec) - - # pop the last input, expect no inputDefinitions - dsl_component_spec.pop_input_from_component_spec( - component_spec, 'input3') - expected_dict = {'executorLabel': 'exec-component1'} - expected_spec = pipeline_spec_pb2.ComponentSpec() - json_format.ParseDict(expected_dict, expected_spec) - self.assertEqual(expected_spec, component_spec) - - # pop an input that doesn't exist, expect no-op. - dsl_component_spec.pop_input_from_component_spec( - component_spec, 'input4') - self.assertEqual(expected_spec, component_spec) - - def test_pop_input_from_task_spec(self): - task_spec = pipeline_spec_pb2.PipelineTaskSpec() - task_spec.component_ref.name = 'comp-component1' - task_spec.inputs.artifacts[ - 'input1'].task_output_artifact.producer_task = 'op-1' - task_spec.inputs.artifacts[ - 'input1'].task_output_artifact.output_artifact_key = 'output1' - task_spec.inputs.parameters[ - 'input2'].task_output_parameter.producer_task = 'op-2' - task_spec.inputs.parameters[ - 'input2'].task_output_parameter.output_parameter_key = 'output2' - task_spec.inputs.parameters[ - 'input3'].component_input_parameter = 'op3-output3' - - # pop an parameter, and there're other inputs left - dsl_component_spec.pop_input_from_task_spec(task_spec, 'input3') - expected_dict = { - 'inputs': { - 'artifacts': { - 'input1': { - 'taskOutputArtifact': { - 'producerTask': 'op-1', - 'outputArtifactKey': 'output1' - } - } - }, - 'parameters': { - 'input2': { - 'taskOutputParameter': { - 'producerTask': 'op-2', - 'outputParameterKey': 'output2' - } - } - } - }, - 'component_ref': { - 'name': 'comp-component1' - } - } - expected_spec = pipeline_spec_pb2.PipelineTaskSpec() - json_format.ParseDict(expected_dict, expected_spec) - self.assertEqual(expected_spec, task_spec) - - # pop an artifact, and there're other inputs left - dsl_component_spec.pop_input_from_task_spec(task_spec, 'input1') - expected_dict = { - 'inputs': { - 'parameters': { - 'input2': { - 'taskOutputParameter': { - 'producerTask': 'op-2', - 'outputParameterKey': 'output2' - } - } - } - }, - 'component_ref': { - 'name': 'comp-component1' - } - } - expected_spec = pipeline_spec_pb2.PipelineTaskSpec() - json_format.ParseDict(expected_dict, expected_spec) - self.assertEqual(expected_spec, task_spec) - - # pop the last input, expect no inputDefinitions - dsl_component_spec.pop_input_from_task_spec(task_spec, 'input2') - expected_dict = {'component_ref': {'name': 'comp-component1'}} - expected_spec = pipeline_spec_pb2.PipelineTaskSpec() - json_format.ParseDict(expected_dict, expected_spec) - self.assertEqual(expected_spec, task_spec) - - # pop an input that doesn't exist, expect no-op. - dsl_component_spec.pop_input_from_task_spec(task_spec, 'input4') - self.assertEqual(expected_spec, task_spec) - - def test_additional_input_name_for_pipelineparam(self): - self.assertEqual( - 'pipelineparam--op1-param1', - dsl_component_spec.additional_input_name_for_pipelineparam( - _pipeline_param.PipelineParam(name='param1', op_name='op1'))) - self.assertEqual( - 'pipelineparam--param2', - dsl_component_spec.additional_input_name_for_pipelineparam( - _pipeline_param.PipelineParam(name='param2'))) - self.assertEqual( - 'pipelineparam--param3', - dsl_component_spec.additional_input_name_for_pipelineparam( - 'param3')) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/dsl/data_passing_methods.py b/sdk/python/kfp/deprecated/dsl/data_passing_methods.py deleted file mode 100644 index 7ad6460d962..00000000000 --- a/sdk/python/kfp/deprecated/dsl/data_passing_methods.py +++ /dev/null @@ -1,27 +0,0 @@ -import kubernetes -from kubernetes.client.models import V1Volume - - -class KubernetesVolume: - """KubernetesVolume data passing method involves passing data by mounting a - single multi-write Kubernetes volume to containers instead of using Argo's - artifact passing method (which stores the data in an S3 blob store).""" - - def __init__(self, volume: V1Volume, path_prefix: str = 'artifact_data/'): - if not isinstance(volume, (dict, V1Volume)): - raise TypeError('volume must be either V1Volume or dict') - self._volume = volume - self._path_prefix = path_prefix - - def transform_workflow(self, workflow: dict) -> dict: - from ..compiler._data_passing_using_volume import rewrite_data_passing_to_use_volumes - if isinstance(self._volume, dict): - volume_dict = self._volume - else: - volume_dict = kubernetes.kubernetes.client.ApiClient( - ).sanitize_for_serialization(self._volume) - return rewrite_data_passing_to_use_volumes(workflow, volume_dict, - self._path_prefix) - - def __call__(self, workflow: dict) -> dict: - return self.transform_workflow(workflow) diff --git a/sdk/python/kfp/deprecated/dsl/dsl_utils.py b/sdk/python/kfp/deprecated/dsl/dsl_utils.py deleted file mode 100644 index f78ae748018..00000000000 --- a/sdk/python/kfp/deprecated/dsl/dsl_utils.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities functions KFP DSL.""" - -import re -from typing import Callable, List, Optional, Union - -from kfp.deprecated.components import _structures -from kfp.pipeline_spec import pipeline_spec_pb2 - -_COMPONENT_NAME_PREFIX = 'comp-' -_EXECUTOR_LABEL_PREFIX = 'exec-' - -# TODO: Support all declared types in -# components._structures.CommandlineArgumenType -_CommandlineArgumentType = Union[str, int, float, - _structures.InputValuePlaceholder, - _structures.InputPathPlaceholder, - _structures.OutputPathPlaceholder, - _structures.InputUriPlaceholder, - _structures.OutputUriPlaceholder, - _structures.ExecutorInputPlaceholder] - - -def sanitize_component_name(name: str) -> str: - """Sanitizes component name.""" - return _COMPONENT_NAME_PREFIX + _sanitize_name(name) - - -def sanitize_task_name(name: str) -> str: - """Sanitizes task name.""" - return _sanitize_name(name) - - -def sanitize_executor_label(label: str) -> str: - """Sanitizes executor label.""" - return _EXECUTOR_LABEL_PREFIX + _sanitize_name(label) - - -def _sanitize_name(name: str) -> str: - """Sanitizes name to comply with IR naming convention. - - The sanitized name contains only lower case alphanumeric characters - and dashes. - """ - return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', - name.lower())).lstrip('-').rstrip('-') - - -def get_value(value: Union[str, int, float]) -> pipeline_spec_pb2.Value: - """Gets pipeline value proto from Python value.""" - result = pipeline_spec_pb2.Value() - if isinstance(value, str): - result.string_value = value - elif isinstance(value, int): - result.int_value = value - elif isinstance(value, float): - result.double_value = value - else: - raise TypeError( - 'Got unexpected type %s for value %s. Currently only support str, int ' - 'and float.' % (type(value), value)) - return result - - -# TODO: share with dsl._component_bridge -def _input_artifact_uri_placeholder(input_key: str) -> str: - return "{{{{$.inputs.artifacts['{}'].uri}}}}".format(input_key) - - -def _input_artifact_path_placeholder(input_key: str) -> str: - return "{{{{$.inputs.artifacts['{}'].path}}}}".format(input_key) - - -def _input_parameter_placeholder(input_key: str) -> str: - return "{{{{$.inputs.parameters['{}']}}}}".format(input_key) - - -def _output_artifact_uri_placeholder(output_key: str) -> str: - return "{{{{$.outputs.artifacts['{}'].uri}}}}".format(output_key) - - -def _output_artifact_path_placeholder(output_key: str) -> str: - return "{{{{$.outputs.artifacts['{}'].path}}}}".format(output_key) - - -def _output_parameter_path_placeholder(output_key: str) -> str: - return "{{{{$.outputs.parameters['{}'].output_file}}}}".format(output_key) - - -def _executor_input_placeholder() -> str: - return "{{{{$}}}}" - - -def resolve_cmd_lines(cmds: Optional[List[_CommandlineArgumentType]], - is_output_parameter: Callable[[str], bool]) -> None: - """Resolves a list of commands/args.""" - - def _resolve_cmd(cmd: Optional[_CommandlineArgumentType]) -> Optional[str]: - """Resolves a single command line cmd/arg.""" - if cmd is None: - return None - elif isinstance(cmd, (str, float, int)): - return str(cmd) - elif isinstance(cmd, _structures.InputValuePlaceholder): - return _input_parameter_placeholder(cmd.input_name) - elif isinstance(cmd, _structures.InputPathPlaceholder): - return _input_artifact_path_placeholder(cmd.input_name) - elif isinstance(cmd, _structures.InputUriPlaceholder): - return _input_artifact_uri_placeholder(cmd.input_name) - elif isinstance(cmd, _structures.OutputPathPlaceholder): - if is_output_parameter(cmd.output_name): - return _output_parameter_path_placeholder(cmd.output_name) - else: - return _output_artifact_path_placeholder(cmd.output_name) - elif isinstance(cmd, _structures.OutputUriPlaceholder): - return _output_artifact_uri_placeholder(cmd.output_name) - elif isinstance(cmd, _structures.ExecutorInputPlaceholder): - return _executor_input_placeholder() - else: - raise TypeError('Got unexpected placeholder type for %s' % cmd) - - if not cmds: - return - for idx, cmd in enumerate(cmds): - cmds[idx] = _resolve_cmd(cmd) diff --git a/sdk/python/kfp/deprecated/dsl/dsl_utils_test.py b/sdk/python/kfp/deprecated/dsl/dsl_utils_test.py deleted file mode 100644 index c78198f4ed8..00000000000 --- a/sdk/python/kfp/deprecated/dsl/dsl_utils_test.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for kfp.dsl.dsl_utils.""" - -import unittest - -from kfp.deprecated.dsl import dsl_utils -from kfp.pipeline_spec import pipeline_spec_pb2 -from google.protobuf import json_format - - -class _DummyClass(object): - pass - - -class DslUtilsTest(unittest.TestCase): - - def test_sanitize_component_name(self): - self.assertEqual('comp-my-component', - dsl_utils.sanitize_component_name('My component')) - - def test_sanitize_executor_label(self): - self.assertEqual('exec-my-component', - dsl_utils.sanitize_executor_label('My component')) - - def test_sanitize_task_name(self): - self.assertEqual('my-component-1', - dsl_utils.sanitize_task_name('My component 1')) - - def test_get_ir_value(self): - self.assertDictEqual( - json_format.MessageToDict(pipeline_spec_pb2.Value(int_value=42)), - json_format.MessageToDict(dsl_utils.get_value(42))) - self.assertDictEqual( - json_format.MessageToDict( - pipeline_spec_pb2.Value(double_value=12.2)), - json_format.MessageToDict(dsl_utils.get_value(12.2))) - self.assertDictEqual( - json_format.MessageToDict( - pipeline_spec_pb2.Value(string_value='hello world')), - json_format.MessageToDict(dsl_utils.get_value('hello world'))) - with self.assertRaisesRegex(TypeError, 'Got unexpected type'): - dsl_utils.get_value(_DummyClass()) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/dsl/extensions/__init__.py b/sdk/python/kfp/deprecated/dsl/extensions/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/kfp/deprecated/dsl/extensions/kubernetes.py b/sdk/python/kfp/deprecated/dsl/extensions/kubernetes.py deleted file mode 100644 index 20be33d16a5..00000000000 --- a/sdk/python/kfp/deprecated/dsl/extensions/kubernetes.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import random -import string - - -def use_secret(secret_name: str, - secret_volume_mount_path: str, - env_variable: str = None, - secret_file_path_in_volume: str = None): - """An operator that configures the container to use a secret. - - This assumes that the secret is created and availabel in the k8s cluster. - - Keyword Arguments: - secret_name {String} -- [Required] The k8s secret name. - secret_volume_mount_path {String} -- [Required] The path to the secret that is mounted. - env_variable {String} -- Env variable pointing to the mounted secret file. Requires both the env_variable and secret_file_path_in_volume to be defined. - The value is the path to the secret. - secret_file_path_in_volume {String} -- The path to the secret in the volume. This will be the value of env_variable. - Both env_variable and secret_file_path_in_volume needs to be set if any env variable should be created. - - Raises: - ValueError: If not the necessary variables (secret_name, volume_name", secret_volume_mount_path) are supplied. - Or only one of env_variable and secret_file_path_in_volume are supplied - - Returns: - [ContainerOperator] -- Returns the container operator after it has been modified. - """ - - secret_name = str(secret_name) - if '{{' in secret_name: - volume_name = ''.join( - random.choices(string.ascii_lowercase + string.digits, - k=10)) + "_volume" - else: - volume_name = secret_name - for param, param_name in zip([secret_name, secret_volume_mount_path], - ["secret_name", "secret_volume_mount_path"]): - if param == "": - raise ValueError("The '{}' must not be empty".format(param_name)) - if bool(env_variable) != bool(secret_file_path_in_volume): - raise ValueError( - "Both {} and {} needs to be supplied together or not at all".format( - env_variable, secret_file_path_in_volume)) - - def _use_secret(task): - import os - from kubernetes import client as k8s_client - task = task.add_volume( - k8s_client.V1Volume( - name=volume_name, - secret=k8s_client.V1SecretVolumeSource( - secret_name=secret_name))).add_volume_mount( - k8s_client.V1VolumeMount( - name=volume_name, - mount_path=secret_volume_mount_path)) - if env_variable: - task.container.add_env_variable( - k8s_client.V1EnvVar( - name=env_variable, - value=os.path.join(secret_volume_mount_path, - secret_file_path_in_volume), - )) - return task - - return _use_secret diff --git a/sdk/python/kfp/deprecated/dsl/io_types.py b/sdk/python/kfp/deprecated/dsl/io_types.py deleted file mode 100644 index fc7d6fdb21b..00000000000 --- a/sdk/python/kfp/deprecated/dsl/io_types.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Deprecated. See kfp.types.artifact_types instead. - -This module will be removed in KFP v2.0. -""" -import warnings -from kfp.components.types import artifact_types - -warnings.warn( - 'Module kfp.dsl.io_types is deprecated and will be removed' - ' in KFP v2.0. Please import types from kfp.dsl instead.', - category=FutureWarning) - -Artifact = artifact_types.Artifact -Dataset = artifact_types.Dataset -Metrics = artifact_types.Metrics -ClassificationMetrics = artifact_types.ClassificationMetrics -Model = artifact_types.Model -SlicedClassificationMetrics = artifact_types.SlicedClassificationMetrics -HTML = artifact_types.HTML -Markdown = artifact_types.Markdown -create_runtime_artifact = artifact_types.create_runtime_artifact diff --git a/sdk/python/kfp/deprecated/dsl/metrics_utils.py b/sdk/python/kfp/deprecated/dsl/metrics_utils.py deleted file mode 100644 index b7a2a9ece50..00000000000 --- a/sdk/python/kfp/deprecated/dsl/metrics_utils.py +++ /dev/null @@ -1,176 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated.dsl import artifact_utils -from typing import Any, List - - -class ComplexMetricsBase(object): - - def get_schema(self): - """Returns the set YAML schema for the metric class. - - Returns: - YAML schema of the metrics type. - """ - return self._schema - - def get_metrics(self): - """Returns the stored metrics. - - The metrics are type checked against the set schema. - - Returns: - Dictionary of metrics data in the format of the set schema. - """ - artifact_utils.verify_schema_instance(self._schema, self._values) - return self._values - - def __init__(self, schema_file: str): - self._schema = artifact_utils.read_schema_file(schema_file) - - self._type_name, self._metric_fields = artifact_utils.parse_schema( - self._schema) - - self._values = {} - - -class ConfidenceMetrics(ComplexMetricsBase): - """Metrics class representing a Confidence Metrics.""" - - # Initialization flag to support setattr / getattr behavior. - _initialized = False - - def __getattr__(self, name: str) -> Any: - """Custom __getattr__ to allow access to metrics schema fields.""" - - if name not in self._metric_fields: - raise AttributeError('No field: {} in metrics.'.format(name)) - - return self._values[name] - - def __setattr__(self, name: str, value: Any): - """Custom __setattr__ to allow access to metrics schema fields.""" - - if not self._initialized: - object.__setattr__(self, name, value) - return - - if name not in self._metric_fields: - raise RuntimeError( - 'Field: {} not defined in metirc schema'.format(name)) - - self._values[name] = value - - def __init__(self): - super().__init__('confidence_metrics.yaml') - self._initialized = True - - -class ConfusionMatrix(ComplexMetricsBase): - """Metrics class representing a confusion matrix.""" - - def __init__(self): - super().__init__('confusion_matrix.yaml') - - self._matrix = [[]] - self._categories = [] - self._initialized = True - - def set_categories(self, categories: List[str]): - """Sets the categories for Confusion Matrix. - - Args: - categories: List of strings specifying the categories. - """ - self._categories = [] - annotation_specs = [] - for category in categories: - annotation_spec = {'displayName': category} - self._categories.append(category) - annotation_specs.append(annotation_spec) - - self._values['annotationSpecs'] = annotation_specs - self._matrix = [[0 - for i in range(len(self._categories))] - for j in range(len(self._categories))] - self._values['row'] = self._matrix - - def log_row(self, row_category: str, row: List[int]): - """Logs a confusion matrix row. - - Args: - row_category: Category to which the row belongs. - row: List of integers specifying the values for the row. - - Raises: - ValueError: If row_category is not in the list of categories set in - set_categories or size of the row does not match the size of - categories. - """ - if row_category not in self._categories: - raise ValueError('Invalid category: {} passed. Expected one of: {}'.\ - format(row_category, self._categories)) - - if len(row) != len(self._categories): - raise ValueError('Invalid row. Expected size: {} got: {}'.\ - format(len(self._categories), len(row))) - - self._matrix[self._categories.index(row_category)] = row - - def log_cell(self, row_category: str, col_category: str, value: int): - """Logs a cell in the confusion matrix. - - Args: - row_category: String representing the name of the row category. - col_category: String representing the name of the column category. - value: Int value of the cell. - - Raises: - ValueError: If row_category or col_category is not in the list of - categories set in set_categories. - """ - if row_category not in self._categories: - raise ValueError('Invalid category: {} passed. Expected one of: {}'.\ - format(row_category, self._categories)) - - if col_category not in self._categories: - raise ValueError('Invalid category: {} passed. Expected one of: {}'.\ - format(row_category, self._categories)) - - self._matrix[self._categories.index(row_category)][ - self._categories.index(col_category)] = value - - def load_matrix(self, categories: List[str], matrix: List[List[int]]): - """Supports bulk loading the whole confusion matrix. - - Args: - categories: List of the category names. - matrix: Complete confusion matrix. - - Raises: - ValueError: Length of categories does not match number of rows or columns. - """ - self.set_categories(categories) - - if len(matrix) != len(categories): - raise ValueError('Invalid matrix: {} passed for categories: {}'.\ - format(matrix, categories)) - - for index in range(len(categories)): - if len(matrix[index]) != len(categories): - raise ValueError('Invalid matrix: {} passed for categories: {}'.\ - format(matrix, categories)) - - self.log_row(categories[index], matrix[index]) diff --git a/sdk/python/kfp/deprecated/dsl/metrics_utils_test.py b/sdk/python/kfp/deprecated/dsl/metrics_utils_test.py deleted file mode 100644 index 570727c7924..00000000000 --- a/sdk/python/kfp/deprecated/dsl/metrics_utils_test.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for kfp.dsl.metrics_utils.""" - -import os -import unittest -import json - -from kfp.deprecated.dsl import metrics_utils - - -class MetricsUtilsTest(unittest.TestCase): - - def test_confusion_matrix(self): - conf_matrix = metrics_utils.ConfusionMatrix() - conf_matrix.set_categories(['dog', 'cat', 'horses']) - conf_matrix.log_row('dog', [2, 6, 0]) - conf_matrix.log_cell('cat', 'dog', 3) - with open( - os.path.join( - os.path.dirname(__file__), 'test_data', - 'expected_confusion_matrix.json')) as json_file: - expected_json = json.load(json_file) - self.assertEqual(expected_json, conf_matrix.get_metrics()) - - def test_bulkload_confusion_matrix(self): - conf_matrix = metrics_utils.ConfusionMatrix() - conf_matrix.load_matrix(['dog', 'cat', 'horses'], - [[2, 6, 0], [3, 5, 6], [5, 7, 8]]) - - with open( - os.path.join( - os.path.dirname(__file__), 'test_data', - 'expected_bulk_loaded_confusion_matrix.json')) as json_file: - expected_json = json.load(json_file) - self.assertEqual(expected_json, conf_matrix.get_metrics()) - - def test_confidence_metrics(self): - confid_metrics = metrics_utils.ConfidenceMetrics() - confid_metrics.confidenceThreshold = 24.3 - confid_metrics.recall = 24.5 - confid_metrics.falsePositiveRate = 98.4 - expected_dict = { - 'confidenceThreshold': 24.3, - 'recall': 24.5, - 'falsePositiveRate': 98.4 - } - self.assertEqual(expected_dict, confid_metrics.get_metrics()) - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/kfp/deprecated/dsl/serialization_utils.py b/sdk/python/kfp/deprecated/dsl/serialization_utils.py deleted file mode 100644 index d56b22aac76..00000000000 --- a/sdk/python/kfp/deprecated/dsl/serialization_utils.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Utilities for serialization/deserialization.""" -from typing import Any, Dict -import yaml - - -# Serialize None as blank instead of 'null'. -def _represent_none(self, _): - return self.represent_scalar('tag:yaml.org,2002:null', '') - - -class _NoneAsBlankDumper(yaml.SafeDumper): - """Alternative dumper to print YAML literal. - - The behavior is mostly identical to yaml.SafeDumper, except for this - dumper dumps None object as blank, instead of 'null'. - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.add_representer(type(None), _represent_none) - - -def yaml_dump(data: Dict[str, Any]) -> str: - """Dumps YAML string. - - None will be represented as blank. - """ - return yaml.dump(data, Dumper=_NoneAsBlankDumper) diff --git a/sdk/python/kfp/deprecated/dsl/serialization_utils_test.py b/sdk/python/kfp/deprecated/dsl/serialization_utils_test.py deleted file mode 100644 index cbee21e608d..00000000000 --- a/sdk/python/kfp/deprecated/dsl/serialization_utils_test.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for kfp.dsl.serialization_utils module.""" -import unittest - -from kfp.deprecated.dsl import serialization_utils - -_DICT_DATA = { - 'int1': 1, - 'str1': 'helloworld', - 'float1': 1.11, - 'none1': None, - 'dict1': { - 'int2': 2, - 'list2': ['inside the list', None, 42] - } -} - -_EXPECTED_YAML_LITERAL = """\ -dict1: - int2: 2 - list2: - - inside the list - - - - 42 -float1: 1.11 -int1: 1 -none1: -str1: helloworld -""" - - -class SerializationUtilsTest(unittest.TestCase): - - def testDumps(self): - self.assertEqual(_EXPECTED_YAML_LITERAL, - serialization_utils.yaml_dump(_DICT_DATA)) diff --git a/sdk/python/kfp/deprecated/dsl/test_data/expected_bulk_loaded_confusion_matrix.json b/sdk/python/kfp/deprecated/dsl/test_data/expected_bulk_loaded_confusion_matrix.json deleted file mode 100644 index 184233e0baa..00000000000 --- a/sdk/python/kfp/deprecated/dsl/test_data/expected_bulk_loaded_confusion_matrix.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "annotationSpecs": [ - {"displayName": "dog"}, - {"displayName": "cat"}, - {"displayName": "horses"}], - "row": [ - [2, 6, 0], - [3, 5, 6], - [5, 7, 8]] -} \ No newline at end of file diff --git a/sdk/python/kfp/deprecated/dsl/test_data/expected_confusion_matrix.json b/sdk/python/kfp/deprecated/dsl/test_data/expected_confusion_matrix.json deleted file mode 100644 index 83312d1daa5..00000000000 --- a/sdk/python/kfp/deprecated/dsl/test_data/expected_confusion_matrix.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "annotationSpecs": [ - {"displayName": "dog"}, - {"displayName": "cat"}, - {"displayName": "horses"}], - "row": [ - [2, 6, 0], - [3, 0, 0], - [0, 0, 0]] -} \ No newline at end of file diff --git a/sdk/python/kfp/deprecated/dsl/type_schemas/classification_metrics.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/classification_metrics.yaml deleted file mode 100644 index 99e47522df5..00000000000 --- a/sdk/python/kfp/deprecated/dsl/type_schemas/classification_metrics.yaml +++ /dev/null @@ -1,81 +0,0 @@ -title: system.ClassificationMetrics -type: object -properties: - auPrc: - type: number - format: float - auRoc: - type: number - format: float - logLoss: - type: number - format: float - confidenceMetrics: - type: array - items: - type: object - properties: - confidenceThreshold: - type: number - format: float - maxPredictions: - type: integer - format: int32 - recall: - type: number - format: float - precision: - type: number - format: float - falsePositiveRate: - type: number - format: float - f1Score: - type: number - format: float - recallAt1: - type: number - format: float - precisionAt1: - type: number - format: float - falsePositiveRateAt1: - type: number - format: float - f1ScoreAt1: - type: number - format: float - truePositiveCount: - type: integer - format: int64 - falsePositiveCount: - type: integer - format: int64 - falseNegativeCount: - type: integer - format: int64 - trueNegativeCount: - type: integer - format: int64 - confusionMatrix: - type: object - properties: - annotationSpecs: - type: array - items: - type: object - properties: - id: - type: string - displayName: - type: string - rows: - type: array - items: - type: object - properties: - row: - type: array - items: - type: integer - format: int64 diff --git a/sdk/python/kfp/deprecated/dsl/type_schemas/confidence_metrics.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/confidence_metrics.yaml deleted file mode 100644 index 5ade735d1f0..00000000000 --- a/sdk/python/kfp/deprecated/dsl/type_schemas/confidence_metrics.yaml +++ /dev/null @@ -1,45 +0,0 @@ -title: system.ConfidenceMetrics -type: object -properties: - confidenceThreshold: - type: number - format: float - maxPredictions: - type: integer - format: int32 - recall: - type: number - format: float - precision: - type: number - format: float - falsePositiveRate: - type: number - format: float - f1Score: - type: number - format: float - recallAt1: - type: number - format: float - precisionAt1: - type: number - format: float - falsePositiveRateAt1: - type: number - format: float - f1ScoreAt1: - type: number - format: float - truePositiveCount: - type: integer - format: int64 - falsePositiveCount: - type: integer - format: int64 - falseNegativeCount: - type: integer - format: int64 - trueNegativeCount: - type: integer - format: int64 diff --git a/sdk/python/kfp/deprecated/dsl/type_schemas/confusion_matrix.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/confusion_matrix.yaml deleted file mode 100644 index f660e7d020e..00000000000 --- a/sdk/python/kfp/deprecated/dsl/type_schemas/confusion_matrix.yaml +++ /dev/null @@ -1,23 +0,0 @@ -title: system.ConfusionMatrix -type: object -properties: - annotationSpecs: - type: array - items: - type: object - properties: - id: - type: string - description: Optional. - displayName: - type: string - rows: - type: array - items: - type: object - properties: - row: - type: array - items: - type: integer - format: int64 diff --git a/sdk/python/kfp/deprecated/dsl/type_schemas/dataset.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/dataset.yaml deleted file mode 100644 index 4306ea4d18b..00000000000 --- a/sdk/python/kfp/deprecated/dsl/type_schemas/dataset.yaml +++ /dev/null @@ -1,7 +0,0 @@ -title: system.Dataset -type: object -properties: - payload_format: - type: string - container_format: - type: string diff --git a/sdk/python/kfp/deprecated/dsl/type_schemas/metrics.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/metrics.yaml deleted file mode 100644 index 3d3dc682685..00000000000 --- a/sdk/python/kfp/deprecated/dsl/type_schemas/metrics.yaml +++ /dev/null @@ -1,15 +0,0 @@ -title: system.Metrics -type: object -properties: - accuracy: - type: number - precision: - type: number - recall: - type: number - f1score: - type: number - mean_absolute_error: - type: number - mean_squared_error: - type: number diff --git a/sdk/python/kfp/deprecated/dsl/type_schemas/model.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/model.yaml deleted file mode 100644 index c4fe7dbad5c..00000000000 --- a/sdk/python/kfp/deprecated/dsl/type_schemas/model.yaml +++ /dev/null @@ -1,7 +0,0 @@ -title: system.Model -type: object -properties: - framework: - type: string - framework_version: - type: string diff --git a/sdk/python/kfp/deprecated/dsl/type_schemas/sliced_classification_metrics.yaml b/sdk/python/kfp/deprecated/dsl/type_schemas/sliced_classification_metrics.yaml deleted file mode 100644 index 46e4a0c7dfb..00000000000 --- a/sdk/python/kfp/deprecated/dsl/type_schemas/sliced_classification_metrics.yaml +++ /dev/null @@ -1,94 +0,0 @@ -title: system.SlicedClassificationMetrics -type: object -properties: - evaluationSlices: - type: array - items: - type: object - properties: - slice: - type: string - sliceClassificationMetrics: - type: object - properties: - auPrc: - type: number - format: float - auPrc: - type: number - format: float - auRoc: - type: number - format: float - logLoss: - type: number - format: float - confidenceMetrics: - type: array - items: - type: object - properties: - confidenceThreshold: - type: number - format: float - maxPredictions: - type: integer - format: int32 - recall: - type: number - format: float - precision: - type: number - format: float - falsePositiveRate: - type: number - format: float - f1Score: - type: number - format: float - recallAt1: - type: number - format: float - precisionAt1: - type: number - format: float - falsePositiveRateAt1: - type: number - format: float - f1ScoreAt1: - type: number - format: float - truePositiveCount: - type: integer - format: int64 - falsePositiveCount: - type: integer - format: int64 - falseNegativeCount: - type: integer - format: int64 - trueNegativeCount: - type: integer - format: int64 - confusionMatrix: - type: object - properties: - annotationSpecs: - type: array - items: - type: object - properties: - id: - type: string - displayName: - type: string - rows: - type: array - items: - type: object - properties: - row: - type: array - items: - type: integer - format: int64 diff --git a/sdk/python/kfp/deprecated/dsl/type_utils.py b/sdk/python/kfp/deprecated/dsl/type_utils.py deleted file mode 100644 index c0ed2238dfd..00000000000 --- a/sdk/python/kfp/deprecated/dsl/type_utils.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Deprecated. See kfp.dsl.types.type_utils instead. - -This module will be removed in KFP v2.0. -""" -import warnings -from kfp.dsl.types import type_utils, artifact_types -from kfp.pipeline_spec import pipeline_spec_pb2 -import inspect -from typing import Union, Type -import re -from typing import List, Optional -from kfp.deprecated.components import _structures - -warnings.warn( - 'Module kfp.dsl.type_utils is deprecated and will be removed' - ' in KFP v2.0. Please use from kfp.dsl.types.type_utils instead.', - category=FutureWarning) - -is_parameter_type = type_utils.is_parameter_type -get_parameter_type = type_utils.get_parameter_type - -# copying a lot of code from v2 to here to avoid certain dependencies of deprecated on v2, since making any changes to this code in v2 would result in breaks in deprecated/ -_GOOGLE_TYPES_PATTERN = r'^google.[A-Za-z]+$' -_GOOGLE_TYPES_VERSION = '0.0.1' -_ARTIFACT_CLASSES_MAPPING = { - 'model': artifact_types.Model, - 'dataset': artifact_types.Dataset, - 'metrics': artifact_types.Metrics, - 'classificationmetrics': artifact_types.ClassificationMetrics, - 'slicedclassificationmetrics': artifact_types.SlicedClassificationMetrics, - 'html': artifact_types.HTML, - 'markdown': artifact_types.Markdown, -} - - -def get_artifact_type_schema( - artifact_class_or_type_name: Optional[Union[str, - Type[artifact_types.Artifact]]] -) -> pipeline_spec_pb2.ArtifactTypeSchema: - """Gets the IR I/O artifact type msg for the given ComponentSpec I/O - type.""" - artifact_class = artifact_types.Artifact - if isinstance(artifact_class_or_type_name, str): - if re.match(_GOOGLE_TYPES_PATTERN, artifact_class_or_type_name): - return pipeline_spec_pb2.ArtifactTypeSchema( - schema_title=artifact_class_or_type_name, - schema_version=_GOOGLE_TYPES_VERSION, - ) - artifact_class = _ARTIFACT_CLASSES_MAPPING.get( - artifact_class_or_type_name.lower(), artifact_types.Artifact) - elif inspect.isclass(artifact_class_or_type_name) and issubclass( - artifact_class_or_type_name, artifact_types.Artifact): - artifact_class = artifact_class_or_type_name - - return pipeline_spec_pb2.ArtifactTypeSchema( - schema_title=artifact_class.schema_title, - schema_version=artifact_class.schema_version) - - -def get_input_artifact_type_schema( - input_name: str, - inputs: List[_structures.InputSpec], -) -> Optional[str]: - """Find the input artifact type by input name. - - Args: - input_name: The name of the component input. - inputs: The list of InputSpec - - Returns: - The artifact type schema of the input. - - Raises: - AssertionError if input not found, or input found but not an artifact type. - """ - - for component_input in inputs: - if component_input.name == input_name: - assert not is_parameter_type( - component_input.type), 'Input is not an artifact type.' - return get_artifact_type_schema_old(component_input.type) - assert False, 'Input not found.' diff --git a/sdk/python/kfp/deprecated/dsl/types.py b/sdk/python/kfp/deprecated/dsl/types.py deleted file mode 100644 index 8c793759aea..00000000000 --- a/sdk/python/kfp/deprecated/dsl/types.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Module for input/output types in Pipeline DSL. - -Feature stage: -[Beta](https://github.com/kubeflow/pipelines/blob/07328e5094ac2981d3059314cc848fbb71437a76/docs/release/feature-stages.md#beta). -""" -from typing import Dict, Union -import warnings - -from kfp.deprecated.dsl import type_utils - - -class BaseType: - """BaseType is a base type for all scalar and artifact types.""" - - def to_dict(self) -> Union[Dict, str]: - """to_dict serializes the type instance into a python dictionary or - string.""" - return { - type(self).__name__: self.__dict__ - } if self.__dict__ else type(self).__name__ - - -# Primitive Types -class Integer(BaseType): - - def __init__(self): - self.openapi_schema_validator = {"type": "integer"} - - -class String(BaseType): - - def __init__(self): - self.openapi_schema_validator = {"type": "string"} - - -class Float(BaseType): - - def __init__(self): - self.openapi_schema_validator = {"type": "number"} - - -class Bool(BaseType): - - def __init__(self): - self.openapi_schema_validator = {"type": "boolean"} - - -class List(BaseType): - - def __init__(self): - self.openapi_schema_validator = {"type": "array"} - - -class Dict(BaseType): - - def __init__(self): - self.openapi_schema_validator = { - "type": "object", - } - - -# GCP Types -class GCSPath(BaseType): - - def __init__(self): - self.openapi_schema_validator = { - "type": "string", - "pattern": "^gs://.*$" - } - - -class GCRPath(BaseType): - - def __init__(self): - self.openapi_schema_validator = { - "type": "string", - "pattern": "^.*gcr\\.io/.*$" - } - - -class GCPRegion(BaseType): - - def __init__(self): - self.openapi_schema_validator = {"type": "string"} - - -class GCPProjectID(BaseType): - """MetaGCPProjectID: GCP project id""" - - def __init__(self): - self.openapi_schema_validator = {"type": "string"} - - -# General Types -class LocalPath(BaseType): - #TODO: add restriction to path - def __init__(self): - self.openapi_schema_validator = {"type": "string"} - - -class InconsistentTypeException(Exception): - """InconsistencyTypeException is raised when two types are not - consistent.""" - - -class InconsistentTypeWarning(Warning): - """InconsistentTypeWarning is issued when two types are not consistent.""" - - -TypeSpecType = Union[str, Dict] - - -def verify_type_compatibility(given_type: TypeSpecType, - expected_type: TypeSpecType, - error_message_prefix: str = ""): - """verify_type_compatibility verifies that the given argument type is - compatible with the expected input type. - - Args: - given_type (str/dict): The type of the argument passed to the - input - expected_type (str/dict): The declared type of the input - """ - # Missing types are treated as being compatible with missing types. - if given_type is None or expected_type is None: - return True - - # Generic artifacts resulted from missing type or explicit "Artifact" type - # is compatible with any artifact types. - # However, generic artifacts resulted from arbitrary unknown types do not - # have such "compatible" feature. - if not type_utils.is_parameter_type( - str(expected_type)) and str(given_type).lower() == "artifact": - return True - if not type_utils.is_parameter_type( - str(given_type)) and str(expected_type).lower() == "artifact": - return True - - types_are_compatible = check_types(given_type, expected_type) - - if not types_are_compatible: - error_text = error_message_prefix + ( - 'Argument type "{}" is incompatible with the input type "{}"' - ).format(str(given_type), str(expected_type)) - import kfp.deprecated as kfp - if kfp.TYPE_CHECK: - raise InconsistentTypeException(error_text) - else: - warnings.warn(InconsistentTypeWarning(error_text)) - return types_are_compatible - - -def check_types(checked_type, expected_type): - """check_types checks the type consistency. - - For each of the attribute in checked_type, there is the same attribute - in expected_type with the same value. - However, expected_type could contain more attributes that checked_type - does not contain. - Args: - checked_type (BaseType/str/dict): it describes a type from the - upstream component output - expected_type (BaseType/str/dict): it describes a type from the - downstream component input - """ - if isinstance(checked_type, BaseType): - checked_type = checked_type.to_dict() - if isinstance(checked_type, str): - checked_type = {checked_type: {}} - if isinstance(expected_type, BaseType): - expected_type = expected_type.to_dict() - if isinstance(expected_type, str): - expected_type = {expected_type: {}} - return _check_dict_types(checked_type, expected_type) - - -def _check_valid_type_dict(payload): - """_check_valid_type_dict checks whether a dict is a correct serialization - of a type. - - Args: payload(dict) - """ - if not isinstance(payload, dict) or len(payload) != 1: - return False - for type_name in payload: - if not isinstance(payload[type_name], dict): - return False - property_types = (int, str, float, bool) - property_value_types = (int, str, float, bool, dict) - for property_name in payload[type_name]: - if not isinstance(property_name, property_types) or not isinstance( - payload[type_name][property_name], property_value_types): - return False - return True - - -def _check_dict_types(checked_type, expected_type): - """_check_dict_types checks the type consistency. - - Args: - checked_type (dict): A dict that describes a type from the upstream - component output - expected_type (dict): A dict that describes a type from the downstream - component input - """ - if not checked_type or not expected_type: - # If the type is empty, it matches any types - return True - checked_type_name, _ = list(checked_type.items())[0] - expected_type_name, _ = list(expected_type.items())[0] - if checked_type_name == "" or expected_type_name == "": - # If the type name is empty, it matches any types - return True - if checked_type_name != expected_type_name: - print("type name " + str(checked_type_name) + - " is different from expected: " + str(expected_type_name)) - return False - type_name = checked_type_name - for type_property in checked_type[type_name]: - if type_property not in expected_type[type_name]: - print(type_name + " has a property " + str(type_property) + - " that the latter does not.") - return False - if checked_type[type_name][type_property] != expected_type[type_name][ - type_property]: - print(type_name + " has a property " + str(type_property) + - " with value: " + - str(checked_type[type_name][type_property]) + " and " + - str(expected_type[type_name][type_property])) - return False - return True diff --git a/sdk/python/kfp/deprecated/gcp.py b/sdk/python/kfp/deprecated/gcp.py deleted file mode 100644 index 6c41056ec31..00000000000 --- a/sdk/python/kfp/deprecated/gcp.py +++ /dev/null @@ -1,146 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Extension module for KFP on GCP deployment.""" - -from kubernetes.client import V1Toleration, V1Affinity, V1NodeAffinity, \ - V1NodeSelector, V1NodeSelectorTerm, V1NodeSelectorRequirement, V1PreferredSchedulingTerm - - -def use_gcp_secret(secret_name='user-gcp-sa', - secret_file_path_in_volume=None, - volume_name=None, - secret_volume_mount_path='/secret/gcp-credentials'): - """An operator that configures the container to use GCP service account by - service account key stored in a Kubernetes secret. - - For cluster setup and alternatives to using service account key, check https://www.kubeflow.org/docs/gke/authentication-pipelines/. - """ - - # permitted values for secret_name = ['admin-gcp-sa', 'user-gcp-sa'] - if secret_file_path_in_volume is None: - secret_file_path_in_volume = '/' + secret_name + '.json' - - if volume_name is None: - volume_name = 'gcp-credentials-' + secret_name - - else: - import warnings - warnings.warn( - 'The volume_name parameter is deprecated and will be removed in next release. The volume names are now generated automatically.', - DeprecationWarning) - - def _use_gcp_secret(task): - from kubernetes import client as k8s_client - task = task.add_volume( - k8s_client.V1Volume( - name=volume_name, - secret=k8s_client.V1SecretVolumeSource( - secret_name=secret_name,))) - task.container \ - .add_volume_mount( - k8s_client.V1VolumeMount( - name=volume_name, - mount_path=secret_volume_mount_path, - ) - ) \ - .add_env_variable( - k8s_client.V1EnvVar( - name='GOOGLE_APPLICATION_CREDENTIALS', - value=secret_volume_mount_path + secret_file_path_in_volume, - ) - ) \ - .add_env_variable( - k8s_client.V1EnvVar( - name='CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE', - value=secret_volume_mount_path + secret_file_path_in_volume, - ) - ) # Set GCloud Credentials by using the env var override. - # TODO: Is there a better way for GCloud to pick up the credential? - return task - - return _use_gcp_secret - - -def use_tpu(tpu_cores: int, tpu_resource: str, tf_version: str): - """An operator that configures GCP TPU spec in a container op. - - Args: - tpu_cores: Required. The number of cores of TPU resource. - For example, the value can be '8', '32', '128', etc. - Check more details at: https://cloud.google.com/tpu/docs/kubernetes-engine-setup#pod-spec. - tpu_resource: Required. The resource name of the TPU resource. - For example, the value can be 'v2', 'preemptible-v1', 'v3' or 'preemptible-v3'. - Check more details at: https://cloud.google.com/tpu/docs/kubernetes-engine-setup#pod-spec. - tf_version: Required. The TensorFlow version that the TPU nodes use. - For example, the value can be '1.12', '1.11', '1.9' or '1.8'. - Check more details at: https://cloud.google.com/tpu/docs/supported-versions. - """ - - def _set_tpu_spec(task): - task.add_pod_annotation('tf-version.cloud-tpus.google.com', tf_version) - task.container.add_resource_limit( - 'cloud-tpus.google.com/{}'.format(tpu_resource), str(tpu_cores)) - return task - - return _set_tpu_spec - - -def use_preemptible_nodepool(toleration: V1Toleration = V1Toleration( - effect='NoSchedule', key='preemptible', operator='Equal', value='true'), - hard_constraint: bool = False): - """An operator that configures the GKE preemptible in a container op. - - Args: - toleration: toleration to pods, default is the preemptible label. - hard_constraint: the constraint of scheduling the pods on preemptible - nodepools is hard. (Default: False) - """ - - def _set_preemptible(task): - task.add_toleration(toleration) - node_selector_term = V1NodeSelectorTerm(match_expressions=[ - V1NodeSelectorRequirement( - key='cloud.google.com/gke-preemptible', - operator='In', - values=['true']) - ]) - if hard_constraint: - node_affinity = V1NodeAffinity( - required_during_scheduling_ignored_during_execution=V1NodeSelector( - node_selector_terms=[node_selector_term])) - else: - node_affinity = V1NodeAffinity( - preferred_during_scheduling_ignored_during_execution=[ - V1PreferredSchedulingTerm( - preference=node_selector_term, weight=50) - ]) - affinity = V1Affinity(node_affinity=node_affinity) - task.add_affinity(affinity=affinity) - return task - - return _set_preemptible - - -def add_gpu_toleration(toleration: V1Toleration = V1Toleration( - effect='NoSchedule', key='nvidia.com/gpu', operator='Equal', value='true')): - """An operator that configures the GKE GPU nodes in a container op. - - Args: - toleration: toleration to pods, default is the nvidia.com/gpu label. - """ - - def _set_toleration(task): - task.add_toleration(toleration) - - return _set_toleration diff --git a/sdk/python/kfp/deprecated/notebook/__init__.py b/sdk/python/kfp/deprecated/notebook/__init__.py deleted file mode 100644 index 90296e2c341..00000000000 --- a/sdk/python/kfp/deprecated/notebook/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from . import _magic diff --git a/sdk/python/kfp/deprecated/notebook/_magic.py b/sdk/python/kfp/deprecated/notebook/_magic.py deleted file mode 100644 index a13c6b838df..00000000000 --- a/sdk/python/kfp/deprecated/notebook/_magic.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/sdk/python/kfp/deprecated/onprem.py b/sdk/python/kfp/deprecated/onprem.py deleted file mode 100644 index 5db04c9724a..00000000000 --- a/sdk/python/kfp/deprecated/onprem.py +++ /dev/null @@ -1,133 +0,0 @@ -from typing import Dict, Optional -from kfp.deprecated import dsl - - -def mount_pvc(pvc_name='pipeline-claim', - volume_name='pipeline', - volume_mount_path='/mnt/pipeline'): - """Modifier function to apply to a Container Op to simplify volume, volume - mount addition and enable better reuse of volumes, volume claims across - container ops. - - Example: - :: - - train = train_op(...) - train.apply(mount_pvc('claim-name', 'pipeline', '/mnt/pipeline')) - """ - - def _mount_pvc(task): - from kubernetes import client as k8s_client - # there can be other ops in a pipeline (e.g. ResourceOp, VolumeOp) - # refer to #3906 - if not hasattr(task, "add_volume") or not hasattr( - task, "add_volume_mount"): - return task - local_pvc = k8s_client.V1PersistentVolumeClaimVolumeSource( - claim_name=pvc_name) - return (task.add_volume( - k8s_client.V1Volume( - name=volume_name, - persistent_volume_claim=local_pvc)).add_volume_mount( - k8s_client.V1VolumeMount( - mount_path=volume_mount_path, name=volume_name))) - - return _mount_pvc - - -def use_k8s_secret( - secret_name: str = 'k8s-secret', - k8s_secret_key_to_env: Optional[Dict] = None, -): - """An operator that configures the container to use k8s credentials. - - k8s_secret_key_to_env specifies a mapping from the name of the keys in the k8s secret to the name of the - environment variables where the values will be added. - - The secret needs to be deployed manually a priori. - - Example: - :: - - train = train_op(...) - train.apply(use_k8s_secret(secret_name='s3-secret', - k8s_secret_key_to_env={'secret_key': 'AWS_SECRET_ACCESS_KEY'})) - - This will load the value in secret 's3-secret' at key 'secret_key' and source it as the environment variable - 'AWS_SECRET_ACCESS_KEY'. I.e. it will produce the following section on the pod: - env: - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: s3-secret - key: secret_key - """ - - k8s_secret_key_to_env = k8s_secret_key_to_env or {} - - def _use_k8s_secret(task): - from kubernetes import client as k8s_client - for secret_key, env_var in k8s_secret_key_to_env.items(): - task.container \ - .add_env_variable( - k8s_client.V1EnvVar( - name=env_var, - value_from=k8s_client.V1EnvVarSource( - secret_key_ref=k8s_client.V1SecretKeySelector( - name=secret_name, - key=secret_key - ) - ) - ) - ) - return task - - return _use_k8s_secret - - -def add_default_resource_spec( - memory_limit: Optional[str] = None, - cpu_limit: Optional[str] = None, - memory_request: Optional[str] = None, - cpu_request: Optional[str] = None, -): - """Add default resource requests and limits. - - For resource units, refer to https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#resource-units-in-kubernetes. - - Args: - memory_limit: optional, memory limit. Format can be 512Mi, 2Gi etc. - cpu_limit: optional, cpu limit. Format can be 0.5, 500m etc. - memory_request: optional, defaults to memory limit. - cpu_request: optional, defaults to cpu limit. - """ - if not memory_request: - memory_request = memory_limit - if not cpu_request: - cpu_request = cpu_limit - - def _add_default_resource_spec(task): - # Skip tasks which are not container ops. - if not isinstance(task, dsl.ContainerOp): - return task - _apply_default_resource(task, 'cpu', cpu_request, cpu_limit) - _apply_default_resource(task, 'memory', memory_request, memory_limit) - return task - - return _add_default_resource_spec - - -def _apply_default_resource(task: dsl.ContainerOp, resource_name: str, - default_request: Optional[str], - default_limit: Optional[str]): - if task.container.get_resource_limit(resource_name): - # Do nothing. - # Limit is set, request will default to limit if not set (Kubernetes default behavior), - # so we do not need to further apply defaults. - return - - if default_limit: - task.container.add_resource_limit(resource_name, default_limit) - if default_request: - if not task.container.get_resource_request(resource_name): - task.container.add_resource_request(resource_name, default_request) diff --git a/sdk/python/setup.py b/sdk/python/setup.py index 39619be6577..6b820ad9150 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -102,7 +102,6 @@ def read_readme() -> str: entry_points={ 'console_scripts': [ 'dsl-compile = kfp.cli.compile_:main', - 'dsl-compile-deprecated = kfp.deprecated.compiler.main:main', 'kfp=kfp.cli.__main__:main', ] }) diff --git a/sdk/python/tests/__init__.py b/sdk/python/tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/tests/compiler/__init__.py b/sdk/python/tests/compiler/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/tests/compiler/component_builder_test.py b/sdk/python/tests/compiler/component_builder_test.py deleted file mode 100644 index 589f4b51532..00000000000 --- a/sdk/python/tests/compiler/component_builder_test.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import unittest - -from kfp.deprecated.containers._component_builder import \ - _dependency_to_requirements -from kfp.deprecated.containers._component_builder import _generate_dockerfile -from kfp.deprecated.containers._component_builder import DependencyHelper -from kfp.deprecated.containers._component_builder import VersionedDependency - - -class TestVersionedDependency(unittest.TestCase): - - def test_version(self): - """test version overrides min_version and max_version.""" - version = VersionedDependency( - name='tensorflow', - version='0.3.0', - min_version='0.1.0', - max_version='0.4.0') - self.assertTrue(version.min_version == '0.3.0') - self.assertTrue(version.max_version == '0.3.0') - self.assertTrue(version.has_versions()) - self.assertTrue(version.name == 'tensorflow') - - def test_minmax_version(self): - """test if min_version and max_version are configured when version is - not given.""" - version = VersionedDependency( - name='tensorflow', min_version='0.1.0', max_version='0.4.0') - self.assertTrue(version.min_version == '0.1.0') - self.assertTrue(version.max_version == '0.4.0') - self.assertTrue(version.has_versions()) - - def test_min_or_max_version(self): - """test if min_version and max_version are configured when version is - not given.""" - version = VersionedDependency(name='tensorflow', min_version='0.1.0') - self.assertTrue(version.min_version == '0.1.0') - self.assertTrue(version.has_versions()) - version = VersionedDependency(name='tensorflow', max_version='0.3.0') - self.assertTrue(version.max_version == '0.3.0') - self.assertTrue(version.has_versions()) - - def test_no_version(self): - """test the no version scenario.""" - version = VersionedDependency(name='tensorflow') - self.assertFalse(version.has_min_version()) - self.assertFalse(version.has_max_version()) - self.assertFalse(version.has_versions()) - - -class TestDependencyHelper(unittest.TestCase): - - def test_generate_requirement(self): - """Test generating requirement file.""" - - # prepare - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - temp_file = os.path.join(test_data_dir, 'test_requirements.tmp') - - dependency_helper = DependencyHelper() - dependency_helper.add_python_package( - dependency=VersionedDependency( - name='tensorflow', min_version='0.10.0', max_version='0.11.0')) - dependency_helper.add_python_package( - dependency=VersionedDependency( - name='kubernetes', min_version='0.6.0')) - dependency_helper.add_python_package( - dependency=VersionedDependency(name='pytorch', max_version='0.3.0')) - dependency_helper.generate_pip_requirements(temp_file) - - golden_requirement_payload = '''\ -tensorflow >= 0.10.0, <= 0.11.0 -kubernetes >= 0.6.0 -pytorch <= 0.3.0 -''' - with open(temp_file, 'r') as f: - target_requirement_payload = f.read() - self.assertEqual(target_requirement_payload, golden_requirement_payload) - os.remove(temp_file) - - def test_add_python_package(self): - """Test add_python_package.""" - - # prepare - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - temp_file = os.path.join(test_data_dir, 'test_requirements.tmp') - - dependency_helper = DependencyHelper() - dependency_helper.add_python_package( - dependency=VersionedDependency( - name='tensorflow', min_version='0.10.0', max_version='0.11.0')) - dependency_helper.add_python_package( - dependency=VersionedDependency( - name='kubernetes', min_version='0.6.0')) - dependency_helper.add_python_package( - dependency=VersionedDependency( - name='tensorflow', min_version='0.12.0'), - override=True) - dependency_helper.add_python_package( - dependency=VersionedDependency( - name='kubernetes', min_version='0.8.0'), - override=False) - dependency_helper.add_python_package( - dependency=VersionedDependency(name='pytorch', version='0.3.0')) - dependency_helper.generate_pip_requirements(temp_file) - golden_requirement_payload = '''\ -tensorflow >= 0.12.0 -kubernetes >= 0.6.0 -pytorch >= 0.3.0, <= 0.3.0 -''' - with open(temp_file, 'r') as f: - target_requirement_payload = f.read() - self.assertEqual(target_requirement_payload, golden_requirement_payload) - os.remove(temp_file) - - -class TestGenerator(unittest.TestCase): - - def test_generate_dockerfile(self): - """Test generate dockerfile.""" - # prepare - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - target_dockerfile = os.path.join(test_data_dir, - 'component.temp.dockerfile') - golden_dockerfile_payload_one = '''\ -FROM gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0 -RUN apt-get update -y && apt-get install --no-install-recommends -y -q python3 python3-pip python3-setuptools -ADD main.py /ml/main.py -''' - golden_dockerfile_payload_two = '''\ -FROM gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0 -RUN apt-get update -y && apt-get install --no-install-recommends -y -q python3 python3-pip python3-setuptools -ADD requirements.txt /ml/requirements.txt -RUN python3 -m pip install -r /ml/requirements.txt -ADD main.py /ml/main.py -''' - - # check - _generate_dockerfile( - filename=target_dockerfile, - base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0', - add_files={'main.py': '/ml/main.py'}) - with open(target_dockerfile, 'r') as f: - target_dockerfile_payload = f.read() - self.assertEqual(target_dockerfile_payload, - golden_dockerfile_payload_one) - - _generate_dockerfile( - filename=target_dockerfile, - base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0', - requirement_filename='requirements.txt', - add_files={'main.py': '/ml/main.py'}) - with open(target_dockerfile, 'r') as f: - target_dockerfile_payload = f.read() - self.assertEqual(target_dockerfile_payload, - golden_dockerfile_payload_two) - - # clean up - os.remove(target_dockerfile) - - def test_generate_requirement(self): - # prepare - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - temp_file = os.path.join(test_data_dir, 'test_requirements.tmp') - - dependencies = [ - VersionedDependency( - name='tensorflow', min_version='0.10.0', max_version='0.11.0'), - VersionedDependency(name='kubernetes', min_version='0.6.0'), - ] - _dependency_to_requirements(dependencies, filename=temp_file) - golden_payload = '''\ -tensorflow >= 0.10.0, <= 0.11.0 -kubernetes >= 0.6.0 -''' - with open(temp_file, 'r') as f: - target_payload = f.read() - self.assertEqual(target_payload, golden_payload) - os.remove(temp_file) diff --git a/sdk/python/tests/compiler/container_builder_test.py b/sdk/python/tests/compiler/container_builder_test.py deleted file mode 100644 index 72c1c68d6d7..00000000000 --- a/sdk/python/tests/compiler/container_builder_test.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import tarfile -import tempfile -import unittest -from unittest import mock - -from kfp.deprecated.containers._component_builder import ContainerBuilder -import yaml - -GCS_BASE = 'gs://kfp-testing/' -DEFAULT_IMAGE_NAME = 'gcr.io/kfp-testing/image' - - -@mock.patch('kfp.deprecated.containers._gcs_helper.GCSHelper') -class TestContainerBuild(unittest.TestCase): - - def test_wrap_dir_in_tarball(self, mock_gcshelper): - """Test wrap files in a tarball.""" - - # prepare - temp_tarball = os.path.join( - os.path.dirname(__file__), 'test_data.tmp.tar.gz') - with tempfile.TemporaryDirectory() as test_data_dir: - temp_file_one = os.path.join(test_data_dir, 'test_data_one.tmp') - temp_file_two = os.path.join(test_data_dir, 'test_data_two.tmp') - with open(temp_file_one, 'w') as f: - f.write('temporary file one content') - with open(temp_file_two, 'w') as f: - f.write('temporary file two content') - - # check - builder = ContainerBuilder( - gcs_staging=GCS_BASE, - default_image_name=DEFAULT_IMAGE_NAME, - namespace='') - builder._wrap_dir_in_tarball(temp_tarball, test_data_dir) - self.assertTrue(os.path.exists(temp_tarball)) - with tarfile.open(temp_tarball) as temp_tarball_handle: - temp_files = temp_tarball_handle.getmembers() - for temp_file in temp_files: - self.assertTrue(temp_file.name in - ['test_data_one.tmp', 'test_data_two.tmp', '']) - - # clean up - os.remove(temp_tarball) - - def test_generate_kaniko_yaml(self, mock_gcshelper): - """Test generating the kaniko job yaml.""" - - # prepare - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - - # check - builder = ContainerBuilder( - gcs_staging=GCS_BASE, - default_image_name=DEFAULT_IMAGE_NAME, - namespace='default') - generated_yaml = builder._generate_kaniko_spec( - docker_filename='dockerfile', - context='gs://mlpipeline/kaniko_build.tar.gz', - target_image='gcr.io/mlpipeline/kaniko_image:latest') - with open(os.path.join(test_data_dir, 'kaniko.basic.yaml'), 'r') as f: - golden = yaml.safe_load(f) - - self.assertEqual(golden, generated_yaml) - - def test_generate_kaniko_yaml_kubeflow(self, mock_gcshelper): - """Test generating the kaniko job yaml for Kubeflow deployment.""" - - # prepare - test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata') - - # check - builder = ContainerBuilder( - gcs_staging=GCS_BASE, - default_image_name=DEFAULT_IMAGE_NAME, - namespace='user', - service_account='default-editor', - ) - generated_yaml = builder._generate_kaniko_spec( - docker_filename='dockerfile', - context='gs://mlpipeline/kaniko_build.tar.gz', - target_image='gcr.io/mlpipeline/kaniko_image:latest', - ) - with open(os.path.join(test_data_dir, 'kaniko.kubeflow.yaml'), - 'r') as f: - golden = yaml.safe_load(f) - - self.assertEqual(golden, generated_yaml) diff --git a/sdk/python/tests/compiler/k8s_helper_tests.py b/sdk/python/tests/compiler/k8s_helper_tests.py deleted file mode 100644 index 95b7e129c0e..00000000000 --- a/sdk/python/tests/compiler/k8s_helper_tests.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from datetime import datetime -import unittest - -from kfp.deprecated.compiler._k8s_helper import convert_k8s_obj_to_json - - -class TestCompiler(unittest.TestCase): - - def test_convert_k8s_obj_to_dic_accepts_dict(self): - now = datetime.now() - converted = convert_k8s_obj_to_json({ - "ENV": "test", - "number": 3, - "list": [1, 2, 3], - "time": now - }) - self.assertEqual(converted, { - "ENV": "test", - "number": 3, - "list": [1, 2, 3], - "time": now.isoformat() - }) diff --git a/sdk/python/tests/compiler/main.py b/sdk/python/tests/compiler/main.py deleted file mode 100644 index c1c034ee862..00000000000 --- a/sdk/python/tests/compiler/main.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import unittest - -import compiler_tests -import component_builder_test -import container_builder_test -import k8s_helper_tests - -if __name__ == '__main__': - suite = unittest.TestSuite() - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(compiler_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(component_builder_test)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(container_builder_test)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(k8s_helper_tests)) - runner = unittest.TextTestRunner() - if not runner.run(suite).wasSuccessful(): - sys.exit(1) diff --git a/sdk/python/tests/compiler/testdata/README.md b/sdk/python/tests/compiler/testdata/README.md deleted file mode 100644 index 4acb8e7bc38..00000000000 --- a/sdk/python/tests/compiler/testdata/README.md +++ /dev/null @@ -1,3 +0,0 @@ -To generate golden workflow yaml, for now please check the comments in [compiler_tests.py](../compiler_tests.py). - -Once you generate a workflow yaml, please also test it with argo to make sure it is "golden". diff --git a/sdk/python/tests/compiler/testdata/add_pod_env.py b/sdk/python/tests/compiler/testdata/add_pod_env.py deleted file mode 100644 index f33824e7f0e..00000000000 --- a/sdk/python/tests/compiler/testdata/add_pod_env.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp - - -@kfp.dsl.pipeline(name='Test adding pod env', description='Test adding pod env') -def test_add_pod_env(): - op = kfp.dsl.ContainerOp( - name='echo', - image='library/bash', - command=['sh', '-c'], - arguments=['echo $KFP_POD_NAME']).add_pod_label('add-pod-env', 'true') - - -if __name__ == '__main__': - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(test_add_pod_env, __file__ + '.yaml') diff --git a/sdk/python/tests/compiler/testdata/add_pod_env.yaml b/sdk/python/tests/compiler/testdata/add_pod_env.yaml deleted file mode 100644 index 4bce1948a2f..00000000000 --- a/sdk/python/tests/compiler/testdata/add_pod_env.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "Test adding pod env", "name": "Test adding pod env"}' - generateName: test-adding-pod-env- -spec: - arguments: - parameters: [] - entrypoint: test-adding-pod-env - serviceAccountName: pipeline-runner - templates: - - container: - args: - - echo $KFP_POD_NAME - command: - - sh - - -c - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KFP_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid - - name: KFP_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: WORKFLOW_ID - valueFrom: - fieldRef: - fieldPath: metadata.labels['workflows.argoproj.io/workflow'] - - name: KFP_RUN_ID - valueFrom: - fieldRef: - fieldPath: metadata.labels['pipeline/runid'] - - name: ENABLE_CACHING - valueFrom: - fieldRef: - fieldPath: metadata.labels['pipelines.kubeflow.org/enable_caching'] - image: library/bash - metadata: - labels: - add-pod-env: 'true' - name: echo - - dag: - tasks: - - name: echo - template: echo - name: test-adding-pod-env diff --git a/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.py b/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.py deleted file mode 100644 index c1a6ea7d57c..00000000000 --- a/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.py +++ /dev/null @@ -1,82 +0,0 @@ -from pathlib import Path -from typing import NamedTuple - -import kfp.deprecated as kfp -from kfp.deprecated.components import create_component_from_func -from kfp.deprecated.components import load_component_from_file - -test_data_dir = Path(__file__).parent / 'test_data' -producer_op = load_component_from_file( - str(test_data_dir / 'produce_2.component.yaml')) -processor_op = load_component_from_file( - str(test_data_dir / 'process_2_2.component.yaml')) -consumer_op = load_component_from_file( - str(test_data_dir / 'consume_2.component.yaml')) - - -def metadata_and_metrics() -> NamedTuple( - "Outputs", - [("mlpipeline_ui_metadata", "UI_metadata"), ("mlpipeline_metrics", "Metrics" - )], -): - metadata = { - "outputs": [{ - "storage": "inline", - "source": "*this should be bold*", - "type": "markdown" - }] - } - metrics = { - "metrics": [ - { - "name": "train-accuracy", - "numberValue": 0.9, - }, - { - "name": "test-accuracy", - "numberValue": 0.7, - }, - ] - } - from collections import namedtuple - import json - - return namedtuple("output", - ["mlpipeline_ui_metadata", "mlpipeline_metrics"])( - json.dumps(metadata), json.dumps(metrics)) - - -@kfp.dsl.pipeline() -def artifact_passing_pipeline(): - producer_task = producer_op() - processor_task = processor_op(producer_task.outputs['output_1'], - producer_task.outputs['output_2']) - consumer_task = consumer_op(processor_task.outputs['output_1'], - processor_task.outputs['output_2']) - - markdown_task = create_component_from_func(func=metadata_and_metrics)() - # This line is only needed for compiling using dsl-compile to work - kfp.dsl.get_pipeline_conf( - ).data_passing_method = volume_based_data_passing_method - - -from kfp.deprecated.dsl import data_passing_methods -from kubernetes.client.models import V1PersistentVolumeClaimVolumeSource -from kubernetes.client.models import V1Volume - -volume_based_data_passing_method = data_passing_methods.KubernetesVolume( - volume=V1Volume( - name='data', - persistent_volume_claim=V1PersistentVolumeClaimVolumeSource( - claim_name='data-volume',), - ), - path_prefix='artifact_data/', -) - -if __name__ == '__main__': - pipeline_conf = kfp.dsl.PipelineConf() - pipeline_conf.data_passing_method = volume_based_data_passing_method - kfp.compiler.Compiler().compile( - artifact_passing_pipeline, - __file__ + '.yaml', - pipeline_conf=pipeline_conf) diff --git a/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.yaml b/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.yaml deleted file mode 100644 index 837f7bc71bf..00000000000 --- a/sdk/python/tests/compiler/testdata/artifact_passing_using_volume.yaml +++ /dev/null @@ -1,234 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: artifact-passing-pipeline- - annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.9, pipelines.kubeflow.org/pipeline_compilation_time: '2021-11-15T11:23:42.469722', - pipelines.kubeflow.org/pipeline_spec: '{"name": "Artifact passing pipeline"}'} - labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.8.9} -spec: - entrypoint: artifact-passing-pipeline - templates: - - name: artifact-passing-pipeline - dag: - tasks: - - name: consumer - template: consumer - dependencies: [processor] - arguments: - parameters: - - {name: processor-Output-1, value: '{{tasks.processor.outputs.parameters.processor-Output-1}}'} - - {name: processor-Output-2-subpath, value: '{{tasks.processor.outputs.parameters.processor-Output-2-subpath}}'} - - {name: metadata-and-metrics, template: metadata-and-metrics} - - name: processor - template: processor - dependencies: [producer] - arguments: - parameters: - - {name: producer-Output-1, value: '{{tasks.producer.outputs.parameters.producer-Output-1}}'} - - {name: producer-Output-2-subpath, value: '{{tasks.producer.outputs.parameters.producer-Output-2-subpath}}'} - - {name: producer, template: producer} - - name: consumer - container: - args: ['{{inputs.parameters.processor-Output-1}}', /tmp/inputs/Input_artifact/data] - command: - - sh - - -c - - | - echo "Input parameter = $0" - echo "Input artifact = " && cat "$1" - image: alpine - volumeMounts: - - {mountPath: /tmp/inputs/Input_artifact, name: data-storage, subPath: '{{inputs.parameters.processor-Output-2-subpath}}', - readOnly: true} - inputs: - parameters: - - {name: processor-Output-1} - - {name: processor-Output-2-subpath} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.9 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": [{"inputValue": "Input parameter"}, {"inputPath": "Input artifact"}], - "command": ["sh", "-c", "echo \"Input parameter = $0\"\necho \"Input artifact - = \" && cat \"$1\"\n"], "image": "alpine"}}, "inputs": [{"name": "Input - parameter"}, {"name": "Input artifact"}], "name": "Consumer"}', pipelines.kubeflow.org/component_ref: '{"digest": - "1a8ea3c29c7853bf63d9b4fbd76a66b273621d2229c3cfe08ed68620ebf02982", "url": - "testdata/test_data/consume_2.component.yaml"}', - pipelines.kubeflow.org/arguments.parameters: '{"Input parameter": "{{inputs.parameters.processor-Output-1}}"}'} - - name: metadata-and-metrics - container: - args: ['----output-paths', /tmp/outputs/mlpipeline_ui_metadata/data, /tmp/outputs/mlpipeline_metrics/data] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def metadata_and_metrics(): - metadata = { - "outputs": [ - {"storage": "inline", "source": "*this should be bold*", "type": "markdown"} - ] - } - metrics = { - "metrics": [ - { - "name": "train-accuracy", - "numberValue": 0.9, - }, - { - "name": "test-accuracy", - "numberValue": 0.7, - }, - ] - } - from collections import namedtuple - import json - - return namedtuple("output", ["mlpipeline_ui_metadata", "mlpipeline_metrics"])( - json.dumps(metadata), json.dumps(metrics) - ) - - import argparse - _parser = argparse.ArgumentParser(prog='Metadata and metrics', description='') - _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=2) - _parsed_args = vars(_parser.parse_args()) - _output_files = _parsed_args.pop("_output_paths", []) - - _outputs = metadata_and_metrics(**_parsed_args) - - _output_serializers = [ - str, - str, - - ] - - import os - for idx, output_file in enumerate(_output_files): - try: - os.makedirs(os.path.dirname(output_file)) - except OSError: - pass - with open(output_file, 'w') as f: - f.write(_output_serializers[idx](_outputs[idx])) - image: python:3.9 - volumeMounts: - - {mountPath: /tmp/outputs/mlpipeline_ui_metadata, name: data-storage, subPath: 'artifact_data/{{workflow.uid}}_{{pod.name}}/mlpipeline-ui-metadata'} - - {mountPath: /tmp/outputs/mlpipeline_metrics, name: data-storage, subPath: 'artifact_data/{{workflow.uid}}_{{pod.name}}/mlpipeline-metrics'} - outputs: - parameters: - - {name: mlpipeline-ui-metadata-subpath, value: 'artifact_data/{{workflow.uid}}_{{pod.name}}/mlpipeline-ui-metadata'} - - {name: mlpipeline-metrics-subpath, value: 'artifact_data/{{workflow.uid}}_{{pod.name}}/mlpipeline-metrics'} - artifacts: - - {name: mlpipeline-ui-metadata, path: /tmp/outputs/mlpipeline_ui_metadata/data} - - {name: mlpipeline-metrics, path: /tmp/outputs/mlpipeline_metrics/data} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.9 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["----output-paths", {"outputPath": "mlpipeline_ui_metadata"}, - {"outputPath": "mlpipeline_metrics"}], "command": ["sh", "-ec", "program_path=$(mktemp)\nprintf - \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", - "def metadata_and_metrics():\n metadata = {\n \"outputs\": [\n {\"storage\": - \"inline\", \"source\": \"*this should be bold*\", \"type\": \"markdown\"}\n ]\n }\n metrics - = {\n \"metrics\": [\n {\n \"name\": \"train-accuracy\",\n \"numberValue\": - 0.9,\n },\n {\n \"name\": \"test-accuracy\",\n \"numberValue\": - 0.7,\n },\n ]\n }\n from collections import namedtuple\n import - json\n\n return namedtuple(\"output\", [\"mlpipeline_ui_metadata\", \"mlpipeline_metrics\"])(\n json.dumps(metadata), - json.dumps(metrics)\n )\n\nimport argparse\n_parser = argparse.ArgumentParser(prog=''Metadata - and metrics'', description='''')\n_parser.add_argument(\"----output-paths\", - dest=\"_output_paths\", type=str, nargs=2)\n_parsed_args = vars(_parser.parse_args())\n_output_files - = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = metadata_and_metrics(**_parsed_args)\n\n_output_serializers - = [\n str,\n str,\n\n]\n\nimport os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except - OSError:\n pass\n with open(output_file, ''w'') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n"], - "image": "python:3.9"}}, "name": "Metadata and metrics", "outputs": [{"name": - "mlpipeline_ui_metadata", "type": "UI_metadata"}, {"name": "mlpipeline_metrics", - "type": "Metrics"}]}', pipelines.kubeflow.org/component_ref: '{}'} - - name: processor - container: - args: ['{{inputs.parameters.producer-Output-1}}', /tmp/inputs/Input_artifact/data, - /tmp/outputs/Output_1/data, /tmp/outputs/Output_2/data] - command: - - sh - - -c - - | - mkdir -p "$(dirname "$2")" - mkdir -p "$(dirname "$3")" - echo "$0" > "$2" - cp "$1" "$3" - image: alpine - volumeMounts: - - {mountPath: /tmp/inputs/Input_artifact, name: data-storage, subPath: '{{inputs.parameters.producer-Output-2-subpath}}', - readOnly: true} - - {mountPath: /tmp/outputs/Output_1, name: data-storage, subPath: 'artifact_data/{{workflow.uid}}_{{pod.name}}/processor-Output-1'} - - {mountPath: /tmp/outputs/Output_2, name: data-storage, subPath: 'artifact_data/{{workflow.uid}}_{{pod.name}}/processor-Output-2'} - inputs: - parameters: - - {name: producer-Output-1} - - {name: producer-Output-2-subpath} - outputs: - parameters: - - name: processor-Output-1 - valueFrom: {path: /tmp/outputs/Output_1/data} - - {name: processor-Output-1-subpath, value: 'artifact_data/{{workflow.uid}}_{{pod.name}}/processor-Output-1'} - - {name: processor-Output-2-subpath, value: 'artifact_data/{{workflow.uid}}_{{pod.name}}/processor-Output-2'} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.9 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": [{"inputValue": "Input parameter"}, {"inputPath": "Input artifact"}, - {"outputPath": "Output 1"}, {"outputPath": "Output 2"}], "command": ["sh", - "-c", "mkdir -p \"$(dirname \"$2\")\"\nmkdir -p \"$(dirname \"$3\")\"\necho - \"$0\" > \"$2\"\ncp \"$1\" \"$3\"\n"], "image": "alpine"}}, "inputs": [{"name": - "Input parameter"}, {"name": "Input artifact"}], "name": "Processor", "outputs": - [{"name": "Output 1"}, {"name": "Output 2"}]}', pipelines.kubeflow.org/component_ref: '{"digest": - "f11a277f5c5cbc27a2e2cda412547b607671d88a4e7aa8a1665dadb836b592b3", "url": - "testdata/test_data/process_2_2.component.yaml"}', - pipelines.kubeflow.org/arguments.parameters: '{"Input parameter": "{{inputs.parameters.producer-Output-1}}"}'} - - name: producer - container: - args: [/tmp/outputs/Output_1/data, /tmp/outputs/Output_2/data] - command: - - sh - - -c - - | - mkdir -p "$(dirname "$0")" - mkdir -p "$(dirname "$1")" - echo "Data 1" > $0 - echo "Data 2" > $1 - image: alpine - volumeMounts: - - {mountPath: /tmp/outputs/Output_1, name: data-storage, subPath: 'artifact_data/{{workflow.uid}}_{{pod.name}}/producer-Output-1'} - - {mountPath: /tmp/outputs/Output_2, name: data-storage, subPath: 'artifact_data/{{workflow.uid}}_{{pod.name}}/producer-Output-2'} - outputs: - parameters: - - name: producer-Output-1 - valueFrom: {path: /tmp/outputs/Output_1/data} - - {name: producer-Output-1-subpath, value: 'artifact_data/{{workflow.uid}}_{{pod.name}}/producer-Output-1'} - - {name: producer-Output-2-subpath, value: 'artifact_data/{{workflow.uid}}_{{pod.name}}/producer-Output-2'} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.8.9 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": [{"outputPath": "Output 1"}, {"outputPath": "Output 2"}], "command": - ["sh", "-c", "mkdir -p \"$(dirname \"$0\")\"\nmkdir -p \"$(dirname \"$1\")\"\necho - \"Data 1\" > $0\necho \"Data 2\" > $1\n"], "image": "alpine"}}, "name": - "Producer", "outputs": [{"name": "Output 1"}, {"name": "Output 2"}]}', pipelines.kubeflow.org/component_ref: '{"digest": - "7399eb54ee94a95708fa9f8a47330a39258b22a319a48458e14a63dcedb87ea4", "url": - "testdata/test_data/produce_2.component.yaml"}'} - arguments: - parameters: [] - serviceAccountName: pipeline-runner - volumes: - - name: data-storage - persistentVolumeClaim: {claimName: data-volume} diff --git a/sdk/python/tests/compiler/testdata/basic.py b/sdk/python/tests/compiler/testdata/basic.py deleted file mode 100644 index 8997bd30548..00000000000 --- a/sdk/python/tests/compiler/testdata/basic.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -import kfp.deprecated.gcp as gcp - - -class GetFrequentWordOp(dsl.ContainerOp): - """A get frequent word class representing a component in ML Pipelines. - - The class provides a nice interface to users by hiding details such - as container, command, arguments. - """ - - def __init__(self, name, message): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - message: a dsl.PipelineParam object representing an input message. - """ - super(GetFrequentWordOp, self).__init__( - name=name, - image='python:3.5-jessie', - command=['sh', '-c'], - arguments=[ - 'python -c "from collections import Counter; ' - 'words = Counter(\'%s\'.split()); print(max(words, key=words.get))" ' - '| tee /tmp/message.txt' % message - ], - file_outputs={'word': '/tmp/message.txt'}) - - -class SaveMessageOp(dsl.ContainerOp): - """A class representing a component in ML Pipelines. - - It saves a message to a given output_path. - """ - - def __init__(self, name, message, output_path): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - message: a dsl.PipelineParam object representing the message to be saved. - output_path: a dsl.PipelineParam object representing the GCS path for output file. - """ - super(SaveMessageOp, self).__init__( - name=name, - image='google/cloud-sdk', - command=['sh', '-c'], - arguments=[ - 'echo %s | tee /tmp/results.txt | gsutil cp /tmp/results.txt %s' - % (message, output_path) - ]) - - -class ExitHandlerOp(dsl.ContainerOp): - """A class representing a component in ML Pipelines.""" - - def __init__(self, name): - super(ExitHandlerOp, self).__init__( - name=name, - image='python:3.5-jessie', - command=['sh', '-c'], - arguments=['echo exit!']) - - -@dsl.pipeline( - name='Save Most Frequent', - description='Get Most Frequent Word and Save to GCS') -def save_most_frequent_word(message: str, outputpath: str): - """A pipeline function describing the orchestration of the workflow.""" - - exit_op = ExitHandlerOp('exiting') - with dsl.ExitHandler(exit_op): - counter = GetFrequentWordOp(name='get-Frequent', message=message) - counter.container.set_memory_request('200M') - - saver = SaveMessageOp( - name='save', message=counter.output, output_path=outputpath) - saver.container.set_cpu_limit('0.5') - saver.container.set_gpu_limit('2') - saver.add_node_selector_constraint('cloud.google.com/gke-accelerator', - 'nvidia-tesla-k80') - saver.apply( - gcp.use_tpu(tpu_cores=8, tpu_resource='v2', tf_version='1.12')) diff --git a/sdk/python/tests/compiler/testdata/basic.yaml b/sdk/python/tests/compiler/testdata/basic.yaml deleted file mode 100644 index 0b15483cc9f..00000000000 --- a/sdk/python/tests/compiler/testdata/basic.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"name": "message", "type": "String"}, {"name": "outputpath", "type": "String"}], "name": "Save Most Frequent"}' - generateName: save-most-frequent- -spec: - arguments: - parameters: - - name: message - - name: outputpath - entrypoint: save-most-frequent - serviceAccountName: pipeline-runner - onExit: exiting - templates: - - dag: - tasks: - - arguments: - parameters: - - name: message - value: '{{inputs.parameters.message}}' - name: get-frequent - template: get-frequent - - arguments: - parameters: - - name: get-frequent-word - value: '{{tasks.get-frequent.outputs.parameters.get-frequent-word}}' - - name: outputpath - value: '{{inputs.parameters.outputpath}}' - dependencies: - - get-frequent - name: save - template: save - inputs: - parameters: - - name: message - - name: outputpath - name: exit-handler-1 - - container: - args: - - echo exit! - command: - - sh - - -c - image: python:3.5-jessie - name: exiting - - container: - args: - - python -c "from collections import Counter; words = Counter('{{inputs.parameters.message}}'.split()); - print(max(words, key=words.get))" | tee /tmp/message.txt - command: - - sh - - -c - image: python:3.5-jessie - resources: - requests: - memory: 200M - inputs: - parameters: - - name: message - name: get-frequent - outputs: - artifacts: - - name: get-frequent-word - path: /tmp/message.txt - parameters: - - name: get-frequent-word - valueFrom: - path: /tmp/message.txt - - container: - args: - - echo {{inputs.parameters.get-frequent-word}} | tee /tmp/results.txt | gsutil - cp /tmp/results.txt {{inputs.parameters.outputpath}} - command: - - sh - - -c - image: google/cloud-sdk - resources: - limits: - cloud-tpus.google.com/v2: "8" - cpu: "0.5" - nvidia.com/gpu: "2" - inputs: - parameters: - - name: get-frequent-word - - name: outputpath - metadata: - annotations: - tf-version.cloud-tpus.google.com: "1.12" - name: save - nodeSelector: - cloud.google.com/gke-accelerator: nvidia-tesla-k80 - - dag: - tasks: - - arguments: - parameters: - - name: message - value: '{{inputs.parameters.message}}' - - name: outputpath - value: '{{inputs.parameters.outputpath}}' - name: exit-handler-1 - template: exit-handler-1 - inputs: - parameters: - - name: message - - name: outputpath - name: save-most-frequent diff --git a/sdk/python/tests/compiler/testdata/basic_no_decorator.py b/sdk/python/tests/compiler/testdata/basic_no_decorator.py deleted file mode 100644 index 8e1063a5d75..00000000000 --- a/sdk/python/tests/compiler/testdata/basic_no_decorator.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -import kfp.deprecated.gcp as gcp - -message_param = dsl.PipelineParam(name='message') -output_path_param = dsl.PipelineParam(name='outputpath', value='default_output') - - -class GetFrequentWordOp(dsl.ContainerOp): - """A get frequent word class representing a component in ML Pipelines. - - The class provides a nice interface to users by hiding details such - as container, command, arguments. - """ - - def __init__(self, name, message): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - message: a dsl.PipelineParam object representing an input message. - """ - super(GetFrequentWordOp, self).__init__( - name=name, - image='python:3.5-jessie', - command=['sh', '-c'], - arguments=[ - 'python -c "from collections import Counter; ' - 'words = Counter(\'%s\'.split()); print(max(words, key=words.get))" ' - '| tee /tmp/message.txt' % message - ], - file_outputs={'word': '/tmp/message.txt'}) - - -class SaveMessageOp(dsl.ContainerOp): - """A class representing a component in ML Pipelines. - - It saves a message to a given output_path. - """ - - def __init__(self, name, message, output_path): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - message: a dsl.PipelineParam object representing the message to be saved. - output_path: a dsl.PipelineParam object representing the GCS path for output file. - """ - super(SaveMessageOp, self).__init__( - name=name, - image='google/cloud-sdk', - command=['sh', '-c'], - arguments=[ - 'echo %s | tee /tmp/results.txt | gsutil cp /tmp/results.txt %s' - % (message, output_path) - ]) - - -class ExitHandlerOp(dsl.ContainerOp): - """A class representing a component in ML Pipelines.""" - - def __init__(self, name): - super(ExitHandlerOp, self).__init__( - name=name, - image='python:3.5-jessie', - command=['sh', '-c'], - arguments=['echo exit!']) - - -def save_most_frequent_word(): - exit_op = ExitHandlerOp('exiting') - with dsl.ExitHandler(exit_op): - counter = GetFrequentWordOp(name='get-Frequent', message=message_param) - counter.container.set_memory_request('200M') - - saver = SaveMessageOp( - name='save', message=counter.output, output_path=output_path_param) - saver.container.set_cpu_limit('0.5') - saver.container.set_gpu_limit('2') - saver.add_node_selector_constraint('cloud.google.com/gke-accelerator', - 'nvidia-tesla-k80') - saver.apply( - gcp.use_tpu(tpu_cores=8, tpu_resource='v2', tf_version='1.12')) diff --git a/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml b/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml deleted file mode 100644 index d548e2c847e..00000000000 --- a/sdk/python/tests/compiler/testdata/basic_no_decorator.yaml +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"name": "message"}, {"default": "default_output", "name": "outputpath"}], "name": "Save Most Frequent"}' - generateName: save-most-frequent- -spec: - arguments: - parameters: - - name: message - - name: outputpath - value: default_output - entrypoint: save-most-frequent - serviceAccountName: pipeline-runner - onExit: exiting - templates: - - dag: - tasks: - - arguments: - parameters: - - name: message - value: '{{inputs.parameters.message}}' - name: get-frequent - template: get-frequent - - arguments: - parameters: - - name: get-frequent-word - value: '{{tasks.get-frequent.outputs.parameters.get-frequent-word}}' - - name: outputpath - value: '{{inputs.parameters.outputpath}}' - dependencies: - - get-frequent - name: save - template: save - inputs: - parameters: - - name: message - - name: outputpath - name: exit-handler-1 - - container: - args: - - echo exit! - command: - - sh - - -c - image: python:3.5-jessie - name: exiting - - container: - args: - - python -c "from collections import Counter; words = Counter('{{inputs.parameters.message}}'.split()); - print(max(words, key=words.get))" | tee /tmp/message.txt - command: - - sh - - -c - image: python:3.5-jessie - resources: - requests: - memory: 200M - inputs: - parameters: - - name: message - name: get-frequent - outputs: - artifacts: - - name: get-frequent-word - path: /tmp/message.txt - parameters: - - name: get-frequent-word - valueFrom: - path: /tmp/message.txt - - container: - args: - - echo {{inputs.parameters.get-frequent-word}} | tee /tmp/results.txt | gsutil - cp /tmp/results.txt {{inputs.parameters.outputpath}} - command: - - sh - - -c - image: google/cloud-sdk - resources: - limits: - cloud-tpus.google.com/v2: "8" - cpu: "0.5" - nvidia.com/gpu: "2" - inputs: - parameters: - - name: get-frequent-word - - name: outputpath - metadata: - annotations: - tf-version.cloud-tpus.google.com: "1.12" - name: save - nodeSelector: - cloud.google.com/gke-accelerator: nvidia-tesla-k80 - - dag: - tasks: - - arguments: - parameters: - - name: message - value: '{{inputs.parameters.message}}' - - name: outputpath - value: '{{inputs.parameters.outputpath}}' - name: exit-handler-1 - template: exit-handler-1 - inputs: - parameters: - - name: message - - name: outputpath - name: save-most-frequent diff --git a/sdk/python/tests/compiler/testdata/coin.py b/sdk/python/tests/compiler/testdata/coin.py deleted file mode 100644 index ed0ce24d4fc..00000000000 --- a/sdk/python/tests/compiler/testdata/coin.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -class FlipCoinOp(dsl.ContainerOp): - - def __init__(self, name): - super(FlipCoinOp, self).__init__( - name=name, - image='python:alpine3.9', - command=['sh', '-c'], - arguments=[ - 'python -c "import random; result = \'heads\' if random.randint(0,1) == 0 ' - 'else \'tails\'; print(result)" | tee /tmp/output' - ], - file_outputs={'output': '/tmp/output'}) - - -class PrintOp(dsl.ContainerOp): - - def __init__(self, name, msg): - super(PrintOp, self).__init__( - name=name, image='alpine:3.9', command=['echo', msg]) - - -@dsl.pipeline( - name='pipeline flip coin', description='shows how to use dsl.Condition.') -def flipcoin(): - flip = FlipCoinOp('flip') - - with dsl.Condition(flip.output == 'heads'): - flip2 = FlipCoinOp('flip-again') - - with dsl.Condition(flip2.output == 'tails'): - PrintOp('print1', flip2.output) - - with dsl.Condition(flip.output == 'tails'): - PrintOp('print2', flip2.output) diff --git a/sdk/python/tests/compiler/testdata/coin.yaml b/sdk/python/tests/compiler/testdata/coin.yaml deleted file mode 100644 index 9e09d640a0f..00000000000 --- a/sdk/python/tests/compiler/testdata/coin.yaml +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", "name": "pipeline flip coin"}' - generateName: pipeline-flip-coin- -spec: - arguments: - parameters: [] - entrypoint: pipeline-flip-coin - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: flip-again-output - value: '{{tasks.flip-again.outputs.parameters.flip-again-output}}' - dependencies: - - flip-again - name: condition-2 - template: condition-2 - when: '"{{tasks.flip-again.outputs.parameters.flip-again-output}}" == "tails"' - - name: flip-again - template: flip-again - name: condition-1 - outputs: - parameters: - - name: flip-again-output - valueFrom: - parameter: '{{tasks.flip-again.outputs.parameters.flip-again-output}}' - - dag: - tasks: - - arguments: - parameters: - - name: flip-again-output - value: '{{inputs.parameters.flip-again-output}}' - name: print1 - template: print1 - inputs: - parameters: - - name: flip-again-output - name: condition-2 - - dag: - tasks: - - arguments: - parameters: - - name: flip-again-output - value: '{{inputs.parameters.flip-again-output}}' - name: print2 - template: print2 - inputs: - parameters: - - name: flip-again-output - name: condition-3 - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.9 - name: flip - outputs: - artifacts: - - name: flip-output - path: /tmp/output - parameters: - - name: flip-output - valueFrom: - path: /tmp/output - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.9 - name: flip-again - outputs: - artifacts: - - name: flip-again-output - path: /tmp/output - parameters: - - name: flip-again-output - valueFrom: - path: /tmp/output - - dag: - tasks: - - dependencies: - - flip - name: condition-1 - template: condition-1 - when: '"{{tasks.flip.outputs.parameters.flip-output}}" == "heads"' - - arguments: - parameters: - - name: flip-again-output - value: '{{tasks.condition-1.outputs.parameters.flip-again-output}}' - dependencies: - - condition-1 - - flip - name: condition-3 - template: condition-3 - when: '"{{tasks.flip.outputs.parameters.flip-output}}" == "tails"' - - name: flip - template: flip - name: pipeline-flip-coin - - container: - command: - - echo - - '{{inputs.parameters.flip-again-output}}' - image: alpine:3.9 - inputs: - parameters: - - name: flip-again-output - name: print1 - - container: - command: - - echo - - '{{inputs.parameters.flip-again-output}}' - image: alpine:3.9 - inputs: - parameters: - - name: flip-again-output - name: print2 diff --git a/sdk/python/tests/compiler/testdata/compose.py b/sdk/python/tests/compiler/testdata/compose.py deleted file mode 100644 index 44049e4b2e8..00000000000 --- a/sdk/python/tests/compiler/testdata/compose.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -class GetFrequentWordOp(dsl.ContainerOp): - """A get frequent word class representing a component in ML Pipelines. - - The class provides a nice interface to users by hiding details such - as container, command, arguments. - """ - - def __init__(self, name, message): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - message: a dsl.PipelineParam object representing an input message. - """ - super(GetFrequentWordOp, self).__init__( - name=name, - image='python:3.5-jessie', - command=['sh', '-c'], - arguments=[ - 'python -c "from collections import Counter; ' - 'words = Counter(\'%s\'.split()); print(max(words, key=words.get))" ' - '| tee /tmp/message.txt' % message - ], - file_outputs={'word': '/tmp/message.txt'}) - - -class SaveMessageOp(dsl.ContainerOp): - """A class representing a component in ML Pipelines. - - It saves a message to a given output_path. - """ - - def __init__(self, name, message, output_path): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - message: a dsl.PipelineParam object representing the message to be saved. - output_path: a dsl.PipelineParam object representing the GCS path for output file. - """ - super(SaveMessageOp, self).__init__( - name=name, - image='google/cloud-sdk', - command=['sh', '-c'], - arguments=[ - 'echo %s | tee /tmp/results.txt | gsutil cp /tmp/results.txt %s' - % (message, output_path) - ]) - - -@dsl.pipeline( - name='Save Most Frequent', - description='Get Most Frequent Word and Save to GCS') -def save_most_frequent_word(message: dsl.PipelineParam, - outputpath: dsl.PipelineParam): - """A pipeline function describing the orchestration of the workflow.""" - - counter = GetFrequentWordOp(name='get-Frequent', message=message) - - saver = SaveMessageOp( - name='save', message=counter.output, output_path=outputpath) - - -class DownloadMessageOp(dsl.ContainerOp): - """A class representing a component in ML Pipelines. - - It downloads a message and outputs it. - """ - - def __init__(self, name, url): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - usl: the gcs url to download the message from. - """ - super(DownloadMessageOp, self).__init__( - name=name, - image='google/cloud-sdk', - command=['sh', '-c'], - arguments=['gsutil cat %s | tee /tmp/results.txt' % url], - file_outputs={'downloaded': '/tmp/results.txt'}) - - -@dsl.pipeline( - name='Download and Save Most Frequent', - description='Download and Get Most Frequent Word and Save to GCS') -def download_save_most_frequent_word(url: str, outputpath: str): - downloader = DownloadMessageOp('download', url) - save_most_frequent_word(downloader.output, outputpath) diff --git a/sdk/python/tests/compiler/testdata/compose.yaml b/sdk/python/tests/compiler/testdata/compose.yaml deleted file mode 100644 index 4d3fa4f0537..00000000000 --- a/sdk/python/tests/compiler/testdata/compose.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "Download and Get Most Frequent Word and Save to GCS", "inputs": [{"name": "url"}, {"name": "outputpath"}], "name": "Download and Save Most Frequent"}' - generateName: download-and-save-most-frequent- -spec: - arguments: - parameters: - - name: url - - name: outputpath - entrypoint: download-and-save-most-frequent - serviceAccountName: pipeline-runner - templates: - - container: - args: - - gsutil cat {{inputs.parameters.url}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk - inputs: - parameters: - - name: url - name: download - outputs: - artifacts: - - name: download-downloaded - path: /tmp/results.txt - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: get-frequent - template: get-frequent - - arguments: - parameters: - - name: get-frequent-word - value: '{{tasks.get-frequent.outputs.parameters.get-frequent-word}}' - - name: outputpath - value: '{{inputs.parameters.outputpath}}' - dependencies: - - get-frequent - name: save - template: save - inputs: - parameters: - - name: outputpath - - name: url - name: download-and-save-most-frequent - - container: - args: - - python -c "from collections import Counter; words = Counter('{{inputs.parameters.download-downloaded}}'.split()); - print(max(words, key=words.get))" | tee /tmp/message.txt - command: - - sh - - -c - image: python:3.5-jessie - inputs: - parameters: - - name: download-downloaded - name: get-frequent - outputs: - artifacts: - - name: get-frequent-word - path: /tmp/message.txt - parameters: - - name: get-frequent-word - valueFrom: - path: /tmp/message.txt - - container: - args: - - echo {{inputs.parameters.get-frequent-word}} | tee /tmp/results.txt | gsutil - cp /tmp/results.txt {{inputs.parameters.outputpath}} - command: - - sh - - -c - image: google/cloud-sdk - inputs: - parameters: - - name: get-frequent-word - - name: outputpath - name: save diff --git a/sdk/python/tests/compiler/testdata/default_value.py b/sdk/python/tests/compiler/testdata/default_value.py deleted file mode 100644 index b6f08dc4709..00000000000 --- a/sdk/python/tests/compiler/testdata/default_value.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline( - name='Default Value', - description='A pipeline with parameter and default value.') -def default_value_pipeline(url='gs://ml-pipeline/shakespeare1.txt'): - - # "url" is a pipeline parameter, meaning users can provide values when running the - # pipeline using UI, CLI, or API to override the default value. - op1 = dsl.ContainerOp( - name='download', - image='google/cloud-sdk', - command=['sh', '-c'], - arguments=['gsutil cat %s | tee /tmp/results.txt' % url], - file_outputs={'downloaded': '/tmp/results.txt'}) - op2 = dsl.ContainerOp( - name='echo', - image='library/bash', - command=['sh', '-c'], - arguments=['echo %s' % op1.output]) diff --git a/sdk/python/tests/compiler/testdata/default_value.yaml b/sdk/python/tests/compiler/testdata/default_value.yaml deleted file mode 100644 index c0be7e2147d..00000000000 --- a/sdk/python/tests/compiler/testdata/default_value.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with parameter and default value.", "inputs": [{"default": "gs://ml-pipeline/shakespeare1.txt", "name": "url"}], "name": "Default Value"}' - generateName: default-value- -spec: - arguments: - parameters: - - name: url - value: gs://ml-pipeline/shakespeare1.txt - entrypoint: default-value - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: url - value: '{{inputs.parameters.url}}' - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: echo - template: echo - inputs: - parameters: - - name: url - name: default-value - - container: - args: - - gsutil cat {{inputs.parameters.url}} | tee /tmp/results.txt - command: - - sh - - -c - image: google/cloud-sdk - inputs: - parameters: - - name: url - name: download - outputs: - artifacts: - - name: download-downloaded - path: /tmp/results.txt - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - - container: - args: - - echo {{inputs.parameters.download-downloaded}} - command: - - sh - - -c - image: library/bash - inputs: - parameters: - - name: download-downloaded - name: echo diff --git a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml b/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml deleted file mode 100644 index 111c70eaddc..00000000000 --- a/sdk/python/tests/compiler/testdata/imagepullsecrets.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "Get Most Frequent Word and Save to GCS", "inputs": [{"name": "message", "type": "String"}], "name": "Save Most Frequent"}' - generateName: save-most-frequent- -spec: - arguments: - parameters: - - name: message - entrypoint: save-most-frequent - imagePullSecrets: - - name: secretA - serviceAccountName: pipeline-runner - templates: - - container: - args: - - python -c "from collections import Counter; words = Counter('{{inputs.parameters.message}}'.split()); - print(max(words, key=words.get))" | tee /tmp/message.txt - command: - - sh - - -c - image: python:3.5-jessie - inputs: - parameters: - - name: message - name: get-frequent - outputs: - artifacts: - - name: get-frequent-word - path: /tmp/message.txt - - dag: - tasks: - - arguments: - parameters: - - name: message - value: '{{inputs.parameters.message}}' - name: get-frequent - template: get-frequent - inputs: - parameters: - - name: message - name: save-most-frequent diff --git a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py deleted file mode 100644 index c1b0c501a76..00000000000 --- a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path -import sys - -sys.path.insert(0, __file__ + '/../../../../') - -from kfp.deprecated import dsl -import kfp.deprecated as kfp - - -def component_with_inline_input_artifact(text: str): - return dsl.ContainerOp( - name='component_with_inline_input_artifact', - image='alpine', - command=[ - 'cat', - dsl.InputArgumentPath( - text, path='/tmp/inputs/text/data', input='text') - ], # path and input are optional - ) - - -def component_with_input_artifact(text): - """A component that passes text as input artifact.""" - - return dsl.ContainerOp( - name='component_with_input_artifact', - artifact_argument_paths=[ - dsl.InputArgumentPath( - argument=text, path='/tmp/inputs/text/data', - input='text'), # path and input are optional - ], - image='alpine', - command=['cat', '/tmp/inputs/text/data'], - ) - - -def component_with_hardcoded_input_artifact_value(): - """A component that passes hard-coded text as input artifact.""" - return component_with_input_artifact('hard-coded artifact value') - - -def component_with_input_artifact_value_from_file(file_path): - """A component that passes contents of a file as input artifact.""" - return component_with_input_artifact(Path(file_path).read_text()) - - -@dsl.pipeline( - name='Pipeline with artifact input raw argument value.', - description='Pipeline shows how to define artifact inputs and pass raw artifacts to them.' -) -def input_artifact_pipeline(): - component_with_inline_input_artifact('Constant artifact value') - component_with_input_artifact('Constant artifact value') - component_with_hardcoded_input_artifact_value() - - file_path = str( - Path(__file__).parent.joinpath('input_artifact_raw_value.txt')) - component_with_input_artifact_value_from_file(file_path) - - -if __name__ == '__main__': - kfp.compiler.Compiler().compile(input_artifact_pipeline, __file__ + '.yaml') diff --git a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.txt b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.txt deleted file mode 100644 index 945a9d53725..00000000000 --- a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.txt +++ /dev/null @@ -1 +0,0 @@ -Text from a file with hard-coded artifact value \ No newline at end of file diff --git a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml b/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml deleted file mode 100644 index 52c3df06063..00000000000 --- a/sdk/python/tests/compiler/testdata/input_artifact_raw_value.yaml +++ /dev/null @@ -1,71 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "Pipeline shows how to define artifact inputs and pass raw artifacts to them.", "name": "Pipeline with artifact input raw argument value."}' - generateName: pipeline-with-artifact-input-raw-argument-value- -spec: - arguments: - parameters: [] - entrypoint: pipeline-with-artifact-input-raw-argument-value - serviceAccountName: pipeline-runner - templates: - - container: - command: - - cat - - /tmp/inputs/text/data - image: alpine - inputs: - artifacts: - - name: text - path: /tmp/inputs/text/data - raw: - data: Constant artifact value - name: component-with-inline-input-artifact - - container: - command: - - cat - - /tmp/inputs/text/data - image: alpine - inputs: - artifacts: - - name: text - path: /tmp/inputs/text/data - raw: - data: Constant artifact value - name: component-with-input-artifact - - container: - command: - - cat - - /tmp/inputs/text/data - image: alpine - inputs: - artifacts: - - name: text - path: /tmp/inputs/text/data - raw: - data: hard-coded artifact value - name: component-with-input-artifact-2 - - container: - command: - - cat - - /tmp/inputs/text/data - image: alpine - inputs: - artifacts: - - name: text - path: /tmp/inputs/text/data - raw: - data: Text from a file with hard-coded artifact value - name: component-with-input-artifact-3 - - dag: - tasks: - - name: component-with-inline-input-artifact - template: component-with-inline-input-artifact - - name: component-with-input-artifact - template: component-with-input-artifact - - name: component-with-input-artifact-2 - template: component-with-input-artifact-2 - - name: component-with-input-artifact-3 - template: component-with-input-artifact-3 - name: pipeline-with-artifact-input-raw-argument-value diff --git a/sdk/python/tests/compiler/testdata/kaniko.basic.yaml b/sdk/python/tests/compiler/testdata/kaniko.basic.yaml deleted file mode 100644 index f709a5dbd38..00000000000 --- a/sdk/python/tests/compiler/testdata/kaniko.basic.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -apiVersion: v1 -kind: Pod -metadata: - generateName: kaniko- - namespace: default - annotations: - sidecar.istio.io/inject: 'false' -spec: - restartPolicy: Never - serviceAccountName: kubeflow-pipelines-container-builder - containers: - - name: kaniko - image: gcr.io/kaniko-project/executor@sha256:78d44ec4e9cb5545d7f85c1924695c89503ded86a59f92c7ae658afa3cff5400 - args: ["--cache=true", - "--dockerfile=dockerfile", - "--context=gs://mlpipeline/kaniko_build.tar.gz", - "--destination=gcr.io/mlpipeline/kaniko_image:latest", - "--digest-file=/dev/termination-log", - ] diff --git a/sdk/python/tests/compiler/testdata/kaniko.kubeflow.yaml b/sdk/python/tests/compiler/testdata/kaniko.kubeflow.yaml deleted file mode 100644 index 126667265eb..00000000000 --- a/sdk/python/tests/compiler/testdata/kaniko.kubeflow.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -apiVersion: v1 -kind: Pod -metadata: - generateName: kaniko- - namespace: user - annotations: - sidecar.istio.io/inject: 'false' -spec: - restartPolicy: Never - serviceAccountName: default-editor - containers: - - name: kaniko - image: gcr.io/kaniko-project/executor@sha256:78d44ec4e9cb5545d7f85c1924695c89503ded86a59f92c7ae658afa3cff5400 - args: ["--cache=true", - "--dockerfile=dockerfile", - "--context=gs://mlpipeline/kaniko_build.tar.gz", - "--destination=gcr.io/mlpipeline/kaniko_image:latest", - "--digest-file=/dev/termination-log", - ] diff --git a/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.py b/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.py deleted file mode 100644 index de3d5b44f71..00000000000 --- a/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated import dsl -import kfp.deprecated as kfp - -produce_op = kfp.components.load_component_from_text('''\ -name: Produce list -outputs: -- name: data_list -implementation: - container: - image: busybox - command: - - sh - - -c - - echo "[1, 2, 3]" > "$0" - - outputPath: data_list -''') - -consume_op = kfp.components.load_component_from_text('''\ -name: Consume data -inputs: -- name: data -implementation: - container: - image: busybox - command: - - echo - - inputValue: data -''') - - -@dsl.pipeline( - name='Loop over lightweight output', - description='Test pipeline to verify functions of par loop.') -def pipeline(): - source_task = produce_op() - with dsl.ParallelFor(source_task.output) as item: - consume_op(item) diff --git a/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.yaml b/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.yaml deleted file mode 100644 index 27112ee71ee..00000000000 --- a/sdk/python/tests/compiler/testdata/loop_over_lightweight_output.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"apiVersion": |- - argoproj.io/v1alpha1 -"kind": |- - Workflow -"metadata": - "annotations": - "pipelines.kubeflow.org/pipeline_spec": |- - {"description": "Test pipeline to verify functions of par loop.", "name": "Loop over lightweight output"} - "generateName": |- - loop-over-lightweight-output- -"spec": - "arguments": - "parameters": [] - "entrypoint": |- - loop-over-lightweight-output - "serviceAccountName": |- - pipeline-runner - "templates": - - "container": - "args": [] - "command": - - |- - echo - - |- - {{inputs.parameters.produce-list-data_list-loop-item}} - "image": |- - busybox - "inputs": - "parameters": - - "name": |- - produce-list-data_list-loop-item - "metadata": - "annotations": - "pipelines.kubeflow.org/component_spec": |- - {"inputs": [{"name": "data"}], "name": "Consume data"} - "name": |- - consume-data - - "dag": - "tasks": - - "arguments": - "parameters": - - "name": |- - produce-list-data_list-loop-item - "value": |- - {{inputs.parameters.produce-list-data_list-loop-item}} - "name": |- - consume-data - "template": |- - consume-data - "inputs": - "parameters": - - "name": |- - produce-list-data_list-loop-item - "name": |- - for-loop-1 - - "dag": - "tasks": - - "arguments": - "parameters": - - "name": |- - produce-list-data_list-loop-item - "value": |- - {{item}} - "dependencies": - - |- - produce-list - "name": |- - for-loop-1 - "template": |- - for-loop-1 - "withParam": |- - {{tasks.produce-list.outputs.parameters.produce-list-data_list}} - - "name": |- - produce-list - "template": |- - produce-list - "name": |- - loop-over-lightweight-output - - "container": - "args": [] - "command": - - |- - sh - - |- - -c - - |- - echo "[1, 2, 3]" > "$0" - - |- - /tmp/outputs/data_list/data - "image": |- - busybox - "metadata": - "annotations": - "pipelines.kubeflow.org/component_spec": |- - {"name": "Produce list", "outputs": [{"name": "data_list"}]} - "name": |- - produce-list - "outputs": - "artifacts": - - "name": |- - produce-list-data_list - "path": |- - /tmp/outputs/data_list/data - "parameters": - - "name": |- - produce-list-data_list - "valueFrom": - "path": |- - /tmp/outputs/data_list/data diff --git a/sdk/python/tests/compiler/testdata/opsgroups.py b/sdk/python/tests/compiler/testdata/opsgroups.py deleted file mode 100644 index 18f3442f568..00000000000 --- a/sdk/python/tests/compiler/testdata/opsgroups.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.graph_component -def echo1_graph_component(text1): - dsl.ContainerOp( - name='echo1-task1', - image='library/bash:4.4.23', - command=['sh', '-c'], - arguments=['echo "$0"', text1]) - - -@dsl.graph_component -def echo2_graph_component(text2): - dsl.ContainerOp( - name='echo2-task1', - image='library/bash:4.4.23', - command=['sh', '-c'], - arguments=['echo "$0"', text2]) - - -@dsl.pipeline() -def opsgroups_pipeline(text1='message 1', text2='message 2'): - step1_graph_component = echo1_graph_component(text1) - step2_graph_component = echo2_graph_component(text2) - step2_graph_component.after(step1_graph_component) diff --git a/sdk/python/tests/compiler/testdata/opsgroups.yaml b/sdk/python/tests/compiler/testdata/opsgroups.yaml deleted file mode 100644 index 8156934eaf9..00000000000 --- a/sdk/python/tests/compiler/testdata/opsgroups.yaml +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: opsgroups-pipeline- - annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.0.0, pipelines.kubeflow.org/pipeline_compilation_time: '2020-08-13T11:25:18.232372', - pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "message 1", "name": - "text1", "optional": true}, {"default": "message 2", "name": "text2", "optional": - true}], "name": "Execution order pipeline"}'} - labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.0.0} -spec: - entrypoint: opsgroups-pipeline - templates: - - name: echo1-task1 - container: - args: [echo "$0", '{{inputs.parameters.text1}}'] - command: [sh, -c] - image: library/bash:4.4.23 - inputs: - parameters: - - {name: text1} - - name: echo2-task1 - container: - args: [echo "$0", '{{inputs.parameters.text2}}'] - command: [sh, -c] - image: library/bash:4.4.23 - inputs: - parameters: - - {name: text2} - - name: graph-echo1-graph-component-1 - inputs: - parameters: - - {name: text1} - dag: - tasks: - - name: echo1-task1 - template: echo1-task1 - arguments: - parameters: - - {name: text1, value: '{{inputs.parameters.text1}}'} - - name: graph-echo2-graph-component-2 - inputs: - parameters: - - {name: text2} - dag: - tasks: - - name: echo2-task1 - template: echo2-task1 - arguments: - parameters: - - {name: text2, value: '{{inputs.parameters.text2}}'} - - name: opsgroups-pipeline - inputs: - parameters: - - {name: text1} - - {name: text2} - dag: - tasks: - - name: graph-echo1-graph-component-1 - template: graph-echo1-graph-component-1 - arguments: - parameters: - - {name: text1, value: '{{inputs.parameters.text1}}'} - - name: graph-echo2-graph-component-2 - template: graph-echo2-graph-component-2 - dependencies: [graph-echo1-graph-component-1] - arguments: - parameters: - - {name: text2, value: '{{inputs.parameters.text2}}'} - arguments: - parameters: - - {name: text1, value: message 1} - - {name: text2, value: message 2} - serviceAccountName: pipeline-runner diff --git a/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.py b/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.py deleted file mode 100644 index 69e19898c55..00000000000 --- a/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.deprecated.components import func_to_container_op - - -@func_to_container_op -def produce_str() -> str: - return "Hello" - - -@func_to_container_op -def produce_list_of_dicts() -> list: - return [{"aaa": "aaa1", "bbb": "bbb1"}, {"aaa": "aaa2", "bbb": "bbb2"}] - - -@func_to_container_op -def produce_list_of_strings() -> list: - return ["a", "z"] - - -@func_to_container_op -def produce_list_of_ints() -> list: - return [1234567890, 987654321] - - -@func_to_container_op -def consume(param1): - print(param1) - - -@kfp.dsl.pipeline() -def parallelfor_item_argument_resolving(): - produce_str_task = produce_str() - produce_list_of_strings_task = produce_list_of_strings() - produce_list_of_ints_task = produce_list_of_ints() - produce_list_of_dicts_task = produce_list_of_dicts() - - with kfp.dsl.ParallelFor(produce_list_of_strings_task.output) as loop_item: - consume(produce_list_of_strings_task.output) - consume(loop_item) - consume(produce_str_task.output) - - with kfp.dsl.ParallelFor(produce_list_of_ints_task.output) as loop_item: - consume(produce_list_of_ints_task.output) - consume(loop_item) - - with kfp.dsl.ParallelFor(produce_list_of_dicts_task.output) as loop_item: - consume(produce_list_of_dicts_task.output) - #consume(loop_item) # Cannot use the full loop item when it's a dict - consume(loop_item.aaa) - - loop_args = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}] - with kfp.dsl.ParallelFor(loop_args) as loop_item: - consume(loop_args) - consume(loop_item) - - -if __name__ == '__main__': - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(parallelfor_item_argument_resolving, - __file__ + '.yaml') diff --git a/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.yaml b/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.yaml deleted file mode 100644 index 09d794fc84f..00000000000 --- a/sdk/python/tests/compiler/testdata/parallelfor_item_argument_resolving.yaml +++ /dev/null @@ -1,788 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: parallelfor-item-argument-resolving- - annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.7.2, pipelines.kubeflow.org/pipeline_compilation_time: '2021-09-03T06:17:22.369925', - pipelines.kubeflow.org/pipeline_spec: '{"name": "Parallelfor item argument resolving"}'} - labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.7.2} -spec: - entrypoint: parallelfor-item-argument-resolving - templates: - - name: consume - container: - args: [--param1, '{{inputs.parameters.produce-list-of-strings-Output}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: produce-list-of-strings-Output} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.produce-list-of-strings-Output}}"}'} - - name: consume-2 - container: - args: [--param1, '{{inputs.parameters.produce-list-of-strings-Output-loop-item}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: produce-list-of-strings-Output-loop-item} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.produce-list-of-strings-Output-loop-item}}"}'} - - name: consume-3 - container: - args: [--param1, '{{inputs.parameters.produce-str-Output}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: produce-str-Output} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.produce-str-Output}}"}'} - - name: consume-4 - container: - args: [--param1, '{{inputs.parameters.produce-list-of-ints-Output}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: produce-list-of-ints-Output} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.produce-list-of-ints-Output}}"}'} - - name: consume-5 - container: - args: [--param1, '{{inputs.parameters.produce-list-of-ints-Output-loop-item}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: produce-list-of-ints-Output-loop-item} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.produce-list-of-ints-Output-loop-item}}"}'} - - name: consume-6 - container: - args: [--param1, '{{inputs.parameters.produce-list-of-dicts-Output}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: produce-list-of-dicts-Output} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.produce-list-of-dicts-Output}}"}'} - - name: consume-7 - container: - args: [--param1, '{{inputs.parameters.produce-list-of-dicts-Output-loop-item-subvar-aaa}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: produce-list-of-dicts-Output-loop-item-subvar-aaa} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.produce-list-of-dicts-Output-loop-item-subvar-aaa}}"}'} - - name: consume-8 - container: - args: [--param1, '[{"a": 1, "b": 2}, {"a": 10, "b": 20}]'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "[{\"a\": 1, \"b\": 2}, {\"a\": 10, \"b\": 20}]"}'} - - name: consume-9 - container: - args: [--param1, '{{inputs.parameters.loop-item-param-4}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: loop-item-param-4} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.loop-item-param-4}}"}'} - - name: for-loop-1 - inputs: - parameters: - - {name: produce-list-of-strings-Output} - - {name: produce-list-of-strings-Output-loop-item} - - {name: produce-str-Output} - dag: - tasks: - - name: consume - template: consume - arguments: - parameters: - - {name: produce-list-of-strings-Output, value: '{{inputs.parameters.produce-list-of-strings-Output}}'} - - name: consume-2 - template: consume-2 - arguments: - parameters: - - {name: produce-list-of-strings-Output-loop-item, value: '{{inputs.parameters.produce-list-of-strings-Output-loop-item}}'} - - name: consume-3 - template: consume-3 - arguments: - parameters: - - {name: produce-str-Output, value: '{{inputs.parameters.produce-str-Output}}'} - - name: for-loop-2 - inputs: - parameters: - - {name: produce-list-of-ints-Output} - - {name: produce-list-of-ints-Output-loop-item} - dag: - tasks: - - name: consume-4 - template: consume-4 - arguments: - parameters: - - {name: produce-list-of-ints-Output, value: '{{inputs.parameters.produce-list-of-ints-Output}}'} - - name: consume-5 - template: consume-5 - arguments: - parameters: - - {name: produce-list-of-ints-Output-loop-item, value: '{{inputs.parameters.produce-list-of-ints-Output-loop-item}}'} - - name: for-loop-3 - inputs: - parameters: - - {name: produce-list-of-dicts-Output} - - {name: produce-list-of-dicts-Output-loop-item-subvar-aaa} - dag: - tasks: - - name: consume-6 - template: consume-6 - arguments: - parameters: - - {name: produce-list-of-dicts-Output, value: '{{inputs.parameters.produce-list-of-dicts-Output}}'} - - name: consume-7 - template: consume-7 - arguments: - parameters: - - {name: produce-list-of-dicts-Output-loop-item-subvar-aaa, value: '{{inputs.parameters.produce-list-of-dicts-Output-loop-item-subvar-aaa}}'} - - name: for-loop-5 - inputs: - parameters: - - {name: loop-item-param-4} - dag: - tasks: - - {name: consume-8, template: consume-8} - - name: consume-9 - template: consume-9 - arguments: - parameters: - - {name: loop-item-param-4, value: '{{inputs.parameters.loop-item-param-4}}'} - - name: parallelfor-item-argument-resolving - dag: - tasks: - - name: for-loop-1 - template: for-loop-1 - dependencies: [produce-list-of-strings, produce-str] - arguments: - parameters: - - {name: produce-list-of-strings-Output, value: '{{tasks.produce-list-of-strings.outputs.parameters.produce-list-of-strings-Output}}'} - - {name: produce-list-of-strings-Output-loop-item, value: '{{item}}'} - - {name: produce-str-Output, value: '{{tasks.produce-str.outputs.parameters.produce-str-Output}}'} - withParam: '{{tasks.produce-list-of-strings.outputs.parameters.produce-list-of-strings-Output}}' - - name: for-loop-2 - template: for-loop-2 - dependencies: [produce-list-of-ints] - arguments: - parameters: - - {name: produce-list-of-ints-Output, value: '{{tasks.produce-list-of-ints.outputs.parameters.produce-list-of-ints-Output}}'} - - {name: produce-list-of-ints-Output-loop-item, value: '{{item}}'} - withParam: '{{tasks.produce-list-of-ints.outputs.parameters.produce-list-of-ints-Output}}' - - name: for-loop-3 - template: for-loop-3 - dependencies: [produce-list-of-dicts] - arguments: - parameters: - - {name: produce-list-of-dicts-Output, value: '{{tasks.produce-list-of-dicts.outputs.parameters.produce-list-of-dicts-Output}}'} - - {name: produce-list-of-dicts-Output-loop-item-subvar-aaa, value: '{{item.aaa}}'} - withParam: '{{tasks.produce-list-of-dicts.outputs.parameters.produce-list-of-dicts-Output}}' - - name: for-loop-5 - template: for-loop-5 - arguments: - parameters: - - {name: loop-item-param-4, value: '{{item}}'} - withItems: - - {a: 1, b: 2} - - {a: 10, b: 20} - - {name: produce-list-of-dicts, template: produce-list-of-dicts} - - {name: produce-list-of-ints, template: produce-list-of-ints} - - {name: produce-list-of-strings, template: produce-list-of-strings} - - {name: produce-str, template: produce-str} - - name: produce-list-of-dicts - container: - args: ['----output-paths', /tmp/outputs/Output/data] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def produce_list_of_dicts(): - return [{"aaa": "aaa1", "bbb": "bbb1"}, {"aaa": "aaa2", "bbb": "bbb2"}] - - def _serialize_json(obj) -> str: - if isinstance(obj, str): - return obj - import json - - def default_serializer(obj): - if hasattr(obj, 'to_struct'): - return obj.to_struct() - else: - raise TypeError( - "Object of type '%s' is not JSON serializable and does not have .to_struct() method." - % obj.__class__.__name__) - - return json.dumps(obj, default=default_serializer, sort_keys=True) - - import argparse - _parser = argparse.ArgumentParser(prog='Produce list of dicts', description='') - _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1) - _parsed_args = vars(_parser.parse_args()) - _output_files = _parsed_args.pop("_output_paths", []) - - _outputs = produce_list_of_dicts(**_parsed_args) - - _outputs = [_outputs] - - _output_serializers = [ - _serialize_json, - - ] - - import os - for idx, output_file in enumerate(_output_files): - try: - os.makedirs(os.path.dirname(output_file)) - except OSError: - pass - with open(output_file, 'w') as f: - f.write(_output_serializers[idx](_outputs[idx])) - image: python:3.9 - outputs: - parameters: - - name: produce-list-of-dicts-Output - valueFrom: {path: /tmp/outputs/Output/data} - artifacts: - - {name: produce-list-of-dicts-Output, path: /tmp/outputs/Output/data} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["----output-paths", {"outputPath": "Output"}], "command": ["sh", - "-ec", "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def produce_list_of_dicts():\n return - [{\"aaa\": \"aaa1\", \"bbb\": \"bbb1\"}, {\"aaa\": \"aaa2\", \"bbb\": \"bbb2\"}]\n\ndef - _serialize_json(obj) -> str:\n if isinstance(obj, str):\n return - obj\n import json\n\n def default_serializer(obj):\n if hasattr(obj, - ''to_struct''):\n return obj.to_struct()\n else:\n raise - TypeError(\n \"Object of type ''%s'' is not JSON serializable - and does not have .to_struct() method.\"\n % obj.__class__.__name__)\n\n return - json.dumps(obj, default=default_serializer, sort_keys=True)\n\nimport argparse\n_parser - = argparse.ArgumentParser(prog=''Produce list of dicts'', description='''')\n_parser.add_argument(\"----output-paths\", - dest=\"_output_paths\", type=str, nargs=1)\n_parsed_args = vars(_parser.parse_args())\n_output_files - = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = produce_list_of_dicts(**_parsed_args)\n\n_outputs - = [_outputs]\n\n_output_serializers = [\n _serialize_json,\n\n]\n\nimport - os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except - OSError:\n pass\n with open(output_file, ''w'') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n"], - "image": "python:3.9"}}, "name": "Produce list of dicts", "outputs": [{"name": - "Output", "type": "JsonArray"}]}', pipelines.kubeflow.org/component_ref: '{}'} - - name: produce-list-of-ints - container: - args: ['----output-paths', /tmp/outputs/Output/data] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def produce_list_of_ints(): - return [1234567890, 987654321] - - def _serialize_json(obj) -> str: - if isinstance(obj, str): - return obj - import json - - def default_serializer(obj): - if hasattr(obj, 'to_struct'): - return obj.to_struct() - else: - raise TypeError( - "Object of type '%s' is not JSON serializable and does not have .to_struct() method." - % obj.__class__.__name__) - - return json.dumps(obj, default=default_serializer, sort_keys=True) - - import argparse - _parser = argparse.ArgumentParser(prog='Produce list of ints', description='') - _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1) - _parsed_args = vars(_parser.parse_args()) - _output_files = _parsed_args.pop("_output_paths", []) - - _outputs = produce_list_of_ints(**_parsed_args) - - _outputs = [_outputs] - - _output_serializers = [ - _serialize_json, - - ] - - import os - for idx, output_file in enumerate(_output_files): - try: - os.makedirs(os.path.dirname(output_file)) - except OSError: - pass - with open(output_file, 'w') as f: - f.write(_output_serializers[idx](_outputs[idx])) - image: python:3.9 - outputs: - parameters: - - name: produce-list-of-ints-Output - valueFrom: {path: /tmp/outputs/Output/data} - artifacts: - - {name: produce-list-of-ints-Output, path: /tmp/outputs/Output/data} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["----output-paths", {"outputPath": "Output"}], "command": ["sh", - "-ec", "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def produce_list_of_ints():\n return - [1234567890, 987654321]\n\ndef _serialize_json(obj) -> str:\n if isinstance(obj, - str):\n return obj\n import json\n\n def default_serializer(obj):\n if - hasattr(obj, ''to_struct''):\n return obj.to_struct()\n else:\n raise - TypeError(\n \"Object of type ''%s'' is not JSON serializable - and does not have .to_struct() method.\"\n % obj.__class__.__name__)\n\n return - json.dumps(obj, default=default_serializer, sort_keys=True)\n\nimport argparse\n_parser - = argparse.ArgumentParser(prog=''Produce list of ints'', description='''')\n_parser.add_argument(\"----output-paths\", - dest=\"_output_paths\", type=str, nargs=1)\n_parsed_args = vars(_parser.parse_args())\n_output_files - = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = produce_list_of_ints(**_parsed_args)\n\n_outputs - = [_outputs]\n\n_output_serializers = [\n _serialize_json,\n\n]\n\nimport - os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except - OSError:\n pass\n with open(output_file, ''w'') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n"], - "image": "python:3.9"}}, "name": "Produce list of ints", "outputs": [{"name": - "Output", "type": "JsonArray"}]}', pipelines.kubeflow.org/component_ref: '{}'} - - name: produce-list-of-strings - container: - args: ['----output-paths', /tmp/outputs/Output/data] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def produce_list_of_strings(): - return ["a", "z"] - - def _serialize_json(obj) -> str: - if isinstance(obj, str): - return obj - import json - - def default_serializer(obj): - if hasattr(obj, 'to_struct'): - return obj.to_struct() - else: - raise TypeError( - "Object of type '%s' is not JSON serializable and does not have .to_struct() method." - % obj.__class__.__name__) - - return json.dumps(obj, default=default_serializer, sort_keys=True) - - import argparse - _parser = argparse.ArgumentParser(prog='Produce list of strings', description='') - _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1) - _parsed_args = vars(_parser.parse_args()) - _output_files = _parsed_args.pop("_output_paths", []) - - _outputs = produce_list_of_strings(**_parsed_args) - - _outputs = [_outputs] - - _output_serializers = [ - _serialize_json, - - ] - - import os - for idx, output_file in enumerate(_output_files): - try: - os.makedirs(os.path.dirname(output_file)) - except OSError: - pass - with open(output_file, 'w') as f: - f.write(_output_serializers[idx](_outputs[idx])) - image: python:3.9 - outputs: - parameters: - - name: produce-list-of-strings-Output - valueFrom: {path: /tmp/outputs/Output/data} - artifacts: - - {name: produce-list-of-strings-Output, path: /tmp/outputs/Output/data} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["----output-paths", {"outputPath": "Output"}], "command": ["sh", - "-ec", "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def produce_list_of_strings():\n return - [\"a\", \"z\"]\n\ndef _serialize_json(obj) -> str:\n if isinstance(obj, - str):\n return obj\n import json\n\n def default_serializer(obj):\n if - hasattr(obj, ''to_struct''):\n return obj.to_struct()\n else:\n raise - TypeError(\n \"Object of type ''%s'' is not JSON serializable - and does not have .to_struct() method.\"\n % obj.__class__.__name__)\n\n return - json.dumps(obj, default=default_serializer, sort_keys=True)\n\nimport argparse\n_parser - = argparse.ArgumentParser(prog=''Produce list of strings'', description='''')\n_parser.add_argument(\"----output-paths\", - dest=\"_output_paths\", type=str, nargs=1)\n_parsed_args = vars(_parser.parse_args())\n_output_files - = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = produce_list_of_strings(**_parsed_args)\n\n_outputs - = [_outputs]\n\n_output_serializers = [\n _serialize_json,\n\n]\n\nimport - os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except - OSError:\n pass\n with open(output_file, ''w'') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n"], - "image": "python:3.9"}}, "name": "Produce list of strings", "outputs": [{"name": - "Output", "type": "JsonArray"}]}', pipelines.kubeflow.org/component_ref: '{}'} - - name: produce-str - container: - args: ['----output-paths', /tmp/outputs/Output/data] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def produce_str(): - return "Hello" - - def _serialize_str(str_value: str) -> str: - if not isinstance(str_value, str): - raise TypeError('Value "{}" has type "{}" instead of str.'.format( - str(str_value), str(type(str_value)))) - return str_value - - import argparse - _parser = argparse.ArgumentParser(prog='Produce str', description='') - _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1) - _parsed_args = vars(_parser.parse_args()) - _output_files = _parsed_args.pop("_output_paths", []) - - _outputs = produce_str(**_parsed_args) - - _outputs = [_outputs] - - _output_serializers = [ - _serialize_str, - - ] - - import os - for idx, output_file in enumerate(_output_files): - try: - os.makedirs(os.path.dirname(output_file)) - except OSError: - pass - with open(output_file, 'w') as f: - f.write(_output_serializers[idx](_outputs[idx])) - image: python:3.9 - outputs: - parameters: - - name: produce-str-Output - valueFrom: {path: /tmp/outputs/Output/data} - artifacts: - - {name: produce-str-Output, path: /tmp/outputs/Output/data} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["----output-paths", {"outputPath": "Output"}], "command": ["sh", - "-ec", "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def produce_str():\n return \"Hello\"\n\ndef - _serialize_str(str_value: str) -> str:\n if not isinstance(str_value, - str):\n raise TypeError(''Value \"{}\" has type \"{}\" instead of - str.''.format(\n str(str_value), str(type(str_value))))\n return - str_value\n\nimport argparse\n_parser = argparse.ArgumentParser(prog=''Produce - str'', description='''')\n_parser.add_argument(\"----output-paths\", dest=\"_output_paths\", - type=str, nargs=1)\n_parsed_args = vars(_parser.parse_args())\n_output_files - = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = produce_str(**_parsed_args)\n\n_outputs - = [_outputs]\n\n_output_serializers = [\n _serialize_str,\n\n]\n\nimport - os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except - OSError:\n pass\n with open(output_file, ''w'') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n"], - "image": "python:3.9"}}, "name": "Produce str", "outputs": [{"name": "Output", - "type": "String"}]}', pipelines.kubeflow.org/component_ref: '{}'} - arguments: - parameters: [] - serviceAccountName: pipeline-runner diff --git a/sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.py b/sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.py deleted file mode 100644 index 675ae60172b..00000000000 --- a/sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated as kfp -from kfp.deprecated.components import func_to_container_op - - -@func_to_container_op -def produce_message(fname1: str) -> str: - return "My name is %s" % fname1 - - -@func_to_container_op -def consume(param1): - print(param1) - - -@kfp.dsl.pipeline() -def parallelfor_pipeline_param_in_items_resolving(fname1: str, fname2: str): - - simple_list = ["My name is %s" % fname1, "My name is %s" % fname2] - - list_of_dict = [{ - "first_name": fname1, - "message": "My name is %s" % fname1 - }, { - "first_name": fname2, - "message": "My name is %s" % fname2 - }] - - list_of_complex_dict = [{ - "first_name": fname1, - "message": produce_message(fname1).output - }, { - "first_name": fname2, - "message": produce_message(fname2).output - }] - - with kfp.dsl.ParallelFor(simple_list) as loop_item: - consume(loop_item) - - with kfp.dsl.ParallelFor(list_of_dict) as loop_item2: - consume(loop_item2.first_name) - consume(loop_item2.message) - - with kfp.dsl.ParallelFor(list_of_complex_dict) as loop_item: - consume(loop_item.first_name) - consume(loop_item.message) - - -if __name__ == '__main__': - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(parallelfor_pipeline_param_in_items_resolving, - __file__ + '.yaml') diff --git a/sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.yaml b/sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.yaml deleted file mode 100644 index a4266ce56cc..00000000000 --- a/sdk/python/tests/compiler/testdata/parallelfor_pipeline_param_in_items_resolving.yaml +++ /dev/null @@ -1,455 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: parallelfor-pipeline-param-in-items-resolving- - annotations: {pipelines.kubeflow.org/kfp_sdk_version: 1.7.2, pipelines.kubeflow.org/pipeline_compilation_time: '2021-09-03T06:18:38.373787', - pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"name": "fname1", "type": - "String"}, {"name": "fname2", "type": "String"}], "name": "Parallelfor pipeline - param in items resolving"}'} - labels: {pipelines.kubeflow.org/kfp_sdk_version: 1.7.2} -spec: - entrypoint: parallelfor-pipeline-param-in-items-resolving - templates: - - name: consume - container: - args: [--param1, '{{inputs.parameters.loop-item-param-1}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: loop-item-param-1} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.loop-item-param-1}}"}'} - - name: consume-2 - container: - args: [--param1, '{{inputs.parameters.loop-item-param-3-subvar-first_name}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: loop-item-param-3-subvar-first_name} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.loop-item-param-3-subvar-first_name}}"}'} - - name: consume-3 - container: - args: [--param1, '{{inputs.parameters.loop-item-param-3-subvar-message}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: loop-item-param-3-subvar-message} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.loop-item-param-3-subvar-message}}"}'} - - name: consume-4 - container: - args: [--param1, '{{inputs.parameters.loop-item-param-5-subvar-first_name}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: loop-item-param-5-subvar-first_name} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.loop-item-param-5-subvar-first_name}}"}'} - - name: consume-5 - container: - args: [--param1, '{{inputs.parameters.loop-item-param-5-subvar-message}}'] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def consume(param1): - print(param1) - - import argparse - _parser = argparse.ArgumentParser(prog='Consume', description='') - _parser.add_argument("--param1", dest="param1", type=str, required=True, default=argparse.SUPPRESS) - _parsed_args = vars(_parser.parse_args()) - - _outputs = consume(**_parsed_args) - image: python:3.9 - inputs: - parameters: - - {name: loop-item-param-5-subvar-message} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--param1", {"inputValue": "param1"}], "command": ["sh", "-ec", - "program_path=$(mktemp)\nprintf \"%s\" \"$0\" > \"$program_path\"\npython3 - -u \"$program_path\" \"$@\"\n", "def consume(param1):\n print(param1)\n\nimport - argparse\n_parser = argparse.ArgumentParser(prog=''Consume'', description='''')\n_parser.add_argument(\"--param1\", - dest=\"param1\", type=str, required=True, default=argparse.SUPPRESS)\n_parsed_args - = vars(_parser.parse_args())\n\n_outputs = consume(**_parsed_args)\n"], - "image": "python:3.9"}}, "inputs": [{"name": "param1"}], "name": "Consume"}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"param1": - "{{inputs.parameters.loop-item-param-5-subvar-message}}"}'} - - name: for-loop-2 - inputs: - parameters: - - {name: loop-item-param-1} - dag: - tasks: - - name: consume - template: consume - arguments: - parameters: - - {name: loop-item-param-1, value: '{{inputs.parameters.loop-item-param-1}}'} - - name: for-loop-4 - inputs: - parameters: - - {name: loop-item-param-3-subvar-first_name} - - {name: loop-item-param-3-subvar-message} - dag: - tasks: - - name: consume-2 - template: consume-2 - arguments: - parameters: - - {name: loop-item-param-3-subvar-first_name, value: '{{inputs.parameters.loop-item-param-3-subvar-first_name}}'} - - name: consume-3 - template: consume-3 - arguments: - parameters: - - {name: loop-item-param-3-subvar-message, value: '{{inputs.parameters.loop-item-param-3-subvar-message}}'} - - name: for-loop-6 - inputs: - parameters: - - {name: loop-item-param-5-subvar-first_name} - - {name: loop-item-param-5-subvar-message} - dag: - tasks: - - name: consume-4 - template: consume-4 - arguments: - parameters: - - {name: loop-item-param-5-subvar-first_name, value: '{{inputs.parameters.loop-item-param-5-subvar-first_name}}'} - - name: consume-5 - template: consume-5 - arguments: - parameters: - - {name: loop-item-param-5-subvar-message, value: '{{inputs.parameters.loop-item-param-5-subvar-message}}'} - - name: parallelfor-pipeline-param-in-items-resolving - inputs: - parameters: - - {name: fname1} - - {name: fname2} - dag: - tasks: - - name: for-loop-2 - template: for-loop-2 - arguments: - parameters: - - {name: loop-item-param-1, value: '{{item}}'} - withItems: ['My name is {{workflow.parameters.fname1}}', 'My name is {{workflow.parameters.fname2}}'] - - name: for-loop-4 - template: for-loop-4 - arguments: - parameters: - - {name: loop-item-param-3-subvar-first_name, value: '{{item.first_name}}'} - - {name: loop-item-param-3-subvar-message, value: '{{item.message}}'} - withItems: - - {first_name: '{{workflow.parameters.fname1}}', message: 'My name is {{workflow.parameters.fname1}}'} - - {first_name: '{{workflow.parameters.fname2}}', message: 'My name is {{workflow.parameters.fname2}}'} - - name: for-loop-6 - template: for-loop-6 - arguments: - parameters: - - {name: loop-item-param-5-subvar-first_name, value: '{{item.first_name}}'} - - {name: loop-item-param-5-subvar-message, value: '{{item.message}}'} - dependencies: [produce-message, produce-message-2] - withItems: - - {first_name: '{{workflow.parameters.fname1}}', message: '{{tasks.produce-message.outputs.parameters.produce-message-Output}}'} - - {first_name: '{{workflow.parameters.fname2}}', message: '{{tasks.produce-message-2.outputs.parameters.produce-message-2-Output}}'} - - name: produce-message - template: produce-message - arguments: - parameters: - - {name: fname1, value: '{{inputs.parameters.fname1}}'} - - name: produce-message-2 - template: produce-message-2 - arguments: - parameters: - - {name: fname2, value: '{{inputs.parameters.fname2}}'} - - name: produce-message - container: - args: [--fname1, '{{inputs.parameters.fname1}}', '----output-paths', /tmp/outputs/Output/data] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def produce_message(fname1): - return "My name is %s" % fname1 - - def _serialize_str(str_value: str) -> str: - if not isinstance(str_value, str): - raise TypeError('Value "{}" has type "{}" instead of str.'.format( - str(str_value), str(type(str_value)))) - return str_value - - import argparse - _parser = argparse.ArgumentParser(prog='Produce message', description='') - _parser.add_argument("--fname1", dest="fname1", type=str, required=True, default=argparse.SUPPRESS) - _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1) - _parsed_args = vars(_parser.parse_args()) - _output_files = _parsed_args.pop("_output_paths", []) - - _outputs = produce_message(**_parsed_args) - - _outputs = [_outputs] - - _output_serializers = [ - _serialize_str, - - ] - - import os - for idx, output_file in enumerate(_output_files): - try: - os.makedirs(os.path.dirname(output_file)) - except OSError: - pass - with open(output_file, 'w') as f: - f.write(_output_serializers[idx](_outputs[idx])) - image: python:3.9 - inputs: - parameters: - - {name: fname1} - outputs: - parameters: - - name: produce-message-Output - valueFrom: {path: /tmp/outputs/Output/data} - artifacts: - - {name: produce-message-Output, path: /tmp/outputs/Output/data} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--fname1", {"inputValue": "fname1"}, "----output-paths", {"outputPath": - "Output"}], "command": ["sh", "-ec", "program_path=$(mktemp)\nprintf \"%s\" - \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", "def - produce_message(fname1):\n return \"My name is %s\" % fname1\n\ndef _serialize_str(str_value: - str) -> str:\n if not isinstance(str_value, str):\n raise TypeError(''Value - \"{}\" has type \"{}\" instead of str.''.format(\n str(str_value), - str(type(str_value))))\n return str_value\n\nimport argparse\n_parser - = argparse.ArgumentParser(prog=''Produce message'', description='''')\n_parser.add_argument(\"--fname1\", - dest=\"fname1\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"----output-paths\", - dest=\"_output_paths\", type=str, nargs=1)\n_parsed_args = vars(_parser.parse_args())\n_output_files - = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = produce_message(**_parsed_args)\n\n_outputs - = [_outputs]\n\n_output_serializers = [\n _serialize_str,\n\n]\n\nimport - os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except - OSError:\n pass\n with open(output_file, ''w'') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n"], - "image": "python:3.9"}}, "inputs": [{"name": "fname1", "type": "String"}], - "name": "Produce message", "outputs": [{"name": "Output", "type": "String"}]}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"fname1": - "{{inputs.parameters.fname1}}"}'} - - name: produce-message-2 - container: - args: [--fname1, '{{inputs.parameters.fname2}}', '----output-paths', /tmp/outputs/Output/data] - command: - - sh - - -ec - - | - program_path=$(mktemp) - printf "%s" "$0" > "$program_path" - python3 -u "$program_path" "$@" - - | - def produce_message(fname1): - return "My name is %s" % fname1 - - def _serialize_str(str_value: str) -> str: - if not isinstance(str_value, str): - raise TypeError('Value "{}" has type "{}" instead of str.'.format( - str(str_value), str(type(str_value)))) - return str_value - - import argparse - _parser = argparse.ArgumentParser(prog='Produce message', description='') - _parser.add_argument("--fname1", dest="fname1", type=str, required=True, default=argparse.SUPPRESS) - _parser.add_argument("----output-paths", dest="_output_paths", type=str, nargs=1) - _parsed_args = vars(_parser.parse_args()) - _output_files = _parsed_args.pop("_output_paths", []) - - _outputs = produce_message(**_parsed_args) - - _outputs = [_outputs] - - _output_serializers = [ - _serialize_str, - - ] - - import os - for idx, output_file in enumerate(_output_files): - try: - os.makedirs(os.path.dirname(output_file)) - except OSError: - pass - with open(output_file, 'w') as f: - f.write(_output_serializers[idx](_outputs[idx])) - image: python:3.9 - inputs: - parameters: - - {name: fname2} - outputs: - parameters: - - name: produce-message-2-Output - valueFrom: {path: /tmp/outputs/Output/data} - artifacts: - - {name: produce-message-2-Output, path: /tmp/outputs/Output/data} - metadata: - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.7.2 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/enable_caching: "true" - annotations: {pipelines.kubeflow.org/component_spec: '{"implementation": {"container": - {"args": ["--fname1", {"inputValue": "fname1"}, "----output-paths", {"outputPath": - "Output"}], "command": ["sh", "-ec", "program_path=$(mktemp)\nprintf \"%s\" - \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", "def - produce_message(fname1):\n return \"My name is %s\" % fname1\n\ndef _serialize_str(str_value: - str) -> str:\n if not isinstance(str_value, str):\n raise TypeError(''Value - \"{}\" has type \"{}\" instead of str.''.format(\n str(str_value), - str(type(str_value))))\n return str_value\n\nimport argparse\n_parser - = argparse.ArgumentParser(prog=''Produce message'', description='''')\n_parser.add_argument(\"--fname1\", - dest=\"fname1\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"----output-paths\", - dest=\"_output_paths\", type=str, nargs=1)\n_parsed_args = vars(_parser.parse_args())\n_output_files - = _parsed_args.pop(\"_output_paths\", [])\n\n_outputs = produce_message(**_parsed_args)\n\n_outputs - = [_outputs]\n\n_output_serializers = [\n _serialize_str,\n\n]\n\nimport - os\nfor idx, output_file in enumerate(_output_files):\n try:\n os.makedirs(os.path.dirname(output_file))\n except - OSError:\n pass\n with open(output_file, ''w'') as f:\n f.write(_output_serializers[idx](_outputs[idx]))\n"], - "image": "python:3.9"}}, "inputs": [{"name": "fname1", "type": "String"}], - "name": "Produce message", "outputs": [{"name": "Output", "type": "String"}]}', - pipelines.kubeflow.org/component_ref: '{}', pipelines.kubeflow.org/arguments.parameters: '{"fname1": - "{{inputs.parameters.fname2}}"}'} - arguments: - parameters: - - {name: fname1} - - {name: fname2} - serviceAccountName: pipeline-runner diff --git a/sdk/python/tests/compiler/testdata/param_op_transform.py b/sdk/python/tests/compiler/testdata/param_op_transform.py deleted file mode 100644 index 418435bd72c..00000000000 --- a/sdk/python/tests/compiler/testdata/param_op_transform.py +++ /dev/null @@ -1,27 +0,0 @@ -import kfp.deprecated.dsl as dsl - - -def add_common_labels(param): - - def _add_common_labels(op: dsl.ContainerOp) -> dsl.ContainerOp: - return op.add_pod_label('param', param) - - return _add_common_labels - - -@dsl.pipeline( - name="Parameters in Op transformation functions", - description="Test that parameters used in Op transformation functions as pod labels " - "would be correcly identified and set as arguments in he generated yaml") -def param_substitutions(param): - dsl.get_pipeline_conf().op_transformers.append(add_common_labels(param)) - - op = dsl.ContainerOp( - name="cop", - image="image", - ) - - -if __name__ == '__main__': - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(param_substitutions, __file__ + '.yaml') diff --git a/sdk/python/tests/compiler/testdata/param_op_transform.yaml b/sdk/python/tests/compiler/testdata/param_op_transform.yaml deleted file mode 100644 index b0da3751f66..00000000000 --- a/sdk/python/tests/compiler/testdata/param_op_transform.yaml +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "Test that parameters used - in Op transformation functions as pod labels would be correcly identified and - set as arguments in he generated yaml", "inputs": [{"name": "param"}], "name": "Parameters in Op transformation - functions"}' - generateName: parameters-in-op-transformation-functions- -spec: - arguments: - parameters: - - name: param - entrypoint: parameters-in-op-transformation-functions - serviceAccountName: pipeline-runner - templates: - - container: - image: image - inputs: - parameters: - - name: param - metadata: - labels: - param: '{{inputs.parameters.param}}' - name: cop - - dag: - tasks: - - arguments: - parameters: - - name: param - value: '{{inputs.parameters.param}}' - name: cop - template: cop - inputs: - parameters: - - name: param - name: parameters-in-op-transformation-functions diff --git a/sdk/python/tests/compiler/testdata/param_substitutions.py b/sdk/python/tests/compiler/testdata/param_substitutions.py deleted file mode 100644 index 8a82bc1ed70..00000000000 --- a/sdk/python/tests/compiler/testdata/param_substitutions.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline( - name="Param Substitutions", - description="Test the same PipelineParam getting substituted in multiple " - "places") -def param_substitutions(): - vop = dsl.VolumeOp(name="create_volume", resource_name="data", size="1Gi") - - op = dsl.ContainerOp( - name="cop", - image="image", - arguments=["--param", vop.output], - pvolumes={"/mnt": vop.volume}) - - -if __name__ == '__main__': - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(param_substitutions, __file__ + '.tar.gz') diff --git a/sdk/python/tests/compiler/testdata/param_substitutions.yaml b/sdk/python/tests/compiler/testdata/param_substitutions.yaml deleted file mode 100644 index 76f1f1a9f8b..00000000000 --- a/sdk/python/tests/compiler/testdata/param_substitutions.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "Test the same PipelineParam - getting substituted in multiple places", "name": "Param Substitutions"}' - generateName: param-substitutions- -spec: - arguments: - parameters: [] - entrypoint: param-substitutions - serviceAccountName: pipeline-runner - templates: - - container: - args: - - --param - - '{{inputs.parameters.create-volume-name}}' - image: image - volumeMounts: - - mountPath: /mnt - name: create-volume - inputs: - parameters: - - name: create-volume-name - name: cop - volumes: - - name: create-volume - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-volume-name}}' - - name: create-volume - outputs: - parameters: - - name: create-volume-manifest - valueFrom: - jsonPath: '{}' - - name: create-volume-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: create-volume-size - valueFrom: - jsonPath: '{.status.capacity.storage}' - resource: - action: create - manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-data'\n\ - spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ - \ storage: 1Gi\n" - - dag: - tasks: - - arguments: - parameters: - - name: create-volume-name - value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' - dependencies: - - create-volume - name: cop - template: cop - - name: create-volume - template: create-volume - name: param-substitutions diff --git a/sdk/python/tests/compiler/testdata/pipelineparams.py b/sdk/python/tests/compiler/testdata/pipelineparams.py deleted file mode 100644 index 0b0e61dab1c..00000000000 --- a/sdk/python/tests/compiler/testdata/pipelineparams.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -from kubernetes.client.models import V1EnvVar - - -@dsl.pipeline( - name='PipelineParams', - description='A pipeline with multiple pipeline params.') -def pipelineparams_pipeline(tag: str = 'latest', sleep_ms: int = 10): - - echo = dsl.Sidecar( - name='echo', - image='hashicorp/http-echo:%s' % tag, - args=['-text="hello world"'], - ) - - op1 = dsl.ContainerOp( - name='download', - image='busybox:%s' % tag, - command=['sh', '-c'], - arguments=[ - 'sleep %s; wget localhost:5678 -O /tmp/results.txt' % sleep_ms - ], - sidecars=[echo], - file_outputs={'downloaded': '/tmp/results.txt'}) - - op2 = dsl.ContainerOp( - name='echo', - image='library/bash', - command=['sh', '-c'], - arguments=['echo $MSG %s' % op1.output]) - - op2.container.add_env_variable( - V1EnvVar(name='MSG', value='pipelineParams: ')) diff --git a/sdk/python/tests/compiler/testdata/pipelineparams.yaml b/sdk/python/tests/compiler/testdata/pipelineparams.yaml deleted file mode 100644 index 966855f9c04..00000000000 --- a/sdk/python/tests/compiler/testdata/pipelineparams.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -apiVersion: argoproj.io/v1alpha1 -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with multiple - pipeline params.", "inputs": [{"default": "latest", "name": - "tag"}, {"default": "10", "name": "sleep_ms"}], "name": "PipelineParams"}' - generateName: pipelineparams- -spec: - entrypoint: pipelineparams - arguments: - parameters: - - name: tag - value: latest - - name: sleep_ms - value: '10' - templates: - - name: download - inputs: - parameters: - - name: sleep_ms - - name: tag - container: - image: busybox:{{inputs.parameters.tag}} - args: - - sleep {{inputs.parameters.sleep_ms}}; wget localhost:5678 -O /tmp/results.txt - command: - - sh - - "-c" - outputs: - artifacts: - - name: download-downloaded - path: "/tmp/results.txt" - parameters: - - name: download-downloaded - valueFrom: - path: "/tmp/results.txt" - sidecars: - - image: hashicorp/http-echo:{{inputs.parameters.tag}} - name: echo - args: - - -text="hello world" - - name: echo - inputs: - parameters: - - name: download-downloaded - container: - image: library/bash - args: - - echo $MSG {{inputs.parameters.download-downloaded}} - command: - - sh - - "-c" - env: - - name: MSG - value: 'pipelineParams: ' - - name: pipelineparams - inputs: - parameters: - - name: sleep_ms - - name: tag - dag: - tasks: - - name: download - arguments: - parameters: - - name: sleep_ms - value: "{{inputs.parameters.sleep_ms}}" - - name: tag - value: "{{inputs.parameters.tag}}" - template: download - - dependencies: - - download - arguments: - parameters: - - name: download-downloaded - value: "{{tasks.download.outputs.parameters.download-downloaded}}" - name: echo - template: echo - serviceAccountName: pipeline-runner -kind: Workflow diff --git a/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml b/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml deleted file mode 100644 index 2710fc7c8d5..00000000000 --- a/sdk/python/tests/compiler/testdata/preemptible_tpu_gpu.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", "name": "pipeline flip coin"}' - generateName: pipeline-flip-coin- -spec: - arguments: - parameters: [] - entrypoint: pipeline-flip-coin - serviceAccountName: pipeline-runner - templates: - - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: cloud.google.com/gke-preemptible - operator: In - values: - - 'true' - weight: 50 - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.9 - resources: - limits: - nvidia.com/gpu: 1 - name: flip - outputs: - artifacts: - - name: flip-output - path: /tmp/output - retryStrategy: - limit: 5 - tolerations: - - effect: NoSchedule - key: preemptible - operator: Equal - value: 'true' - - dag: - tasks: - - name: flip - template: flip - name: pipeline-flip-coin \ No newline at end of file diff --git a/sdk/python/tests/compiler/testdata/recursive_do_while.py b/sdk/python/tests/compiler/testdata/recursive_do_while.py deleted file mode 100644 index 188595f276b..00000000000 --- a/sdk/python/tests/compiler/testdata/recursive_do_while.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from kfp.deprecated.dsl import graph_component -import kfp.deprecated.dsl as dsl - - -class FlipCoinOp(dsl.ContainerOp): - """Flip a coin and output heads or tails randomly.""" - - def __init__(self): - super(FlipCoinOp, self).__init__( - name='Flip', - image='python:alpine3.9', - command=['sh', '-c'], - arguments=[ - 'python -c "import random; result = \'heads\' if random.randint(0,1) == 0 ' - 'else \'tails\'; print(result)" | tee /tmp/output' - ], - file_outputs={'output': '/tmp/output'}) - - -class PrintOp(dsl.ContainerOp): - """Print a message.""" - - def __init__(self, msg): - super(PrintOp, self).__init__( - name='Print', - image='alpine:3.9', - command=['echo', msg], - ) - - -@graph_component -def flip_component(flip_result): - print_flip = PrintOp(flip_result) - flipA = FlipCoinOp().after(print_flip) - with dsl.Condition(flipA.output == 'heads'): - flip_component(flipA.output) - - -@dsl.pipeline( - name='pipeline flip coin', description='shows how to use graph_component.') -def recursive(): - flipA = FlipCoinOp() - flipB = FlipCoinOp() - flip_loop = flip_component(flipA.output) - flip_loop.after(flipB) - PrintOp('cool, it is over. %s' % flipA.output).after(flip_loop) - - -if __name__ == '__main__': - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(recursive, __file__ + '.tar.gz') diff --git a/sdk/python/tests/compiler/testdata/recursive_do_while.yaml b/sdk/python/tests/compiler/testdata/recursive_do_while.yaml deleted file mode 100644 index 2f7ce454fe3..00000000000 --- a/sdk/python/tests/compiler/testdata/recursive_do_while.yaml +++ /dev/null @@ -1,140 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use graph_component.", - "name": "pipeline flip coin"}' - generateName: pipeline-flip-coin- -spec: - arguments: - parameters: [] - entrypoint: pipeline-flip-coin - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-3-output}}' - name: graph-flip-component-1 - template: graph-flip-component-1 - inputs: - parameters: - - name: flip-3-output - name: condition-2 - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.9 - name: flip - outputs: - artifacts: - - name: flip-output - path: /tmp/output - parameters: - - name: flip-output - valueFrom: - path: /tmp/output - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.9 - name: flip-2 - outputs: - artifacts: - - name: flip-2-output - path: /tmp/output - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.9 - name: flip-3 - outputs: - artifacts: - - name: flip-3-output - path: /tmp/output - parameters: - - name: flip-3-output - valueFrom: - path: /tmp/output - - dag: - tasks: - - arguments: - parameters: - - name: flip-3-output - value: '{{tasks.flip-3.outputs.parameters.flip-3-output}}' - dependencies: - - flip-3 - name: condition-2 - template: condition-2 - when: '"{{tasks.flip-3.outputs.parameters.flip-3-output}}" == "heads"' - - dependencies: - - print - name: flip-3 - template: flip-3 - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: print - template: print - inputs: - parameters: - - name: flip-output - name: graph-flip-component-1 - - dag: - tasks: - - name: flip - template: flip - - name: flip-2 - template: flip-2 - - arguments: - parameters: - - name: flip-output - value: '{{tasks.flip.outputs.parameters.flip-output}}' - dependencies: - - flip - - flip-2 - name: graph-flip-component-1 - template: graph-flip-component-1 - - arguments: - parameters: - - name: flip-output - value: '{{tasks.flip.outputs.parameters.flip-output}}' - dependencies: - - flip - - graph-flip-component-1 - name: print-2 - template: print-2 - name: pipeline-flip-coin - - container: - command: - - echo - - '{{inputs.parameters.flip-output}}' - image: alpine:3.9 - inputs: - parameters: - - name: flip-output - name: print - - container: - command: - - echo - - cool, it is over. {{inputs.parameters.flip-output}} - image: alpine:3.9 - inputs: - parameters: - - name: flip-output - name: print-2 diff --git a/sdk/python/tests/compiler/testdata/recursive_while.py b/sdk/python/tests/compiler/testdata/recursive_while.py deleted file mode 100644 index 4b7b442bd01..00000000000 --- a/sdk/python/tests/compiler/testdata/recursive_while.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -class FlipCoinOp(dsl.ContainerOp): - """Flip a coin and output heads or tails randomly.""" - - def __init__(self): - super(FlipCoinOp, self).__init__( - name='Flip', - image='python:alpine3.9', - command=['sh', '-c'], - arguments=[ - 'python -c "import random; result = \'heads\' if random.randint(0,1) == 0 ' - 'else \'tails\'; print(result)" | tee /tmp/output' - ], - file_outputs={'output': '/tmp/output'}) - - -class PrintOp(dsl.ContainerOp): - """Print a message.""" - - def __init__(self, msg): - super(PrintOp, self).__init__( - name='Print', - image='alpine:3.9', - command=['echo', msg], - ) - - -@dsl._component.graph_component -def flip_component(flip_result, maxVal): - with dsl.Condition(flip_result == 'heads'): - print_flip = PrintOp(flip_result) - flipA = FlipCoinOp().after(print_flip) - flip_component(flipA.output, maxVal) - - -@dsl.pipeline( - name='pipeline flip coin', description='shows how to use dsl.Condition.') -def flipcoin(maxVal=12): - flipA = FlipCoinOp() - flipB = FlipCoinOp() - flip_loop = flip_component(flipA.output, maxVal) - flip_loop.after(flipB) - PrintOp('cool, it is over. %s' % flipA.output).after(flip_loop) - - -if __name__ == '__main__': - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(flipcoin, __file__ + '.tar.gz') diff --git a/sdk/python/tests/compiler/testdata/recursive_while.yaml b/sdk/python/tests/compiler/testdata/recursive_while.yaml deleted file mode 100644 index b6448522cd7..00000000000 --- a/sdk/python/tests/compiler/testdata/recursive_while.yaml +++ /dev/null @@ -1,142 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use dsl.Condition.", - "inputs": [{"default": "12", "name": "maxVal"}], "name": "pipeline flip coin"}' - generateName: pipeline-flip-coin- -spec: - arguments: - parameters: - - name: maxVal - value: '12' - entrypoint: pipeline-flip-coin - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - dependencies: - - print - name: flip-3 - template: flip-3 - - arguments: - parameters: - - name: flip-output - value: '{{tasks.flip-3.outputs.parameters.flip-3-output}}' - dependencies: - - flip-3 - name: graph-flip-component-1 - template: graph-flip-component-1 - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: print - template: print - inputs: - parameters: - - name: flip-output - name: condition-2 - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.9 - name: flip - outputs: - artifacts: - - name: flip-output - path: /tmp/output - parameters: - - name: flip-output - valueFrom: - path: /tmp/output - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.9 - name: flip-2 - outputs: - artifacts: - - name: flip-2-output - path: /tmp/output - - container: - args: - - python -c "import random; result = 'heads' if random.randint(0,1) == 0 else - 'tails'; print(result)" | tee /tmp/output - command: - - sh - - -c - image: python:alpine3.9 - name: flip-3 - outputs: - artifacts: - - name: flip-3-output - path: /tmp/output - parameters: - - name: flip-3-output - valueFrom: - path: /tmp/output - - dag: - tasks: - - arguments: - parameters: - - name: flip-output - value: '{{inputs.parameters.flip-output}}' - name: condition-2 - template: condition-2 - when: '"{{inputs.parameters.flip-output}}" == "heads"' - inputs: - parameters: - - name: flip-output - name: graph-flip-component-1 - - dag: - tasks: - - name: flip - template: flip - - name: flip-2 - template: flip-2 - - arguments: - parameters: - - name: flip-output - value: '{{tasks.flip.outputs.parameters.flip-output}}' - dependencies: - - flip - - flip-2 - name: graph-flip-component-1 - template: graph-flip-component-1 - - arguments: - parameters: - - name: flip-output - value: '{{tasks.flip.outputs.parameters.flip-output}}' - dependencies: - - flip - - graph-flip-component-1 - name: print-2 - template: print-2 - name: pipeline-flip-coin - - container: - command: - - echo - - '{{inputs.parameters.flip-output}}' - image: alpine:3.9 - inputs: - parameters: - - name: flip-output - name: print - - container: - command: - - echo - - cool, it is over. {{inputs.parameters.flip-output}} - image: alpine:3.9 - inputs: - parameters: - - name: flip-output - name: print-2 diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.py b/sdk/python/tests/compiler/testdata/resourceop_basic.py deleted file mode 100644 index a43da2dc615..00000000000 --- a/sdk/python/tests/compiler/testdata/resourceop_basic.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Note that this sample is just to show the ResourceOp's usage. - -It is not a good practice to put password as a pipeline argument, since -it will be visible on KFP UI. -""" - -import kfp.deprecated.dsl as dsl -from kubernetes import client as k8s_client - - -@dsl.pipeline( - name="ResourceOp Basic", description="A Basic Example on ResourceOp Usage.") -def resourceop_basic(username, password): - secret_resource = k8s_client.V1Secret( - api_version="v1", - kind="Secret", - metadata=k8s_client.V1ObjectMeta(generate_name="my-secret-"), - type="Opaque", - data={ - "username": username, - "password": password - }) - rop = dsl.ResourceOp( - name="create-my-secret", - k8s_resource=secret_resource, - attribute_outputs={"name": "{.metadata.name}"}) - - secret = k8s_client.V1Volume( - name="my-secret", - secret=k8s_client.V1SecretVolumeSource(secret_name=rop.output)) - - cop = dsl.ContainerOp( - name="cop", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["ls /etc/secret-volume"], - pvolumes={"/etc/secret-volume": secret}) - - -if __name__ == "__main__": - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(resourceop_basic, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml b/sdk/python/tests/compiler/testdata/resourceop_basic.yaml deleted file mode 100644 index 666606a62ab..00000000000 --- a/sdk/python/tests/compiler/testdata/resourceop_basic.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "A Basic Example on ResourceOp - Usage.", "inputs": [{"name": "username"}, {"name": "password"}], "name": "ResourceOp Basic"}' - generateName: resourceop-basic- -spec: - arguments: - parameters: - - name: username - - name: password - entrypoint: resourceop-basic - serviceAccountName: pipeline-runner - templates: - - container: - args: - - ls /etc/secret-volume - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /etc/secret-volume - name: my-secret - inputs: - parameters: - - name: create-my-secret-name - name: cop - volumes: - - name: my-secret - secret: - secretName: '{{inputs.parameters.create-my-secret-name}}' - - inputs: - parameters: - - name: password - - name: username - name: create-my-secret - outputs: - parameters: - - name: create-my-secret-manifest - valueFrom: - jsonPath: '{}' - - name: create-my-secret-name - valueFrom: - jsonPath: '{.metadata.name}' - resource: - action: create - manifest: "apiVersion: v1\ndata:\n password: '{{inputs.parameters.password}}'\n\ - \ username: '{{inputs.parameters.username}}'\nkind: Secret\nmetadata:\n \ - \ generateName: my-secret-\ntype: Opaque\n" - - dag: - tasks: - - arguments: - parameters: - - name: create-my-secret-name - value: '{{tasks.create-my-secret.outputs.parameters.create-my-secret-name}}' - dependencies: - - create-my-secret - name: cop - template: cop - - arguments: - parameters: - - name: password - value: '{{inputs.parameters.password}}' - - name: username - value: '{{inputs.parameters.username}}' - name: create-my-secret - template: create-my-secret - inputs: - parameters: - - name: password - - name: username - name: resourceop-basic diff --git a/sdk/python/tests/compiler/testdata/sidecar.py b/sdk/python/tests/compiler/testdata/sidecar.py deleted file mode 100644 index 1b38c04df33..00000000000 --- a/sdk/python/tests/compiler/testdata/sidecar.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline(name='Sidecar', description='A pipeline with sidecars.') -def sidecar_pipeline(): - - echo = dsl.Sidecar( - name='echo', image='hashicorp/http-echo', args=['-text="hello world"']) - - op1 = dsl.ContainerOp( - name='download', - image='busybox', - command=['sh', '-c'], - arguments=['sleep 10; wget localhost:5678 -O /tmp/results.txt'], - sidecars=[echo], - file_outputs={'downloaded': '/tmp/results.txt'}) - - op2 = dsl.ContainerOp( - name='echo', - image='library/bash', - command=['sh', '-c'], - arguments=['echo %s' % op1.output]) diff --git a/sdk/python/tests/compiler/testdata/sidecar.yaml b/sdk/python/tests/compiler/testdata/sidecar.yaml deleted file mode 100644 index 0be3f326404..00000000000 --- a/sdk/python/tests/compiler/testdata/sidecar.yaml +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with sidecars.", - "name": "Sidecar"}' - generateName: sidecar- -apiVersion: argoproj.io/v1alpha1 -spec: - arguments: - parameters: [] - templates: - - outputs: - artifacts: - - name: download-downloaded - path: "/tmp/results.txt" - parameters: - - name: download-downloaded - valueFrom: - path: "/tmp/results.txt" - name: download - sidecars: - - image: hashicorp/http-echo - name: echo - args: - - -text="hello world" - container: - image: busybox - args: - - sleep 10; wget localhost:5678 -O /tmp/results.txt - command: - - sh - - "-c" - - name: echo - inputs: - parameters: - - name: download-downloaded - container: - image: library/bash - args: - - echo {{inputs.parameters.download-downloaded}} - command: - - sh - - "-c" - - name: sidecar - dag: - tasks: - - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: "{{tasks.download.outputs.parameters.download-downloaded}}" - name: echo - dependencies: - - download - template: echo - serviceAccountName: pipeline-runner - entrypoint: sidecar diff --git a/sdk/python/tests/compiler/testdata/test_data/consume_2.component.yaml b/sdk/python/tests/compiler/testdata/test_data/consume_2.component.yaml deleted file mode 100644 index f550e0894d9..00000000000 --- a/sdk/python/tests/compiler/testdata/test_data/consume_2.component.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: Consumer -inputs: -- {name: Input parameter} -- {name: Input artifact} -implementation: - container: - image: alpine - command: - - sh - - -c - - | - echo "Input parameter = $0" - echo "Input artifact = " && cat "$1" - args: - - {inputValue: Input parameter} - - {inputPath: Input artifact} diff --git a/sdk/python/tests/compiler/testdata/test_data/process_2_2.component.yaml b/sdk/python/tests/compiler/testdata/test_data/process_2_2.component.yaml deleted file mode 100644 index 66748561932..00000000000 --- a/sdk/python/tests/compiler/testdata/test_data/process_2_2.component.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: Processor -inputs: -- {name: Input parameter} -- {name: Input artifact} -outputs: -- {name: Output 1} -- {name: Output 2} -implementation: - container: - image: alpine - command: - - sh - - -c - - | - mkdir -p "$(dirname "$2")" - mkdir -p "$(dirname "$3")" - echo "$0" > "$2" - cp "$1" "$3" - args: - - {inputValue: Input parameter} - - {inputPath: Input artifact} - - {outputPath: Output 1} - - {outputPath: Output 2} diff --git a/sdk/python/tests/compiler/testdata/test_data/produce_2.component.yaml b/sdk/python/tests/compiler/testdata/test_data/produce_2.component.yaml deleted file mode 100644 index 4153610f55b..00000000000 --- a/sdk/python/tests/compiler/testdata/test_data/produce_2.component.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: Producer -outputs: -- {name: Output 1} -- {name: Output 2} -implementation: - container: - image: alpine - command: - - sh - - -c - - | - mkdir -p "$(dirname "$0")" - mkdir -p "$(dirname "$1")" - echo "Data 1" > $0 - echo "Data 2" > $1 - args: - - {outputPath: Output 1} - - {outputPath: Output 2} diff --git a/sdk/python/tests/compiler/testdata/testpackage/mypipeline/__init__.py b/sdk/python/tests/compiler/testdata/testpackage/mypipeline/__init__.py deleted file mode 100644 index 54602474f47..00000000000 --- a/sdk/python/tests/compiler/testdata/testpackage/mypipeline/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from .compose import * diff --git a/sdk/python/tests/compiler/testdata/testpackage/mypipeline/compose.py b/sdk/python/tests/compiler/testdata/testpackage/mypipeline/compose.py deleted file mode 100644 index 44049e4b2e8..00000000000 --- a/sdk/python/tests/compiler/testdata/testpackage/mypipeline/compose.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -class GetFrequentWordOp(dsl.ContainerOp): - """A get frequent word class representing a component in ML Pipelines. - - The class provides a nice interface to users by hiding details such - as container, command, arguments. - """ - - def __init__(self, name, message): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - message: a dsl.PipelineParam object representing an input message. - """ - super(GetFrequentWordOp, self).__init__( - name=name, - image='python:3.5-jessie', - command=['sh', '-c'], - arguments=[ - 'python -c "from collections import Counter; ' - 'words = Counter(\'%s\'.split()); print(max(words, key=words.get))" ' - '| tee /tmp/message.txt' % message - ], - file_outputs={'word': '/tmp/message.txt'}) - - -class SaveMessageOp(dsl.ContainerOp): - """A class representing a component in ML Pipelines. - - It saves a message to a given output_path. - """ - - def __init__(self, name, message, output_path): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - message: a dsl.PipelineParam object representing the message to be saved. - output_path: a dsl.PipelineParam object representing the GCS path for output file. - """ - super(SaveMessageOp, self).__init__( - name=name, - image='google/cloud-sdk', - command=['sh', '-c'], - arguments=[ - 'echo %s | tee /tmp/results.txt | gsutil cp /tmp/results.txt %s' - % (message, output_path) - ]) - - -@dsl.pipeline( - name='Save Most Frequent', - description='Get Most Frequent Word and Save to GCS') -def save_most_frequent_word(message: dsl.PipelineParam, - outputpath: dsl.PipelineParam): - """A pipeline function describing the orchestration of the workflow.""" - - counter = GetFrequentWordOp(name='get-Frequent', message=message) - - saver = SaveMessageOp( - name='save', message=counter.output, output_path=outputpath) - - -class DownloadMessageOp(dsl.ContainerOp): - """A class representing a component in ML Pipelines. - - It downloads a message and outputs it. - """ - - def __init__(self, name, url): - """Args: - name: An identifier of the step which needs to be unique within a pipeline. - usl: the gcs url to download the message from. - """ - super(DownloadMessageOp, self).__init__( - name=name, - image='google/cloud-sdk', - command=['sh', '-c'], - arguments=['gsutil cat %s | tee /tmp/results.txt' % url], - file_outputs={'downloaded': '/tmp/results.txt'}) - - -@dsl.pipeline( - name='Download and Save Most Frequent', - description='Download and Get Most Frequent Word and Save to GCS') -def download_save_most_frequent_word(url: str, outputpath: str): - downloader = DownloadMessageOp('download', url) - save_most_frequent_word(downloader.output, outputpath) diff --git a/sdk/python/tests/compiler/testdata/testpackage/mypipeline/kaniko.basic.yaml b/sdk/python/tests/compiler/testdata/testpackage/mypipeline/kaniko.basic.yaml deleted file mode 100644 index 73a79a1f449..00000000000 --- a/sdk/python/tests/compiler/testdata/testpackage/mypipeline/kaniko.basic.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -apiVersion: v1 -kind: Pod -metadata: - generateName: kaniko- - namespace: default -spec: - restartPolicy: Never - serviceAccountName: default - containers: - - name: kaniko - image: gcr.io/kaniko-project/executor:v0.5.0 - args: ["--cache=true", - "--dockerfile=dockerfile", - "--context=gs://mlpipeline/kaniko_build.tar.gz", - "--destination=gcr.io/mlpipeline/kaniko_image:latest"] \ No newline at end of file diff --git a/sdk/python/tests/compiler/testdata/testpackage/setup.py b/sdk/python/tests/compiler/testdata/testpackage/setup.py deleted file mode 100644 index d4bf51c848f..00000000000 --- a/sdk/python/tests/compiler/testdata/testpackage/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import setup - -version = '0.1' - -setup( - name='testsample', - version=version, - packages=['mypipeline'], -) diff --git a/sdk/python/tests/compiler/testdata/timeout.py b/sdk/python/tests/compiler/testdata/timeout.py deleted file mode 100755 index 574f24b3eb0..00000000000 --- a/sdk/python/tests/compiler/testdata/timeout.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -class RandomFailure1Op(dsl.ContainerOp): - """A component that fails randomly.""" - - def __init__(self, exit_codes): - super(RandomFailure1Op, self).__init__( - name='random_failure', - image='python:alpine3.9', - command=['python', '-c'], - arguments=[ - "import random; import sys; exit_code = random.choice([%s]); print(exit_code); sys.exit(exit_code)" - % exit_codes - ]) - - -@dsl.pipeline( - name='pipeline includes two steps which fail randomly.', - description='shows how to use ContainerOp set_retry().') -def retry_sample_pipeline(): - op1 = RandomFailure1Op('0,1,2,3').set_timeout(10) - op2 = RandomFailure1Op('0,1') - dsl.get_pipeline_conf().set_timeout(50) - - -if __name__ == '__main__': - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(retry_sample_pipeline, __file__ + '.tar.gz') diff --git a/sdk/python/tests/compiler/testdata/timeout.yaml b/sdk/python/tests/compiler/testdata/timeout.yaml deleted file mode 100644 index 8bf286a8104..00000000000 --- a/sdk/python/tests/compiler/testdata/timeout.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "shows how to use ContainerOp - set_retry().", "name": "pipeline includes two steps which fail - randomly."}' - generateName: pipeline-includes-two-steps-which-fail-randomly- -spec: - activeDeadlineSeconds: 50 - arguments: - parameters: [] - entrypoint: pipeline-includes-two-steps-which-fail-randomly - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - name: random-failure - template: random-failure - - name: random-failure-2 - template: random-failure-2 - name: pipeline-includes-two-steps-which-fail-randomly - - activeDeadlineSeconds: 10 - container: - args: - - import random; import sys; exit_code = random.choice([0,1,2,3]); print(exit_code); - sys.exit(exit_code) - command: - - python - - -c - image: python:alpine3.9 - name: random-failure - - container: - args: - - import random; import sys; exit_code = random.choice([0,1]); print(exit_code); - sys.exit(exit_code) - command: - - python - - -c - image: python:alpine3.9 - name: random-failure-2 diff --git a/sdk/python/tests/compiler/testdata/tolerations.yaml b/sdk/python/tests/compiler/testdata/tolerations.yaml deleted file mode 100644 index bc7f60c7ca5..00000000000 --- a/sdk/python/tests/compiler/testdata/tolerations.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -kind: Workflow -metadata: - generateName: tolerations- -apiVersion: argoproj.io/v1alpha1 -spec: - arguments: - parameters: [] - templates: - - outputs: - artifacts: - - name: download-downloaded - path: "/tmp/results.txt" - parameters: - - name: download-downloaded - valueFrom: - path: "/tmp/results.txt" - name: download - container: - image: busybox - args: - - sleep 10; wget localhost:5678 -O /tmp/results.txt - command: - - sh - - "-c" - tolerations: - - effect: NoSchedule - key: gpu - operator: Equal - value: run - serviceAccountName: pipeline-runner diff --git a/sdk/python/tests/compiler/testdata/two_step.py b/sdk/python/tests/compiler/testdata/two_step.py deleted file mode 100644 index 4558aebdccb..00000000000 --- a/sdk/python/tests/compiler/testdata/two_step.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Two step v2-compatible pipeline.""" - -from kfp.deprecated import components -from kfp.deprecated import dsl -from kfp.deprecated.components import InputPath -from kfp.deprecated.components import OutputPath - - -def preprocess(uri: str, some_int: int, output_parameter_one: OutputPath(int), - output_dataset_one: OutputPath('Dataset')): - """Dummy Preprocess Step.""" - with open(output_dataset_one, 'w') as f: - f.write('Output dataset') - with open(output_parameter_one, 'w') as f: - f.write("{}".format(1234)) - - -preprocess_op = components.create_component_from_func( - preprocess, base_image='python:3.9') - - -@components.create_component_from_func -def train_op(dataset: InputPath('Dataset'), - model: OutputPath('Model'), - num_steps: int = 100): - """Dummy Training Step.""" - - with open(dataset, 'r') as input_file: - input_string = input_file.read() - with open(model, 'w') as output_file: - for i in range(num_steps): - output_file.write("Step {}\n{}\n=====\n".format( - i, input_string)) - - -@dsl.pipeline(name='two_step_pipeline') -def two_step_pipeline(): - preprocess_task = preprocess_op(uri='uri-to-import', some_int=12) - train_task = train_op( - num_steps=preprocess_task.outputs['output_parameter_one'], - dataset=preprocess_task.outputs['output_dataset_one']) diff --git a/sdk/python/tests/compiler/testdata/uri_artifacts.py b/sdk/python/tests/compiler/testdata/uri_artifacts.py deleted file mode 100644 index 43c529bbca4..00000000000 --- a/sdk/python/tests/compiler/testdata/uri_artifacts.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Pipeline DSL code for testing URI-based artifact passing.""" -from kfp.deprecated import compiler -from kfp.deprecated import components -from kfp.deprecated import dsl - - -# Patch to make the test result deterministic. -class Coder: - - def __init__(self,): - self._code_id = 0 - - def get_code(self,): - self._code_id += 1 - return '{code:0{num_chars:}d}'.format( - code=self._code_id, - num_chars=dsl._for_loop.LoopArguments.NUM_CODE_CHARS) - - -dsl.ParallelFor._get_unique_id_code = Coder().get_code - -write_to_gcs = components.load_component_from_text(""" -name: Write to GCS -inputs: -- {name: text, type: String, description: 'Content to be written to GCS'} -outputs: -- {name: output_gcs_path, type: Artifact, description: 'GCS file path'} -implementation: - container: - image: google/cloud-sdk:slim - command: - - sh - - -c - - | - set -e -x - echo "$0" | gsutil cp - "$1" - - {inputValue: text} - - {outputUri: output_gcs_path} -""") - -read_from_gcs = components.load_component_from_text(""" -name: Read from GCS -inputs: -- {name: input_gcs_path, type: Artifact, description: 'GCS file path'} -implementation: - container: - image: google/cloud-sdk:slim - command: - - sh - - -c - - | - set -e -x - gsutil cat "$0" - - {inputUri: input_gcs_path} -""") - -flip_coin_op = components.load_component_from_text(""" -name: Flip coin -outputs: -- {name: output, type: String} -implementation: - container: - image: python:alpine3.9 - command: - - sh - - -c - args: - - mkdir -p "$(dirname $0)" && python -c "import random; result = \'heads\' if random.randint(0,1) == 0 else \'tails\'; print(result, end='')" | tee $0 - - {outputPath: output} -""") - - -@dsl.pipeline( - name='uri-artifact-pipeline', pipeline_root='gs://my-bucket/my-output-dir') -def uri_artifact(text: str = 'Hello world!'): - task_1 = write_to_gcs(text=text) - task_2 = read_from_gcs(input_gcs_path=task_1.outputs['output_gcs_path']) - - # Test use URI within ParFor loop. - loop_args = [1, 2, 3, 4] - with dsl.ParallelFor(loop_args) as loop_arg: - loop_task_2 = read_from_gcs( - input_gcs_path=task_1.outputs['output_gcs_path']) - - # Test use URI within condition. - flip = flip_coin_op() - with dsl.Condition(flip.output == 'heads'): - condition_task_2 = read_from_gcs( - input_gcs_path=task_1.outputs['output_gcs_path']) - - -if __name__ == '__main__': - compiler.Compiler(mode=dsl.PipelineExecutionMode.V2_COMPATIBLE).compile( - uri_artifact, __file__.replace('.py', '.yaml')) diff --git a/sdk/python/tests/compiler/testdata/uri_artifacts.yaml b/sdk/python/tests/compiler/testdata/uri_artifacts.yaml deleted file mode 100644 index 60192fcc6ff..00000000000 --- a/sdk/python/tests/compiler/testdata/uri_artifacts.yaml +++ /dev/null @@ -1,447 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: uri-artifact-pipeline- - annotations: - pipelines.kubeflow.org/kfp_sdk_version: 1.6.6 - pipelines.kubeflow.org/pipeline_compilation_time: '2021-08-07T00:27:52.329848' - pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "Hello world!", - "name": "text", "optional": true, "type": "String"}, {"default": "gs://my-bucket/my-output-dir", - "name": "pipeline-root"}, {"default": "pipeline/uri-artifact-pipeline", "name": - "pipeline-name"}], "name": "uri-artifact-pipeline"}' - pipelines.kubeflow.org/v2_pipeline: "true" - labels: - pipelines.kubeflow.org/v2_pipeline: "true" - pipelines.kubeflow.org/kfp_sdk_version: 1.6.6 -spec: - entrypoint: uri-artifact-pipeline - templates: - - name: condition-3 - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - artifacts: - - {name: write-to-gcs-output_gcs_path} - dag: - tasks: - - name: read-from-gcs-3 - template: read-from-gcs-3 - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - artifacts: - - {name: write-to-gcs-output_gcs_path, from: '{{inputs.artifacts.write-to-gcs-output_gcs_path}}'} - - name: flip-coin - container: - args: [sh, -c, 'mkdir -p "$(dirname $0)" && python -c "import random; result - = ''heads'' if random.randint(0,1) == 0 else ''tails''; print(result, end='''')" - | tee $0', '{{$.outputs.parameters[''output''].output_file}}'] - command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST), - --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO), - --container_image, $(KFP_V2_IMAGE), --task_name, flip-coin, --pipeline_name, - '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource, - workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name, - $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}', - --enable_caching, $(ENABLE_CACHING), --, --] - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: {fieldPath: metadata.name} - - name: KFP_POD_UID - valueFrom: - fieldRef: {fieldPath: metadata.uid} - - name: KFP_NAMESPACE - valueFrom: - fieldRef: {fieldPath: metadata.namespace} - - name: WORKFLOW_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'} - - name: KFP_RUN_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'} - - name: ENABLE_CACHING - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'python:alpine3.6'} - - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts": - {}, "outputParameters": {"output": {"type": "STRING", "path": "/tmp/outputs/output/data"}}, - "outputArtifacts": {}}'} - envFrom: - - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: python:alpine3.9 - volumeMounts: - - {mountPath: /kfp-launcher, name: kfp-launcher} - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - outputs: - parameters: - - name: flip-coin-output - valueFrom: {path: /tmp/outputs/output/data} - artifacts: - - {name: flip-coin-output, path: /tmp/outputs/output/data} - metadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/component_ref: '{"digest": "ea0f712c589f5eefb51b9ca8a7c98fdcaa44796168214a6338ac8210fb816f15"}' - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.6.6 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/enable_caching: "true" - initContainers: - - command: [launcher, --copy, /kfp-launcher/launch] - image: gcr.io/ml-pipeline/kfp-launcher:1.6.6 - name: kfp-launcher - mirrorVolumeMounts: true - volumes: - - {name: kfp-launcher} - - name: for-loop-2 - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - artifacts: - - {name: write-to-gcs-output_gcs_path} - dag: - tasks: - - name: read-from-gcs-2 - template: read-from-gcs-2 - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - artifacts: - - {name: write-to-gcs-output_gcs_path, from: '{{inputs.artifacts.write-to-gcs-output_gcs_path}}'} - - name: read-from-gcs - container: - args: - - sh - - -c - - | - set -e -x - gsutil cat "$0" - - '{{$.inputs.artifacts[''input_gcs_path''].uri}}' - command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST), - --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO), - --container_image, $(KFP_V2_IMAGE), --task_name, read-from-gcs, --pipeline_name, - '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource, - workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name, - $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}', - --enable_caching, $(ENABLE_CACHING), --, --] - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: {fieldPath: metadata.name} - - name: KFP_POD_UID - valueFrom: - fieldRef: {fieldPath: metadata.uid} - - name: KFP_NAMESPACE - valueFrom: - fieldRef: {fieldPath: metadata.namespace} - - name: WORKFLOW_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'} - - name: KFP_RUN_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'} - - name: ENABLE_CACHING - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'google/cloud-sdk:slim'} - - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts": - {"input_gcs_path": {"metadataPath": "/tmp/inputs/input_gcs_path/data", "schemaTitle": - "system.Artifact", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters": - {}, "outputArtifacts": {}}'} - envFrom: - - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: google/cloud-sdk:slim - volumeMounts: - - {mountPath: /kfp-launcher, name: kfp-launcher} - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - artifacts: - - {name: write-to-gcs-output_gcs_path, path: /tmp/inputs/input_gcs_path/data} - metadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/component_ref: '{"digest": "d6b3accc859ff354de84b11d6bd74abc44c893b399807b502dab02ca6872befe"}' - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.6.6 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/enable_caching: "true" - initContainers: - - command: [launcher, --copy, /kfp-launcher/launch] - image: gcr.io/ml-pipeline/kfp-launcher:1.6.6 - name: kfp-launcher - mirrorVolumeMounts: true - volumes: - - {name: kfp-launcher} - - name: read-from-gcs-2 - container: - args: - - sh - - -c - - | - set -e -x - gsutil cat "$0" - - '{{$.inputs.artifacts[''input_gcs_path''].uri}}' - command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST), - --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO), - --container_image, $(KFP_V2_IMAGE), --task_name, read-from-gcs-2, --pipeline_name, - '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource, - workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name, - $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}', - --enable_caching, $(ENABLE_CACHING), --, --] - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: {fieldPath: metadata.name} - - name: KFP_POD_UID - valueFrom: - fieldRef: {fieldPath: metadata.uid} - - name: KFP_NAMESPACE - valueFrom: - fieldRef: {fieldPath: metadata.namespace} - - name: WORKFLOW_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'} - - name: KFP_RUN_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'} - - name: ENABLE_CACHING - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'google/cloud-sdk:slim'} - - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts": - {"input_gcs_path": {"metadataPath": "/tmp/inputs/input_gcs_path/data", "schemaTitle": - "system.Artifact", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters": - {}, "outputArtifacts": {}}'} - envFrom: - - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: google/cloud-sdk:slim - volumeMounts: - - {mountPath: /kfp-launcher, name: kfp-launcher} - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - artifacts: - - {name: write-to-gcs-output_gcs_path, path: /tmp/inputs/input_gcs_path/data} - metadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/component_ref: '{"digest": "d6b3accc859ff354de84b11d6bd74abc44c893b399807b502dab02ca6872befe"}' - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.6.6 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/enable_caching: "true" - initContainers: - - command: [launcher, --copy, /kfp-launcher/launch] - image: gcr.io/ml-pipeline/kfp-launcher:1.6.6 - name: kfp-launcher - mirrorVolumeMounts: true - volumes: - - {name: kfp-launcher} - - name: read-from-gcs-3 - container: - args: - - sh - - -c - - | - set -e -x - gsutil cat "$0" - - '{{$.inputs.artifacts[''input_gcs_path''].uri}}' - command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST), - --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO), - --container_image, $(KFP_V2_IMAGE), --task_name, read-from-gcs-3, --pipeline_name, - '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource, - workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name, - $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}', - --enable_caching, $(ENABLE_CACHING), --, --] - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: {fieldPath: metadata.name} - - name: KFP_POD_UID - valueFrom: - fieldRef: {fieldPath: metadata.uid} - - name: KFP_NAMESPACE - valueFrom: - fieldRef: {fieldPath: metadata.namespace} - - name: WORKFLOW_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'} - - name: KFP_RUN_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'} - - name: ENABLE_CACHING - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'google/cloud-sdk:slim'} - - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {}, "inputArtifacts": - {"input_gcs_path": {"metadataPath": "/tmp/inputs/input_gcs_path/data", "schemaTitle": - "system.Artifact", "instanceSchema": "", "schemaVersion": "0.0.1"}}, "outputParameters": - {}, "outputArtifacts": {}}'} - envFrom: - - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: google/cloud-sdk:slim - volumeMounts: - - {mountPath: /kfp-launcher, name: kfp-launcher} - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - artifacts: - - {name: write-to-gcs-output_gcs_path, path: /tmp/inputs/input_gcs_path/data} - metadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/component_ref: '{"digest": "d6b3accc859ff354de84b11d6bd74abc44c893b399807b502dab02ca6872befe"}' - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.6.6 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/enable_caching: "true" - initContainers: - - command: [launcher, --copy, /kfp-launcher/launch] - image: gcr.io/ml-pipeline/kfp-launcher:1.6.6 - name: kfp-launcher - mirrorVolumeMounts: true - volumes: - - {name: kfp-launcher} - - name: uri-artifact-pipeline - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - - {name: text} - dag: - tasks: - - name: condition-3 - template: condition-3 - when: '"{{tasks.flip-coin.outputs.parameters.flip-coin-output}}" == "heads"' - dependencies: [flip-coin, write-to-gcs] - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - artifacts: - - {name: write-to-gcs-output_gcs_path, from: '{{tasks.write-to-gcs.outputs.artifacts.write-to-gcs-output_gcs_path}}'} - - name: flip-coin - template: flip-coin - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - - name: for-loop-2 - template: for-loop-2 - dependencies: [write-to-gcs] - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - artifacts: - - {name: write-to-gcs-output_gcs_path, from: '{{tasks.write-to-gcs.outputs.artifacts.write-to-gcs-output_gcs_path}}'} - withItems: [1, 2, 3, 4] - - name: read-from-gcs - template: read-from-gcs - dependencies: [write-to-gcs] - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - artifacts: - - {name: write-to-gcs-output_gcs_path, from: '{{tasks.write-to-gcs.outputs.artifacts.write-to-gcs-output_gcs_path}}'} - - name: write-to-gcs - template: write-to-gcs - arguments: - parameters: - - {name: pipeline-name, value: '{{inputs.parameters.pipeline-name}}'} - - {name: pipeline-root, value: '{{inputs.parameters.pipeline-root}}'} - - {name: text, value: '{{inputs.parameters.text}}'} - - name: write-to-gcs - container: - args: - - sh - - -c - - | - set -e -x - echo "$0" | gsutil cp - "$1" - - '{{$.inputs.parameters[''text'']}}' - - '{{$.outputs.artifacts[''output_gcs_path''].uri}}' - command: [/kfp-launcher/launch, --mlmd_server_address, $(METADATA_GRPC_SERVICE_HOST), - --mlmd_server_port, $(METADATA_GRPC_SERVICE_PORT), --runtime_info_json, $(KFP_V2_RUNTIME_INFO), - --container_image, $(KFP_V2_IMAGE), --task_name, write-to-gcs, --pipeline_name, - '{{inputs.parameters.pipeline-name}}', --run_id, $(KFP_RUN_ID), --run_resource, - workflows.argoproj.io/$(WORKFLOW_ID), --namespace, $(KFP_NAMESPACE), --pod_name, - $(KFP_POD_NAME), --pod_uid, $(KFP_POD_UID), --pipeline_root, '{{inputs.parameters.pipeline-root}}', - --enable_caching, $(ENABLE_CACHING), --, 'text={{inputs.parameters.text}}', - --] - env: - - name: KFP_POD_NAME - valueFrom: - fieldRef: {fieldPath: metadata.name} - - name: KFP_POD_UID - valueFrom: - fieldRef: {fieldPath: metadata.uid} - - name: KFP_NAMESPACE - valueFrom: - fieldRef: {fieldPath: metadata.namespace} - - name: WORKFLOW_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''workflows.argoproj.io/workflow'']'} - - name: KFP_RUN_ID - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipeline/runid'']'} - - name: ENABLE_CACHING - valueFrom: - fieldRef: {fieldPath: 'metadata.labels[''pipelines.kubeflow.org/enable_caching'']'} - - {name: KFP_V2_IMAGE, value: 'google/cloud-sdk:slim'} - - {name: KFP_V2_RUNTIME_INFO, value: '{"inputParameters": {"text": {"type": - "STRING"}}, "inputArtifacts": {}, "outputParameters": {}, "outputArtifacts": - {"output_gcs_path": {"schemaTitle": "system.Artifact", "instanceSchema": - "", "schemaVersion": "0.0.1", "metadataPath": "/tmp/outputs/output_gcs_path/data"}}}'} - envFrom: - - configMapRef: {name: metadata-grpc-configmap, optional: true} - image: google/cloud-sdk:slim - volumeMounts: - - {mountPath: /kfp-launcher, name: kfp-launcher} - inputs: - parameters: - - {name: pipeline-name} - - {name: pipeline-root} - - {name: text} - outputs: - artifacts: - - {name: write-to-gcs-output_gcs_path, path: /tmp/outputs/output_gcs_path/data} - metadata: - annotations: - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/component_ref: '{"digest": "c4b517e602f5844888f6d6b7ad543c7dfdcb65d844a802f582e5438e17ea550b"}' - pipelines.kubeflow.org/arguments.parameters: '{"text": "{{inputs.parameters.text}}"}' - labels: - pipelines.kubeflow.org/kfp_sdk_version: 1.6.6 - pipelines.kubeflow.org/pipeline-sdk-type: kfp - pipelines.kubeflow.org/v2_component: "true" - pipelines.kubeflow.org/enable_caching: "true" - initContainers: - - command: [launcher, --copy, /kfp-launcher/launch] - image: gcr.io/ml-pipeline/kfp-launcher:1.6.6 - name: kfp-launcher - mirrorVolumeMounts: true - volumes: - - {name: kfp-launcher} - arguments: - parameters: - - {name: text, value: Hello world!} - - {name: pipeline-root, value: 'gs://my-bucket/my-output-dir'} - - {name: pipeline-name, value: pipeline/uri-artifact-pipeline} - serviceAccountName: pipeline-runner diff --git a/sdk/python/tests/compiler/testdata/volume.py b/sdk/python/tests/compiler/testdata/volume.py deleted file mode 100644 index 948a45004c9..00000000000 --- a/sdk/python/tests/compiler/testdata/volume.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl -from kubernetes import client as k8s_client - - -@dsl.pipeline(name='Volume', description='A pipeline with volume.') -def volume_pipeline(): - op1 = dsl.ContainerOp( - name='download', - image='google/cloud-sdk', - command=['sh', '-c'], - arguments=['ls | tee /tmp/results.txt'], - file_outputs={'downloaded': '/tmp/results.txt'}) \ - .add_volume(k8s_client.V1Volume(name='gcp-credentials', - secret=k8s_client.V1SecretVolumeSource( - secret_name='user-gcp-sa'))) \ - .add_volume_mount(k8s_client.V1VolumeMount( - mount_path='/secret/gcp-credentials', name='gcp-credentials')) \ - .add_env_variable(k8s_client.V1EnvVar( - name='GOOGLE_APPLICATION_CREDENTIALS', - value='/secret/gcp-credentials/user-gcp-sa.json')) \ - .add_env_variable(k8s_client.V1EnvVar(name='Foo', value='bar')) - op2 = dsl.ContainerOp( - name='echo', - image='library/bash', - command=['sh', '-c'], - arguments=['echo %s' % op1.output]) diff --git a/sdk/python/tests/compiler/testdata/volume.yaml b/sdk/python/tests/compiler/testdata/volume.yaml deleted file mode 100644 index 484f7a23c64..00000000000 --- a/sdk/python/tests/compiler/testdata/volume.yaml +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline with volume.", - "name": "Volume"}' - generateName: volume- -spec: - arguments: - parameters: [] - entrypoint: volume - serviceAccountName: pipeline-runner - templates: - - container: - args: - - ls | tee /tmp/results.txt - command: - - sh - - -c - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /secret/gcp-credentials/user-gcp-sa.json - - name: Foo - value: bar - image: google/cloud-sdk - volumeMounts: - - mountPath: /secret/gcp-credentials - name: gcp-credentials - name: download - outputs: - artifacts: - - name: download-downloaded - path: /tmp/results.txt - parameters: - - name: download-downloaded - valueFrom: - path: /tmp/results.txt - volumes: - - name: gcp-credentials - secret: - secretName: user-gcp-sa - - container: - args: - - echo {{inputs.parameters.download-downloaded}} - command: - - sh - - -c - image: library/bash - inputs: - parameters: - - name: download-downloaded - name: echo - - dag: - tasks: - - name: download - template: download - - arguments: - parameters: - - name: download-downloaded - value: '{{tasks.download.outputs.parameters.download-downloaded}}' - dependencies: - - download - name: echo - template: echo - name: volume diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py deleted file mode 100644 index 4ba7d60ebc3..00000000000 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""This sample uses Rok as an example to show case how VolumeOp accepts -annotations as an extra argument, and how we can use arbitrary PipelineParams -to determine their contents. - -The specific annotation is Rok-specific, but the use of annotations in -such way is widespread in storage systems integrated with K8s. -""" - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline( - name="VolumeSnapshotOp RokURL", - description="The fifth example of the design doc.") -def volume_snapshotop_rokurl(rok_url): - vop1 = dsl.VolumeOp( - name="create_volume_1", - resource_name="vol1", - size="1Gi", - annotations={"rok/origin": rok_url}, - modes=dsl.VOLUME_MODE_RWM) - - step1 = dsl.ContainerOp( - name="step1_concat", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["cat /data/file*| gzip -c >/data/full.gz"], - pvolumes={"/data": vop1.volume}) - - step1_snap = dsl.VolumeSnapshotOp( - name="create_snapshot_1", resource_name="snap1", volume=step1.pvolume) - - vop2 = dsl.VolumeOp( - name="create_volume_2", - resource_name="vol2", - data_source=step1_snap.snapshot, - size=step1_snap.outputs["size"]) - - step2 = dsl.ContainerOp( - name="step2_gunzip", - image="library/bash:4.4.23", - command=["gunzip", "-k", "/data/full.gz"], - pvolumes={"/data": vop2.volume}) - - step2_snap = dsl.VolumeSnapshotOp( - name="create_snapshot_2", resource_name="snap2", volume=step2.pvolume) - - vop3 = dsl.VolumeOp( - name="create_volume_3", - resource_name="vol3", - data_source=step2_snap.snapshot, - size=step2_snap.outputs["size"]) - - step3 = dsl.ContainerOp( - name="step3_output", - image="library/bash:4.4.23", - command=["cat", "/data/full"], - pvolumes={"/data": vop3.volume}) - - -if __name__ == "__main__": - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(volume_snapshotop_rokurl, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml deleted file mode 100644 index 2d9e6570671..00000000000 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_rokurl.yaml +++ /dev/null @@ -1,246 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "The fifth example of the - design doc.", "inputs": [{"name": "rok_url"}], "name": "VolumeSnapshotOp RokURL"}' - generateName: volumesnapshotop-rokurl- -spec: - arguments: - parameters: - - name: rok_url - entrypoint: volumesnapshotop-rokurl - serviceAccountName: pipeline-runner - templates: - - inputs: - parameters: - - name: create-volume-1-name - name: create-snapshot-1 - outputs: - parameters: - - name: create-snapshot-1-manifest - valueFrom: - jsonPath: '{}' - - name: create-snapshot-1-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: create-snapshot-1-size - valueFrom: - jsonPath: '{.status.restoreSize}' - resource: - action: create - manifest: "apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot\n\ - metadata:\n name: '{{workflow.name}}-snap1'\nspec:\n source:\n kind:\ - \ PersistentVolumeClaim\n name: '{{inputs.parameters.create-volume-1-name}}'\n" - successCondition: status.readyToUse == true - - inputs: - parameters: - - name: create-volume-2-name - name: create-snapshot-2 - outputs: - parameters: - - name: create-snapshot-2-manifest - valueFrom: - jsonPath: '{}' - - name: create-snapshot-2-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: create-snapshot-2-size - valueFrom: - jsonPath: '{.status.restoreSize}' - resource: - action: create - manifest: "apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot\n\ - metadata:\n name: '{{workflow.name}}-snap2'\nspec:\n source:\n kind:\ - \ PersistentVolumeClaim\n name: '{{inputs.parameters.create-volume-2-name}}'\n" - successCondition: status.readyToUse == true - - inputs: - parameters: - - name: rok_url - name: create-volume-1 - outputs: - parameters: - - name: create-volume-1-manifest - valueFrom: - jsonPath: '{}' - - name: create-volume-1-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: create-volume-1-size - valueFrom: - jsonPath: '{.status.capacity.storage}' - resource: - action: create - manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n annotations:\n\ - \ rok/origin: '{{inputs.parameters.rok_url}}'\n name: '{{workflow.name}}-vol1'\n\ - spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ - \ storage: 1Gi\n" - - inputs: - parameters: - - name: create-snapshot-1-name - - name: create-snapshot-1-size - name: create-volume-2 - outputs: - parameters: - - name: create-volume-2-manifest - valueFrom: - jsonPath: '{}' - - name: create-volume-2-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: create-volume-2-size - valueFrom: - jsonPath: '{.status.capacity.storage}' - resource: - action: create - manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-vol2'\n\ - spec:\n accessModes:\n - ReadWriteMany\n dataSource:\n apiGroup: snapshot.storage.k8s.io\n\ - \ kind: VolumeSnapshot\n name: '{{inputs.parameters.create-snapshot-1-name}}'\n\ - \ resources:\n requests:\n storage: '{{inputs.parameters.create-snapshot-1-size}}'\n" - - inputs: - parameters: - - name: create-snapshot-2-name - - name: create-snapshot-2-size - name: create-volume-3 - outputs: - parameters: - - name: create-volume-3-manifest - valueFrom: - jsonPath: '{}' - - name: create-volume-3-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: create-volume-3-size - valueFrom: - jsonPath: '{.status.capacity.storage}' - resource: - action: create - manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-vol3'\n\ - spec:\n accessModes:\n - ReadWriteMany\n dataSource:\n apiGroup: snapshot.storage.k8s.io\n\ - \ kind: VolumeSnapshot\n name: '{{inputs.parameters.create-snapshot-2-name}}'\n\ - \ resources:\n requests:\n storage: '{{inputs.parameters.create-snapshot-2-size}}'\n" - - container: - args: - - cat /data/file*| gzip -c >/data/full.gz - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /data - name: create-volume-1 - inputs: - parameters: - - name: create-volume-1-name - name: step1-concat - volumes: - - name: create-volume-1 - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-volume-1-name}}' - - container: - command: - - gunzip - - -k - - /data/full.gz - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /data - name: create-volume-2 - inputs: - parameters: - - name: create-volume-2-name - name: step2-gunzip - volumes: - - name: create-volume-2 - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-volume-2-name}}' - - container: - command: - - cat - - /data/full - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /data - name: create-volume-3 - inputs: - parameters: - - name: create-volume-3-name - name: step3-output - volumes: - - name: create-volume-3 - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-volume-3-name}}' - - dag: - tasks: - - arguments: - parameters: - - name: create-volume-1-name - value: '{{tasks.create-volume-1.outputs.parameters.create-volume-1-name}}' - dependencies: - - create-volume-1 - - step1-concat - name: create-snapshot-1 - template: create-snapshot-1 - - arguments: - parameters: - - name: create-volume-2-name - value: '{{tasks.create-volume-2.outputs.parameters.create-volume-2-name}}' - dependencies: - - create-volume-2 - - step2-gunzip - name: create-snapshot-2 - template: create-snapshot-2 - - arguments: - parameters: - - name: rok_url - value: '{{inputs.parameters.rok_url}}' - name: create-volume-1 - template: create-volume-1 - - arguments: - parameters: - - name: create-snapshot-1-name - value: '{{tasks.create-snapshot-1.outputs.parameters.create-snapshot-1-name}}' - - name: create-snapshot-1-size - value: '{{tasks.create-snapshot-1.outputs.parameters.create-snapshot-1-size}}' - dependencies: - - create-snapshot-1 - name: create-volume-2 - template: create-volume-2 - - arguments: - parameters: - - name: create-snapshot-2-name - value: '{{tasks.create-snapshot-2.outputs.parameters.create-snapshot-2-name}}' - - name: create-snapshot-2-size - value: '{{tasks.create-snapshot-2.outputs.parameters.create-snapshot-2-size}}' - dependencies: - - create-snapshot-2 - name: create-volume-3 - template: create-volume-3 - - arguments: - parameters: - - name: create-volume-1-name - value: '{{tasks.create-volume-1.outputs.parameters.create-volume-1-name}}' - dependencies: - - create-volume-1 - name: step1-concat - template: step1-concat - - arguments: - parameters: - - name: create-volume-2-name - value: '{{tasks.create-volume-2.outputs.parameters.create-volume-2-name}}' - dependencies: - - create-volume-2 - name: step2-gunzip - template: step2-gunzip - - arguments: - parameters: - - name: create-volume-3-name - value: '{{tasks.create-volume-3.outputs.parameters.create-volume-3-name}}' - dependencies: - - create-volume-3 - name: step3-output - template: step3-output - inputs: - parameters: - - name: rok_url - name: volumesnapshotop-rokurl diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py deleted file mode 100644 index 94ea448cb6d..00000000000 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline( - name="VolumeSnapshotOp Sequential", - description="The fourth example of the design doc.") -def volume_snapshotop_sequential(url): - vop = dsl.VolumeOp( - name="create_volume", - resource_name="vol1", - size="1Gi", - modes=dsl.VOLUME_MODE_RWM) - - step1 = dsl.ContainerOp( - name="step1_ingest", - image="google/cloud-sdk:279.0.0", - command=["sh", "-c"], - arguments=[ - "mkdir /data/step1 && " - "gsutil cat %s | gzip -c >/data/step1/file1.gz" % url - ], - pvolumes={"/data": vop.volume}) - - step1_snap = dsl.VolumeSnapshotOp( - name="step1_snap", resource_name="step1_snap", volume=step1.pvolume) - - step2 = dsl.ContainerOp( - name="step2_gunzip", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=[ - "mkdir /data/step2 && " - "gunzip /data/step1/file1.gz -c >/data/step2/file1" - ], - pvolumes={"/data": step1.pvolume}) - - step2_snap = dsl.VolumeSnapshotOp( - name="step2_snap", resource_name="step2_snap", volume=step2.pvolume) - - step3 = dsl.ContainerOp( - name="step3_copy", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=[ - "mkdir /data/step3 && " - "cp -av /data/step2/file1 /data/step3/file3" - ], - pvolumes={"/data": step2.pvolume}) - - step3_snap = dsl.VolumeSnapshotOp( - name="step3_snap", resource_name="step3_snap", volume=step3.pvolume) - - step4 = dsl.ContainerOp( - name="step4_output", - image="library/bash:4.4.23", - command=["cat", "/data/step2/file1", "/data/step3/file3"], - pvolumes={"/data": step3.pvolume}) - - -if __name__ == "__main__": - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(volume_snapshotop_sequential, - __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml b/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml deleted file mode 100644 index a0d9ac8f2ab..00000000000 --- a/sdk/python/tests/compiler/testdata/volume_snapshotop_sequential.yaml +++ /dev/null @@ -1,238 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "The fourth example of - the design doc.", "inputs": [{"name": "url"}], "name": "VolumeSnapshotOp Sequential"}' - generateName: volumesnapshotop-sequential- -spec: - arguments: - parameters: - - name: url - entrypoint: volumesnapshotop-sequential - serviceAccountName: pipeline-runner - templates: - - name: create-volume - outputs: - parameters: - - name: create-volume-manifest - valueFrom: - jsonPath: '{}' - - name: create-volume-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: create-volume-size - valueFrom: - jsonPath: '{.status.capacity.storage}' - resource: - action: create - manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-vol1'\n\ - spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ - \ storage: 1Gi\n" - - container: - args: - - mkdir /data/step1 && gsutil cat {{inputs.parameters.url}} | gzip -c >/data/step1/file1.gz - command: - - sh - - -c - image: google/cloud-sdk:279.0.0 - volumeMounts: - - mountPath: /data - name: create-volume - inputs: - parameters: - - name: create-volume-name - - name: url - name: step1-ingest - volumes: - - name: create-volume - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-volume-name}}' - - inputs: - parameters: - - name: create-volume-name - name: step1-snap - outputs: - parameters: - - name: step1-snap-manifest - valueFrom: - jsonPath: '{}' - - name: step1-snap-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: step1-snap-size - valueFrom: - jsonPath: '{.status.restoreSize}' - resource: - action: create - manifest: "apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot\n\ - metadata:\n name: '{{workflow.name}}-step1-snap'\nspec:\n source:\n kind:\ - \ PersistentVolumeClaim\n name: '{{inputs.parameters.create-volume-name}}'\n" - successCondition: status.readyToUse == true - - container: - args: - - mkdir /data/step2 && gunzip /data/step1/file1.gz -c >/data/step2/file1 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /data - name: create-volume - inputs: - parameters: - - name: create-volume-name - name: step2-gunzip - volumes: - - name: create-volume - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-volume-name}}' - - inputs: - parameters: - - name: create-volume-name - name: step2-snap - outputs: - parameters: - - name: step2-snap-manifest - valueFrom: - jsonPath: '{}' - - name: step2-snap-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: step2-snap-size - valueFrom: - jsonPath: '{.status.restoreSize}' - resource: - action: create - manifest: "apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot\n\ - metadata:\n name: '{{workflow.name}}-step2-snap'\nspec:\n source:\n kind:\ - \ PersistentVolumeClaim\n name: '{{inputs.parameters.create-volume-name}}'\n" - successCondition: status.readyToUse == true - - container: - args: - - mkdir /data/step3 && cp -av /data/step2/file1 /data/step3/file3 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /data - name: create-volume - inputs: - parameters: - - name: create-volume-name - name: step3-copy - volumes: - - name: create-volume - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-volume-name}}' - - inputs: - parameters: - - name: create-volume-name - name: step3-snap - outputs: - parameters: - - name: step3-snap-manifest - valueFrom: - jsonPath: '{}' - - name: step3-snap-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: step3-snap-size - valueFrom: - jsonPath: '{.status.restoreSize}' - resource: - action: create - manifest: "apiVersion: snapshot.storage.k8s.io/v1alpha1\nkind: VolumeSnapshot\n\ - metadata:\n name: '{{workflow.name}}-step3-snap'\nspec:\n source:\n kind:\ - \ PersistentVolumeClaim\n name: '{{inputs.parameters.create-volume-name}}'\n" - successCondition: status.readyToUse == true - - container: - command: - - cat - - /data/step2/file1 - - /data/step3/file3 - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /data - name: create-volume - inputs: - parameters: - - name: create-volume-name - name: step4-output - volumes: - - name: create-volume - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-volume-name}}' - - dag: - tasks: - - name: create-volume - template: create-volume - - arguments: - parameters: - - name: create-volume-name - value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' - - name: url - value: '{{inputs.parameters.url}}' - dependencies: - - create-volume - name: step1-ingest - template: step1-ingest - - arguments: - parameters: - - name: create-volume-name - value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' - dependencies: - - create-volume - - step1-ingest - name: step1-snap - template: step1-snap - - arguments: - parameters: - - name: create-volume-name - value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' - dependencies: - - create-volume - - step1-ingest - name: step2-gunzip - template: step2-gunzip - - arguments: - parameters: - - name: create-volume-name - value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' - dependencies: - - create-volume - - step2-gunzip - name: step2-snap - template: step2-snap - - arguments: - parameters: - - name: create-volume-name - value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' - dependencies: - - create-volume - - step2-gunzip - name: step3-copy - template: step3-copy - - arguments: - parameters: - - name: create-volume-name - value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' - dependencies: - - create-volume - - step3-copy - name: step3-snap - template: step3-snap - - arguments: - parameters: - - name: create-volume-name - value: '{{tasks.create-volume.outputs.parameters.create-volume-name}}' - dependencies: - - create-volume - - step3-copy - name: step4-output - template: step4-output - inputs: - parameters: - - name: url - name: volumesnapshotop-sequential diff --git a/sdk/python/tests/compiler/testdata/volumeop_basic.py b/sdk/python/tests/compiler/testdata/volumeop_basic.py deleted file mode 100644 index fe9d5465e54..00000000000 --- a/sdk/python/tests/compiler/testdata/volumeop_basic.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline( - name="VolumeOp Basic", description="A Basic Example on VolumeOp Usage.") -def volumeop_basic(size): - vop = dsl.VolumeOp( - name="create_pvc", - resource_name="my-pvc", - modes=dsl.VOLUME_MODE_RWM, - size=size) - - cop = dsl.ContainerOp( - name="cop", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo foo > /mnt/file1"], - pvolumes={"/mnt": vop.volume}) - - -if __name__ == "__main__": - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(volumeop_basic, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_basic.yaml b/sdk/python/tests/compiler/testdata/volumeop_basic.yaml deleted file mode 100644 index e2ab8103572..00000000000 --- a/sdk/python/tests/compiler/testdata/volumeop_basic.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "A Basic Example on VolumeOp - Usage.", "inputs": [{"name": "size"}], "name": "VolumeOp Basic"}' - generateName: volumeop-basic- -spec: - arguments: - parameters: - - name: size - entrypoint: volumeop-basic - serviceAccountName: pipeline-runner - templates: - - container: - args: - - echo foo > /mnt/file1 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /mnt - name: create-pvc - inputs: - parameters: - - name: create-pvc-name - name: cop - volumes: - - name: create-pvc - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-pvc-name}}' - - inputs: - parameters: - - name: size - name: create-pvc - outputs: - parameters: - - name: create-pvc-manifest - valueFrom: - jsonPath: '{}' - - name: create-pvc-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: create-pvc-size - valueFrom: - jsonPath: '{.status.capacity.storage}' - resource: - action: create - manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-my-pvc'\n\ - spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ - \ storage: '{{inputs.parameters.size}}'\n" - - dag: - tasks: - - arguments: - parameters: - - name: create-pvc-name - value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' - dependencies: - - create-pvc - name: cop - template: cop - - arguments: - parameters: - - name: size - value: '{{inputs.parameters.size}}' - name: create-pvc - template: create-pvc - inputs: - parameters: - - name: size - name: volumeop-basic diff --git a/sdk/python/tests/compiler/testdata/volumeop_dag.py b/sdk/python/tests/compiler/testdata/volumeop_dag.py deleted file mode 100644 index 4e37016aff7..00000000000 --- a/sdk/python/tests/compiler/testdata/volumeop_dag.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline( - name="Volume Op DAG", description="The second example of the design doc.") -def volume_op_dag(): - vop = dsl.VolumeOp( - name="create_pvc", - resource_name="my-pvc", - size="10Gi", - modes=dsl.VOLUME_MODE_RWM) - - step1 = dsl.ContainerOp( - name="step1", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo 1 | tee /mnt/file1"], - pvolumes={"/mnt": vop.volume}) - - step2 = dsl.ContainerOp( - name="step2", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo 2 | tee /mnt2/file2"], - pvolumes={"/mnt2": vop.volume}) - - step3 = dsl.ContainerOp( - name="step3", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["cat /mnt/file1 /mnt/file2"], - pvolumes={"/mnt": vop.volume.after(step1, step2)}) - - -if __name__ == "__main__": - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(volume_op_dag, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_dag.yaml b/sdk/python/tests/compiler/testdata/volumeop_dag.yaml deleted file mode 100644 index 420deff0e56..00000000000 --- a/sdk/python/tests/compiler/testdata/volumeop_dag.yaml +++ /dev/null @@ -1,115 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "The second example of - the design doc.", "name": "Volume Op DAG"}' - generateName: volume-op-dag- -spec: - arguments: - parameters: [] - entrypoint: volume-op-dag - serviceAccountName: pipeline-runner - templates: - - name: create-pvc - outputs: - parameters: - - name: create-pvc-manifest - valueFrom: - jsonPath: '{}' - - name: create-pvc-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: create-pvc-size - valueFrom: - jsonPath: '{.status.capacity.storage}' - resource: - action: create - manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-my-pvc'\n\ - spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ - \ storage: 10Gi\n" - - container: - args: - - echo 1 | tee /mnt/file1 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /mnt - name: create-pvc - inputs: - parameters: - - name: create-pvc-name - name: step1 - volumes: - - name: create-pvc - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-pvc-name}}' - - container: - args: - - echo 2 | tee /mnt2/file2 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /mnt2 - name: create-pvc - inputs: - parameters: - - name: create-pvc-name - name: step2 - volumes: - - name: create-pvc - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-pvc-name}}' - - container: - args: - - cat /mnt/file1 /mnt/file2 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /mnt - name: create-pvc - inputs: - parameters: - - name: create-pvc-name - name: step3 - volumes: - - name: create-pvc - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-pvc-name}}' - - dag: - tasks: - - name: create-pvc - template: create-pvc - - arguments: - parameters: - - name: create-pvc-name - value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' - dependencies: - - create-pvc - name: step1 - template: step1 - - arguments: - parameters: - - name: create-pvc-name - value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' - dependencies: - - create-pvc - name: step2 - template: step2 - - arguments: - parameters: - - name: create-pvc-name - value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' - dependencies: - - create-pvc - - step1 - - step2 - name: step3 - template: step3 - name: volume-op-dag diff --git a/sdk/python/tests/compiler/testdata/volumeop_parallel.py b/sdk/python/tests/compiler/testdata/volumeop_parallel.py deleted file mode 100644 index 53b5c63309c..00000000000 --- a/sdk/python/tests/compiler/testdata/volumeop_parallel.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline( - name="VolumeOp Parallel", - description="The first example of the design doc.") -def volumeop_parallel(): - vop = dsl.VolumeOp( - name="create_pvc", - resource_name="my-pvc", - size="10Gi", - modes=dsl.VOLUME_MODE_RWM) - - step1 = dsl.ContainerOp( - name="step1", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo 1 | tee /mnt/file1"], - pvolumes={"/mnt": vop.volume}) - - step2 = dsl.ContainerOp( - name="step2", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo 2 | tee /common/file2"], - pvolumes={"/common": vop.volume}) - - step3 = dsl.ContainerOp( - name="step3", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo 3 | tee /mnt3/file3"], - pvolumes={"/mnt3": vop.volume}) - - -if __name__ == "__main__": - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(volumeop_parallel, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml b/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml deleted file mode 100644 index cf278e50692..00000000000 --- a/sdk/python/tests/compiler/testdata/volumeop_parallel.yaml +++ /dev/null @@ -1,113 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "The first example of the - design doc.", "name": "VolumeOp Parallel"}' - generateName: volumeop-parallel- -spec: - arguments: - parameters: [] - entrypoint: volumeop-parallel - serviceAccountName: pipeline-runner - templates: - - name: create-pvc - outputs: - parameters: - - name: create-pvc-manifest - valueFrom: - jsonPath: '{}' - - name: create-pvc-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: create-pvc-size - valueFrom: - jsonPath: '{.status.capacity.storage}' - resource: - action: create - manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-my-pvc'\n\ - spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ - \ storage: 10Gi\n" - - container: - args: - - echo 1 | tee /mnt/file1 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /mnt - name: create-pvc - inputs: - parameters: - - name: create-pvc-name - name: step1 - volumes: - - name: create-pvc - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-pvc-name}}' - - container: - args: - - echo 2 | tee /common/file2 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /common - name: create-pvc - inputs: - parameters: - - name: create-pvc-name - name: step2 - volumes: - - name: create-pvc - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-pvc-name}}' - - container: - args: - - echo 3 | tee /mnt3/file3 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /mnt3 - name: create-pvc - inputs: - parameters: - - name: create-pvc-name - name: step3 - volumes: - - name: create-pvc - persistentVolumeClaim: - claimName: '{{inputs.parameters.create-pvc-name}}' - - dag: - tasks: - - name: create-pvc - template: create-pvc - - arguments: - parameters: - - name: create-pvc-name - value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' - dependencies: - - create-pvc - name: step1 - template: step1 - - arguments: - parameters: - - name: create-pvc-name - value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' - dependencies: - - create-pvc - name: step2 - template: step2 - - arguments: - parameters: - - name: create-pvc-name - value: '{{tasks.create-pvc.outputs.parameters.create-pvc-name}}' - dependencies: - - create-pvc - name: step3 - template: step3 - name: volumeop-parallel diff --git a/sdk/python/tests/compiler/testdata/volumeop_sequential.py b/sdk/python/tests/compiler/testdata/volumeop_sequential.py deleted file mode 100644 index 1945f09bc3a..00000000000 --- a/sdk/python/tests/compiler/testdata/volumeop_sequential.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline( - name="VolumeOp Sequential", - description="The third example of the design doc.") -def volumeop_sequential(): - vop = dsl.VolumeOp( - name="mypvc", - resource_name="newpvc", - size="10Gi", - modes=dsl.VOLUME_MODE_RWM) - - step1 = dsl.ContainerOp( - name="step1", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo 1|tee /data/file1"], - pvolumes={"/data": vop.volume}) - - step2 = dsl.ContainerOp( - name="step2", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["cp /data/file1 /data/file2"], - pvolumes={"/data": step1.pvolume}) - - step3 = dsl.ContainerOp( - name="step3", - image="library/bash:4.4.23", - command=["cat", "/mnt/file1", "/mnt/file2"], - pvolumes={"/mnt": step2.pvolume}) - - -if __name__ == "__main__": - import kfp.deprecated.compiler as compiler - compiler.Compiler().compile(volumeop_sequential, __file__ + ".tar.gz") diff --git a/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml b/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml deleted file mode 100644 index 0d9fe372799..00000000000 --- a/sdk/python/tests/compiler/testdata/volumeop_sequential.yaml +++ /dev/null @@ -1,114 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"description": "The third example of the - design doc.", "name": "VolumeOp Sequential"}' - generateName: volumeop-sequential- -spec: - arguments: - parameters: [] - entrypoint: volumeop-sequential - serviceAccountName: pipeline-runner - templates: - - name: mypvc - outputs: - parameters: - - name: mypvc-manifest - valueFrom: - jsonPath: '{}' - - name: mypvc-name - valueFrom: - jsonPath: '{.metadata.name}' - - name: mypvc-size - valueFrom: - jsonPath: '{.status.capacity.storage}' - resource: - action: create - manifest: "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: '{{workflow.name}}-newpvc'\n\ - spec:\n accessModes:\n - ReadWriteMany\n resources:\n requests:\n \ - \ storage: 10Gi\n" - - container: - args: - - echo 1|tee /data/file1 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /data - name: mypvc - inputs: - parameters: - - name: mypvc-name - name: step1 - volumes: - - name: mypvc - persistentVolumeClaim: - claimName: '{{inputs.parameters.mypvc-name}}' - - container: - args: - - cp /data/file1 /data/file2 - command: - - sh - - -c - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /data - name: mypvc - inputs: - parameters: - - name: mypvc-name - name: step2 - volumes: - - name: mypvc - persistentVolumeClaim: - claimName: '{{inputs.parameters.mypvc-name}}' - - container: - command: - - cat - - /mnt/file1 - - /mnt/file2 - image: library/bash:4.4.23 - volumeMounts: - - mountPath: /mnt - name: mypvc - inputs: - parameters: - - name: mypvc-name - name: step3 - volumes: - - name: mypvc - persistentVolumeClaim: - claimName: '{{inputs.parameters.mypvc-name}}' - - dag: - tasks: - - name: mypvc - template: mypvc - - arguments: - parameters: - - name: mypvc-name - value: '{{tasks.mypvc.outputs.parameters.mypvc-name}}' - dependencies: - - mypvc - name: step1 - template: step1 - - arguments: - parameters: - - name: mypvc-name - value: '{{tasks.mypvc.outputs.parameters.mypvc-name}}' - dependencies: - - mypvc - - step1 - name: step2 - template: step2 - - arguments: - parameters: - - name: mypvc-name - value: '{{tasks.mypvc.outputs.parameters.mypvc-name}}' - dependencies: - - mypvc - - step2 - name: step3 - template: step3 - name: volumeop-sequential diff --git a/sdk/python/tests/compiler/testdata/withitem_basic.py b/sdk/python/tests/compiler/testdata/withitem_basic.py deleted file mode 100644 index 7cf07968924..00000000000 --- a/sdk/python/tests/compiler/testdata/withitem_basic.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline(name='my-pipeline') -def pipeline(my_pipe_param: int = 10): - loop_args = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}] - with dsl.ParallelFor(loop_args) as item: - op1 = dsl.ContainerOp( - name="my-in-coop1", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo op1 %s %s" % (item.a, my_pipe_param)], - ) - - op2 = dsl.ContainerOp( - name="my-in-coop2", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo op2 %s" % item.b], - ) - - op_out = dsl.ContainerOp( - name="my-out-cop", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo %s" % my_pipe_param], - ) - - -if __name__ == '__main__': - from kfp.deprecated import compiler - print(compiler.Compiler().compile(pipeline, package_path=None)) - - import kfp.deprecated as kfp - client = kfp.Client(host='127.0.0.1:8080/pipeline') - - pkg_path = '/tmp/witest_pkg.tar.gz' - compiler.Compiler().compile(pipeline, package_path=pkg_path) - exp = client.create_experiment('withparams_exp') - client.run_pipeline( - experiment_id=exp.id, - job_name='withitem_basic', - pipeline_package_path=pkg_path, - params={}, - ) diff --git a/sdk/python/tests/compiler/testdata/withitem_basic.yaml b/sdk/python/tests/compiler/testdata/withitem_basic.yaml deleted file mode 100644 index eaf4aa0aed2..00000000000 --- a/sdk/python/tests/compiler/testdata/withitem_basic.yaml +++ /dev/null @@ -1,98 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "10", "name": "my_pipe_param", "type": "Integer"}], - "name": "my-pipeline"}' - generateName: my-pipeline- -spec: - arguments: - parameters: - - name: my_pipe_param - value: '10' - entrypoint: my-pipeline - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: loop-item-param-1-subvar-a - value: '{{inputs.parameters.loop-item-param-1-subvar-a}}' - - name: my_pipe_param - value: '{{inputs.parameters.my_pipe_param}}' - name: my-in-coop1 - template: my-in-coop1 - - arguments: - parameters: - - name: loop-item-param-1-subvar-b - value: '{{inputs.parameters.loop-item-param-1-subvar-b}}' - name: my-in-coop2 - template: my-in-coop2 - inputs: - parameters: - - name: loop-item-param-1-subvar-a - - name: loop-item-param-1-subvar-b - - name: my_pipe_param - name: for-loop-2 - - container: - args: - - echo op1 {{inputs.parameters.loop-item-param-1-subvar-a}} {{inputs.parameters.my_pipe_param}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: loop-item-param-1-subvar-a - - name: my_pipe_param - name: my-in-coop1 - - container: - args: - - echo op2 {{inputs.parameters.loop-item-param-1-subvar-b}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: loop-item-param-1-subvar-b - name: my-in-coop2 - - container: - args: - - echo {{inputs.parameters.my_pipe_param}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: my_pipe_param - name: my-out-cop - - dag: - tasks: - - arguments: - parameters: - - name: loop-item-param-1-subvar-a - value: '{{item.a}}' - - name: loop-item-param-1-subvar-b - value: '{{item.b}}' - - name: my_pipe_param - value: '{{inputs.parameters.my_pipe_param}}' - name: for-loop-2 - template: for-loop-2 - withItems: - - a: 1 - b: 2 - - a: 10 - b: 20 - - arguments: - parameters: - - name: my_pipe_param - value: '{{inputs.parameters.my_pipe_param}}' - name: my-out-cop - template: my-out-cop - inputs: - parameters: - - name: my_pipe_param - name: my-pipeline diff --git a/sdk/python/tests/compiler/testdata/withitem_nested.py b/sdk/python/tests/compiler/testdata/withitem_nested.py deleted file mode 100644 index 16c0899e63e..00000000000 --- a/sdk/python/tests/compiler/testdata/withitem_nested.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline(name='my-pipeline') -def pipeline(my_pipe_param: int = 10): - loop_args = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}] - with dsl.ParallelFor(loop_args) as item: - op1 = dsl.ContainerOp( - name="my-in-coop1", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo op1 %s %s" % (item.a, my_pipe_param)], - ) - - with dsl.ParallelFor([100, 200, 300]) as inner_item: - op11 = dsl.ContainerOp( - name="my-inner-inner-coop", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=[ - "echo op1 %s %s %s" % (item.a, inner_item, my_pipe_param) - ], - ) - - op2 = dsl.ContainerOp( - name="my-in-coop2", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo op2 %s" % item.b], - ) - - op_out = dsl.ContainerOp( - name="my-out-cop", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo %s" % my_pipe_param], - ) - - -if __name__ == '__main__': - import time - - from kfp.deprecated import compiler - import kfp.deprecated as kfp - client = kfp.Client(host='127.0.0.1:8080/pipeline') - print(compiler.Compiler().compile(pipeline, package_path=None)) - - pkg_path = '/tmp/witest_pkg.tar.gz' - compiler.Compiler().compile(pipeline, package_path=pkg_path) - exp = client.create_experiment('withparams_exp') - client.run_pipeline( - experiment_id=exp.id, - job_name='withitem_nested_{}'.format(time.time()), - pipeline_package_path=pkg_path, - params={}, - ) diff --git a/sdk/python/tests/compiler/testdata/withitem_nested.yaml b/sdk/python/tests/compiler/testdata/withitem_nested.yaml deleted file mode 100644 index 7c650846d8f..00000000000 --- a/sdk/python/tests/compiler/testdata/withitem_nested.yaml +++ /dev/null @@ -1,144 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "10", "name": "my_pipe_param", "type": "Integer"}], - "name": "my-pipeline"}' - generateName: my-pipeline- -spec: - arguments: - parameters: - - name: my_pipe_param - value: '10' - entrypoint: my-pipeline - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: loop-item-param-1-subvar-a - value: '{{inputs.parameters.loop-item-param-1-subvar-a}}' - - name: loop-item-param-3 - value: '{{item}}' - - name: my_pipe_param - value: '{{inputs.parameters.my_pipe_param}}' - name: for-loop-4 - template: for-loop-4 - withItems: - - 100 - - 200 - - 300 - - arguments: - parameters: - - name: loop-item-param-1-subvar-a - value: '{{inputs.parameters.loop-item-param-1-subvar-a}}' - - name: my_pipe_param - value: '{{inputs.parameters.my_pipe_param}}' - name: my-in-coop1 - template: my-in-coop1 - - arguments: - parameters: - - name: loop-item-param-1-subvar-b - value: '{{inputs.parameters.loop-item-param-1-subvar-b}}' - name: my-in-coop2 - template: my-in-coop2 - inputs: - parameters: - - name: loop-item-param-1-subvar-a - - name: loop-item-param-1-subvar-b - - name: my_pipe_param - name: for-loop-2 - - dag: - tasks: - - arguments: - parameters: - - name: loop-item-param-1-subvar-a - value: '{{inputs.parameters.loop-item-param-1-subvar-a}}' - - name: loop-item-param-3 - value: '{{inputs.parameters.loop-item-param-3}}' - - name: my_pipe_param - value: '{{inputs.parameters.my_pipe_param}}' - name: my-inner-inner-coop - template: my-inner-inner-coop - inputs: - parameters: - - name: loop-item-param-1-subvar-a - - name: loop-item-param-3 - - name: my_pipe_param - name: for-loop-4 - - container: - args: - - echo op1 {{inputs.parameters.loop-item-param-1-subvar-a}} {{inputs.parameters.my_pipe_param}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: loop-item-param-1-subvar-a - - name: my_pipe_param - name: my-in-coop1 - - container: - args: - - echo op2 {{inputs.parameters.loop-item-param-1-subvar-b}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: loop-item-param-1-subvar-b - name: my-in-coop2 - - container: - args: - - echo op1 {{inputs.parameters.loop-item-param-1-subvar-a}} {{inputs.parameters.loop-item-param-3}} - {{inputs.parameters.my_pipe_param}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: loop-item-param-1-subvar-a - - name: loop-item-param-3 - - name: my_pipe_param - name: my-inner-inner-coop - - container: - args: - - echo {{inputs.parameters.my_pipe_param}} - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: my_pipe_param - name: my-out-cop - - dag: - tasks: - - arguments: - parameters: - - name: loop-item-param-1-subvar-a - value: '{{item.a}}' - - name: loop-item-param-1-subvar-b - value: '{{item.b}}' - - name: my_pipe_param - value: '{{inputs.parameters.my_pipe_param}}' - name: for-loop-2 - template: for-loop-2 - withItems: - - a: 1 - b: 2 - - a: 10 - b: 20 - - arguments: - parameters: - - name: my_pipe_param - value: '{{inputs.parameters.my_pipe_param}}' - name: my-out-cop - template: my-out-cop - inputs: - parameters: - - name: my_pipe_param - name: my-pipeline diff --git a/sdk/python/tests/compiler/testdata/withparam_global.py b/sdk/python/tests/compiler/testdata/withparam_global.py deleted file mode 100644 index fd8575d1052..00000000000 --- a/sdk/python/tests/compiler/testdata/withparam_global.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline(name='my-pipeline') -def pipeline(loopidy_doop: list = [3, 5, 7, 9]): - op0 = dsl.ContainerOp( - name="my-out-cop0", - image='python:alpine3.9', - command=["sh", "-c"], - arguments=[ - 'python -c "import json; import sys; json.dump([i for i in range(20, 31)], open(\'/tmp/out.json\', \'w\'))"' - ], - file_outputs={'out': '/tmp/out.json'}, - ) - - with dsl.ParallelFor(loopidy_doop) as item: - op1 = dsl.ContainerOp( - name="my-in-cop1", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo no output global op1, item: %s" % item], - ).after(op0) - - op_out = dsl.ContainerOp( - name="my-out-cop2", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo no output global op2, outp: %s" % op0.output], - ) - - -if __name__ == '__main__': - import time - - from kfp.deprecated import compiler - import kfp.deprecated as kfp - client = kfp.Client(host='127.0.0.1:8080/pipeline') - print(compiler.Compiler().compile(pipeline, package_path=None)) - - pkg_path = '/tmp/witest_pkg.tar.gz' - compiler.Compiler().compile(pipeline, package_path=pkg_path) - exp = client.create_experiment('withparams_exp') - client.run_pipeline( - experiment_id=exp.id, - job_name='withparam_global_{}'.format(time.time()), - pipeline_package_path=pkg_path, - params={}, - ) diff --git a/sdk/python/tests/compiler/testdata/withparam_global.yaml b/sdk/python/tests/compiler/testdata/withparam_global.yaml deleted file mode 100644 index 303c9a1d764..00000000000 --- a/sdk/python/tests/compiler/testdata/withparam_global.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[3, 5, 7, 9]", "name": - "loopidy_doop", "type": "JsonArray"}], "name": "my-pipeline"}' - generateName: my-pipeline- -spec: - arguments: - parameters: - - name: loopidy_doop - value: '[3, 5, 7, 9]' - entrypoint: my-pipeline - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: loopidy_doop-loop-item - value: '{{inputs.parameters.loopidy_doop-loop-item}}' - name: my-in-cop1 - template: my-in-cop1 - inputs: - parameters: - - name: loopidy_doop-loop-item - name: for-loop-1 - - container: - args: - - 'echo no output global op1, item: {{inputs.parameters.loopidy_doop-loop-item}}' - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: loopidy_doop-loop-item - name: my-in-cop1 - - container: - args: - - python -c "import json; import sys; json.dump([i for i in range(20, 31)], - open('/tmp/out.json', 'w'))" - command: - - sh - - -c - image: python:alpine3.9 - name: my-out-cop0 - outputs: - artifacts: - - name: my-out-cop0-out - path: /tmp/out.json - parameters: - - name: my-out-cop0-out - valueFrom: - path: /tmp/out.json - - container: - args: - - 'echo no output global op2, outp: {{inputs.parameters.my-out-cop0-out}}' - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: my-out-cop0-out - name: my-out-cop2 - - dag: - tasks: - - arguments: - parameters: - - name: loopidy_doop-loop-item - value: '{{item}}' - dependencies: - - my-out-cop0 - name: for-loop-1 - template: for-loop-1 - withParam: '{{workflow.parameters.loopidy_doop}}' - - name: my-out-cop0 - template: my-out-cop0 - - arguments: - parameters: - - name: my-out-cop0-out - value: '{{tasks.my-out-cop0.outputs.parameters.my-out-cop0-out}}' - dependencies: - - my-out-cop0 - name: my-out-cop2 - template: my-out-cop2 - name: my-pipeline diff --git a/sdk/python/tests/compiler/testdata/withparam_global_dict.py b/sdk/python/tests/compiler/testdata/withparam_global_dict.py deleted file mode 100644 index d7501f0dc8a..00000000000 --- a/sdk/python/tests/compiler/testdata/withparam_global_dict.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline(name='my-pipeline') -def pipeline(loopidy_doop: dict = [{'a': 1, 'b': 2}, {'a': 10, 'b': 20}]): - op0 = dsl.ContainerOp( - name="my-out-cop0", - image='python:alpine3.9', - command=["sh", "-c"], - arguments=[ - 'python -c "import json; import sys; json.dump([i for i in range(20, 31)], open(\'/tmp/out.json\', \'w\'))"' - ], - file_outputs={'out': '/tmp/out.json'}, - ) - - with dsl.ParallelFor(loopidy_doop) as item: - op1 = dsl.ContainerOp( - name="my-in-cop1", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo no output global op1, item.a: %s" % item.a], - ).after(op0) - - op_out = dsl.ContainerOp( - name="my-out-cop2", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo no output global op2, outp: %s" % op0.output], - ) - - -if __name__ == '__main__': - import time - - from kfp.deprecated import compiler - import kfp.deprecated as kfp - client = kfp.Client(host='127.0.0.1:8080/pipeline') - print(compiler.Compiler().compile(pipeline, package_path=None)) - - pkg_path = '/tmp/witest_pkg.tar.gz' - compiler.Compiler().compile(pipeline, package_path=pkg_path) - exp = client.create_experiment('withparams_exp') - client.run_pipeline( - experiment_id=exp.id, - job_name='withparam_global_dict_{}'.format(time.time()), - pipeline_package_path=pkg_path, - params={}, - ) diff --git a/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml b/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml deleted file mode 100644 index 3880e7dfd2b..00000000000 --- a/sdk/python/tests/compiler/testdata/withparam_global_dict.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "[{\"a\": 1, \"b\": - 2}, {\"a\": 10, \"b\": 20}]", "name": "loopidy_doop", "type": "JsonObject"}], "name": "my-pipeline"}' - generateName: my-pipeline- -spec: - arguments: - parameters: - - name: loopidy_doop - value: '[{"a": 1, "b": 2}, {"a": 10, "b": 20}]' - entrypoint: my-pipeline - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: loopidy_doop-loop-item-subvar-a - value: '{{inputs.parameters.loopidy_doop-loop-item-subvar-a}}' - name: my-in-cop1 - template: my-in-cop1 - inputs: - parameters: - - name: loopidy_doop-loop-item-subvar-a - name: for-loop-1 - - container: - args: - - 'echo no output global op1, item.a: {{inputs.parameters.loopidy_doop-loop-item-subvar-a}}' - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: loopidy_doop-loop-item-subvar-a - name: my-in-cop1 - - container: - args: - - python -c "import json; import sys; json.dump([i for i in range(20, 31)], - open('/tmp/out.json', 'w'))" - command: - - sh - - -c - image: python:alpine3.9 - name: my-out-cop0 - outputs: - artifacts: - - name: my-out-cop0-out - path: /tmp/out.json - parameters: - - name: my-out-cop0-out - valueFrom: - path: /tmp/out.json - - container: - args: - - 'echo no output global op2, outp: {{inputs.parameters.my-out-cop0-out}}' - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: my-out-cop0-out - name: my-out-cop2 - - dag: - tasks: - - arguments: - parameters: - - name: loopidy_doop-loop-item-subvar-a - value: '{{item.a}}' - dependencies: - - my-out-cop0 - name: for-loop-1 - template: for-loop-1 - withParam: '{{workflow.parameters.loopidy_doop}}' - - name: my-out-cop0 - template: my-out-cop0 - - arguments: - parameters: - - name: my-out-cop0-out - value: '{{tasks.my-out-cop0.outputs.parameters.my-out-cop0-out}}' - dependencies: - - my-out-cop0 - name: my-out-cop2 - template: my-out-cop2 - name: my-pipeline diff --git a/sdk/python/tests/compiler/testdata/withparam_output.py b/sdk/python/tests/compiler/testdata/withparam_output.py deleted file mode 100644 index 845e46afe64..00000000000 --- a/sdk/python/tests/compiler/testdata/withparam_output.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline(name='my-pipeline') -def pipeline(): - op0 = dsl.ContainerOp( - name="my-out-cop0", - image='python:alpine3.9', - command=["sh", "-c"], - arguments=[ - 'python -c "import json; import sys; json.dump([i for i in range(20, 31)], open(\'/tmp/out.json\', \'w\'))"' - ], - file_outputs={'out': '/tmp/out.json'}, - ) - - with dsl.ParallelFor(op0.output) as item: - op1 = dsl.ContainerOp( - name="my-in-cop1", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo do output op1 item: %s" % item], - ) - - op_out = dsl.ContainerOp( - name="my-out-cop2", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo do output op2, outp: %s" % op0.output], - ) - - -if __name__ == '__main__': - import time - - from kfp.deprecated import compiler - import kfp.deprecated as kfp - client = kfp.Client(host='127.0.0.1:8080/pipeline') - print(compiler.Compiler().compile(pipeline, package_path=None)) - - pkg_path = '/tmp/witest_pkg.tar.gz' - compiler.Compiler().compile(pipeline, package_path=pkg_path) - exp = client.create_experiment('withparams_exp') - client.run_pipeline( - experiment_id=exp.id, - job_name='withparam_output_{}'.format(time.time()), - pipeline_package_path=pkg_path, - params={}, - ) diff --git a/sdk/python/tests/compiler/testdata/withparam_output.yaml b/sdk/python/tests/compiler/testdata/withparam_output.yaml deleted file mode 100644 index 4a59c3cb9b1..00000000000 --- a/sdk/python/tests/compiler/testdata/withparam_output.yaml +++ /dev/null @@ -1,85 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"name": "my-pipeline"}' - generateName: my-pipeline- -spec: - arguments: - parameters: [] - entrypoint: my-pipeline - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: my-out-cop0-out-loop-item - value: '{{inputs.parameters.my-out-cop0-out-loop-item}}' - name: my-in-cop1 - template: my-in-cop1 - inputs: - parameters: - - name: my-out-cop0-out-loop-item - name: for-loop-1 - - container: - args: - - 'echo do output op1 item: {{inputs.parameters.my-out-cop0-out-loop-item}}' - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: my-out-cop0-out-loop-item - name: my-in-cop1 - - container: - args: - - python -c "import json; import sys; json.dump([i for i in range(20, 31)], - open('/tmp/out.json', 'w'))" - command: - - sh - - -c - image: python:alpine3.9 - name: my-out-cop0 - outputs: - artifacts: - - name: my-out-cop0-out - path: /tmp/out.json - parameters: - - name: my-out-cop0-out - valueFrom: - path: /tmp/out.json - - container: - args: - - 'echo do output op2, outp: {{inputs.parameters.my-out-cop0-out}}' - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: my-out-cop0-out - name: my-out-cop2 - - dag: - tasks: - - arguments: - parameters: - - name: my-out-cop0-out-loop-item - value: '{{item}}' - dependencies: - - my-out-cop0 - name: for-loop-1 - template: for-loop-1 - withParam: '{{tasks.my-out-cop0.outputs.parameters.my-out-cop0-out}}' - - name: my-out-cop0 - template: my-out-cop0 - - arguments: - parameters: - - name: my-out-cop0-out - value: '{{tasks.my-out-cop0.outputs.parameters.my-out-cop0-out}}' - dependencies: - - my-out-cop0 - name: my-out-cop2 - template: my-out-cop2 - name: my-pipeline diff --git a/sdk/python/tests/compiler/testdata/withparam_output_dict.py b/sdk/python/tests/compiler/testdata/withparam_output_dict.py deleted file mode 100644 index e4e46fd8fc6..00000000000 --- a/sdk/python/tests/compiler/testdata/withparam_output_dict.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import kfp.deprecated.dsl as dsl - - -@dsl.pipeline(name='my-pipeline') -def pipeline(): - op0 = dsl.ContainerOp( - name="my-out-cop0", - image='python:alpine3.9', - command=["sh", "-c"], - arguments=[ - 'python -c "import json; import sys; json.dump([{\'a\': 1, \'b\': 2}, {\'a\': 10, \'b\': 20}], open(\'/tmp/out.json\', \'w\'))"' - ], - file_outputs={'out': '/tmp/out.json'}, - ) - - with dsl.ParallelFor(op0.output) as item: - op1 = dsl.ContainerOp( - name="my-in-cop1", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo do output op1 item.a: %s" % item.a], - ) - - op_out = dsl.ContainerOp( - name="my-out-cop2", - image="library/bash:4.4.23", - command=["sh", "-c"], - arguments=["echo do output op2, outp: %s" % op0.output], - ) - - -if __name__ == '__main__': - import time - - from kfp.deprecated import compiler - import kfp.deprecated as kfp - client = kfp.Client(host='127.0.0.1:8080/pipeline') - print(compiler.Compiler().compile(pipeline, package_path=None)) - - pkg_path = '/tmp/witest_pkg.tar.gz' - compiler.Compiler().compile(pipeline, package_path=pkg_path) - exp = client.create_experiment('withparams_exp') - client.run_pipeline( - experiment_id=exp.id, - job_name='withparam_output_dict_{}'.format(time.time()), - pipeline_package_path=pkg_path, - params={}, - ) diff --git a/sdk/python/tests/compiler/testdata/withparam_output_dict.yaml b/sdk/python/tests/compiler/testdata/withparam_output_dict.yaml deleted file mode 100644 index a396d442ad2..00000000000 --- a/sdk/python/tests/compiler/testdata/withparam_output_dict.yaml +++ /dev/null @@ -1,85 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - annotations: - pipelines.kubeflow.org/pipeline_spec: '{"name": "my-pipeline"}' - generateName: my-pipeline- -spec: - arguments: - parameters: [] - entrypoint: my-pipeline - serviceAccountName: pipeline-runner - templates: - - dag: - tasks: - - arguments: - parameters: - - name: my-out-cop0-out-loop-item-subvar-a - value: '{{inputs.parameters.my-out-cop0-out-loop-item-subvar-a}}' - name: my-in-cop1 - template: my-in-cop1 - inputs: - parameters: - - name: my-out-cop0-out-loop-item-subvar-a - name: for-loop-1 - - container: - args: - - 'echo do output op1 item.a: {{inputs.parameters.my-out-cop0-out-loop-item-subvar-a}}' - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: my-out-cop0-out-loop-item-subvar-a - name: my-in-cop1 - - container: - args: - - 'python -c "import json; import sys; json.dump([{''a'': 1, ''b'': 2}, {''a'': - 10, ''b'': 20}], open(''/tmp/out.json'', ''w''))"' - command: - - sh - - -c - image: python:alpine3.9 - name: my-out-cop0 - outputs: - artifacts: - - name: my-out-cop0-out - path: /tmp/out.json - parameters: - - name: my-out-cop0-out - valueFrom: - path: /tmp/out.json - - container: - args: - - 'echo do output op2, outp: {{inputs.parameters.my-out-cop0-out}}' - command: - - sh - - -c - image: library/bash:4.4.23 - inputs: - parameters: - - name: my-out-cop0-out - name: my-out-cop2 - - dag: - tasks: - - arguments: - parameters: - - name: my-out-cop0-out-loop-item-subvar-a - value: '{{item.a}}' - dependencies: - - my-out-cop0 - name: for-loop-1 - template: for-loop-1 - withParam: '{{tasks.my-out-cop0.outputs.parameters.my-out-cop0-out}}' - - name: my-out-cop0 - template: my-out-cop0 - - arguments: - parameters: - - name: my-out-cop0-out - value: '{{tasks.my-out-cop0.outputs.parameters.my-out-cop0-out}}' - dependencies: - - my-out-cop0 - name: my-out-cop2 - template: my-out-cop2 - name: my-pipeline diff --git a/sdk/python/tests/dsl/__init__.py b/sdk/python/tests/dsl/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/tests/dsl/aws_extensions_tests.py b/sdk/python/tests/dsl/aws_extensions_tests.py deleted file mode 100644 index 210e1a04aae..00000000000 --- a/sdk/python/tests/dsl/aws_extensions_tests.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -import unittest - -from kfp.deprecated.aws import use_aws_secret -from kfp.deprecated.dsl import ContainerOp - - -class TestAwsExtension(unittest.TestCase): - - def test_default_aws_secret_name(self): - spec = inspect.getfullargspec(use_aws_secret) - assert len(spec.defaults) == 4 - assert spec.defaults[0] == 'aws-secret' - assert spec.defaults[1] == 'AWS_ACCESS_KEY_ID' - assert spec.defaults[2] == 'AWS_SECRET_ACCESS_KEY' - assert spec.defaults[3] == None - - def test_use_aws_secret(self): - op1 = ContainerOp(name='op1', image='image') - op1 = op1.apply(use_aws_secret('myaws-secret', 'key_id', 'access_key')) - assert len(op1.container.env) == 2 - - index = 0 - for expected_name, expected_key in [('AWS_ACCESS_KEY_ID', 'key_id'), - ('AWS_SECRET_ACCESS_KEY', - 'access_key')]: - assert op1.container.env[index].name == expected_name - assert op1.container.env[ - index].value_from.secret_key_ref.name == 'myaws-secret' - assert op1.container.env[ - index].value_from.secret_key_ref.key == expected_key - index += 1 - - def test_use_aws_secret_with_region(self): - op1 = ContainerOp(name='op1', image='image') - aws_region = 'us-west-2' - op1 = op1.apply( - use_aws_secret('myaws-secret', 'key_id', 'access_key', aws_region)) - assert len(op1.container.env) == 3 - - index = 0 - for expected_name, expected_key in [('AWS_ACCESS_KEY_ID', 'key_id'), - ('AWS_SECRET_ACCESS_KEY', - 'access_key')]: - assert op1.container.env[index].name == expected_name - assert op1.container.env[ - index].value_from.secret_key_ref.name == 'myaws-secret' - assert op1.container.env[ - index].value_from.secret_key_ref.key == expected_key - index += 1 - - for expected_name, expected_key in [('AWS_REGION', aws_region)]: - assert op1.container.env[index].name == expected_name - assert op1.container.env[index].value == expected_key diff --git a/sdk/python/tests/dsl/component_bridge_tests.py b/sdk/python/tests/dsl/component_bridge_tests.py deleted file mode 100644 index da8478fc6f6..00000000000 --- a/sdk/python/tests/dsl/component_bridge_tests.py +++ /dev/null @@ -1,414 +0,0 @@ -# Copyright 2020 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path -import tempfile -import textwrap -import unittest -import warnings - -import kfp.deprecated as kfp -from kfp.deprecated.components import create_component_from_func -from kfp.deprecated.components import load_component_from_text -from kfp.deprecated.dsl import PipelineParam - - -class TestComponentBridge(unittest.TestCase): - # Alternatively, we could use kfp.dsl.Pipleine().__enter__ and __exit__ - def setUp(self): - self.old_container_task_constructor = kfp.components._components._container_task_constructor - kfp.components._components._container_task_constructor = kfp.dsl._component_bridge._create_container_op_from_component_and_arguments - - def tearDown(self): - kfp.components._components._container_task_constructor = self.old_container_task_constructor - - def test_conversion_to_container_op(self): - component_text = textwrap.dedent('''\ - name: Custom component - implementation: - container: - image: busybox - ''') - task_factory1 = load_component_from_text(component_text) - task1 = task_factory1() - - self.assertEqual(task1.human_name, 'Custom component') - - def test_passing_env_to_container_op(self): - component_text = textwrap.dedent('''\ - implementation: - container: - image: busybox - env: - key1: value 1 - key2: value 2 - ''') - task_factory1 = load_component_from_text(component_text) - - task1 = task_factory1() - actual_env = { - env_var.name: env_var.value for env_var in task1.container.env - } - expected_env = {'key1': 'value 1', 'key2': 'value 2'} - self.assertDictEqual(expected_env, actual_env) - - def test_input_path_placeholder_with_constant_argument(self): - component_text = textwrap.dedent('''\ - inputs: - - {name: input 1} - implementation: - container: - image: busybox - command: - - --input-data - - {inputPath: input 1} - ''') - task_factory1 = load_component_from_text(component_text) - task1 = task_factory1('Text') - - self.assertEqual( - task1.command, - ['--input-data', task1.input_artifact_paths['input 1']]) - self.assertEqual(task1.artifact_arguments, {'input 1': 'Text'}) - - def test_passing_component_metadata_to_container_op(self): - component_text = textwrap.dedent('''\ - metadata: - annotations: - key1: value1 - labels: - key1: value1 - implementation: - container: - image: busybox - ''') - task_factory1 = load_component_from_text(text=component_text) - - task1 = task_factory1() - self.assertEqual(task1.pod_annotations['key1'], 'value1') - self.assertEqual(task1.pod_labels['key1'], 'value1') - - def test_volatile_components(self): - component_text = textwrap.dedent('''\ - metadata: - annotations: - volatile_component: "true" - implementation: - container: - image: busybox - ''') - task_factory1 = load_component_from_text(text=component_text) - - task1 = task_factory1() - self.assertEqual( - task1.execution_options.caching_strategy.max_cache_staleness, 'P0D') - - def test_type_compatibility_check_not_failing_when_disabled(self): - component_a = textwrap.dedent('''\ - outputs: - - {name: out1, type: type_A} - implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] - ''') - component_b = textwrap.dedent('''\ - inputs: - - {name: in1, type: type_Z} - implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] - ''') - kfp.TYPE_CHECK = False - task_factory_a = load_component_from_text(component_a) - task_factory_b = load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1']) - kfp.TYPE_CHECK = True - - def test_type_compatibility_check_not_failing_when_type_is_ignored(self): - component_a = textwrap.dedent('''\ - outputs: - - {name: out1, type: type_A} - implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] - ''') - component_b = textwrap.dedent('''\ - inputs: - - {name: in1, type: type_Z} - implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] - ''') - task_factory_a = load_component_from_text(component_a) - task_factory_b = load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1'].ignore_type()) - - def test_end_to_end_python_component_pipeline_compilation(self): - import kfp.deprecated.components as comp - - #Defining the Python function - def add(a: float, b: float) -> float: - """Returns sum of two arguments.""" - return a + b - - with tempfile.TemporaryDirectory() as temp_dir_name: - add_component_file = str( - Path(temp_dir_name).joinpath('add.component.yaml')) - - #Converting the function to a component. Instantiate it to create a pipeline task (ContaineOp instance) - add_op = comp.func_to_container_op( - add, - base_image='python:3.5', - output_component_file=add_component_file) - - #Checking that the component artifact is usable: - add_op2 = comp.load_component_from_file(add_component_file) - - #Building the pipeline - @kfp.dsl.pipeline( - name='Calculation pipeline', - description='A pipeline that performs arithmetic calculations.') - def calc_pipeline( - a1, - a2='7', - a3='17', - ): - task_1 = add_op(a1, a2) - task_2 = add_op2(a1, a2) - task_3 = add_op(task_1.output, task_2.output) - task_4 = add_op2(task_3.output, a3) - - #Compiling the pipleine: - pipeline_filename = str( - Path(temp_dir_name).joinpath(calc_pipeline.__name__ + - '.pipeline.tar.gz')) - kfp.compiler.Compiler().compile(calc_pipeline, pipeline_filename) - - def test_handling_list_arguments_containing_pipelineparam(self): - """Checks that lists containing PipelineParam can be properly - serialized.""" - - def consume_list(list_param: list) -> int: - pass - - import kfp.deprecated as kfp - task_factory = create_component_from_func(consume_list) - task = task_factory([1, 2, 3, kfp.dsl.PipelineParam('aaa'), 4, 5, 6]) - - full_command_line = task.command + task.arguments - for arg in full_command_line: - self.assertNotIn('PipelineParam', arg) - - def test_converted_outputs(self): - component_text = textwrap.dedent('''\ - outputs: - - name: Output 1 - implementation: - container: - image: busybox - command: - - producer - - {outputPath: Output 1} # Outputs must be used in the implementation - ''') - task_factory1 = load_component_from_text(component_text) - task1 = task_factory1() - - self.assertSetEqual(set(task1.outputs.keys()), {'Output 1', 'output_1'}) - self.assertIsNotNone(task1.output) - - def test_reusable_component_warnings(self): - op1 = load_component_from_text('''\ - implementation: - container: - image: busybox - ''') - with warnings.catch_warnings(record=True) as warning_messages: - op1() - deprecation_messages = list( - str(message) - for message in warning_messages - if message.category == DeprecationWarning) - self.assertListEqual(deprecation_messages, []) - - with self.assertWarnsRegex(FutureWarning, expected_regex='reusable'): - kfp.dsl.ContainerOp(name='name', image='image') - - with self.assertWarnsRegex(FutureWarning, expected_regex='reusable'): - kfp.dsl.ContainerOp( - name='name', - image='image', - arguments=[PipelineParam('param1'), - PipelineParam('param2')]) - - def test_prevent_passing_container_op_as_argument(self): - component_text = textwrap.dedent('''\ - inputs: - - {name: input 1} - - {name: input 2} - implementation: - container: - image: busybox - command: - - prog - - {inputValue: input 1} - - {inputPath: input 2} - ''') - component = load_component_from_text(component_text) - # Passing normal values to component - task1 = component(input_1="value 1", input_2="value 2") - # Passing unserializable values to component - with self.assertRaises(TypeError): - component(input_1=task1, input_2="value 2") - with self.assertRaises(TypeError): - component(input_1="value 1", input_2=task1) - - def test_pythonic_container_output_handled_by_graph(self): - component_a = textwrap.dedent('''\ - inputs: [] - outputs: - - {name: out1, type: str} - implementation: - graph: - tasks: - some-container: - arguments: {} - componentRef: - spec: - outputs: - - {name: out1, type: str} - implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out1}] - outputValues: - out1: - taskOutput: - taskId: some-container - outputName: out1 - ''') - component_b = textwrap.dedent('''\ - inputs: - - {name: in1, type: str} - implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] - ''') - task_factory_a = load_component_from_text(component_a) - task_factory_b = load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1']) - - def test_nonpythonic_container_output_handled_by_graph(self): - component_a = textwrap.dedent('''\ - inputs: [] - outputs: - - {name: out1, type: str} - implementation: - graph: - tasks: - some-container: - arguments: {} - componentRef: - spec: - outputs: - - {name: out-1, type: str} - implementation: - container: - image: busybox - command: [bash, -c, 'mkdir -p "$(dirname "$0")"; date > "$0"', {outputPath: out-1}] - outputValues: - out1: - taskOutput: - taskId: some-container - outputName: out-1 - ''') - component_b = textwrap.dedent('''\ - inputs: - - {name: in1, type: str} - implementation: - container: - image: busybox - command: [echo, {inputValue: in1}] - ''') - task_factory_a = load_component_from_text(component_a) - task_factory_b = load_component_from_text(component_b) - a_task = task_factory_a() - b_task = task_factory_b(in1=a_task.outputs['out1']) - - # v2 tests - - def test_input_output_uri_resolving(self): - component_text = textwrap.dedent('''\ - inputs: - - {name: In1} - outputs: - - {name: Out1} - implementation: - container: - image: busybox - command: - - program - - --in1-uri - - {inputUri: In1} - - --out1-uri - - {outputUri: Out1} - ''') - op = load_component_from_text(text=component_text) - task = op(in1='foo') - - self.assertEqual([ - 'program', - '--in1-uri', - '{{$.inputs.artifacts[\'In1\'].uri}}', - '--out1-uri', - '{{$.outputs.artifacts[\'Out1\'].uri}}', - ], task.command) - - def test_convert_executor_input_and_output_metadata_placeholder(self): - test_component = textwrap.dedent("""\ - inputs: - - {name: in1} - outputs: - - {name: out1} - implementation: - container: - image: busybox - command: [echo, {executorInput}, {outputMetadata}] - """) - task_factory = load_component_from_text(test_component) - task = task_factory(in1='foo') - self.assertListEqual( - ['echo', '{{$}}', '/tmp/outputs/executor_output.json'], - task.command) - - def test_fail_executor_input_with_key(self): - test_component = textwrap.dedent("""\ - inputs: - - {name: in1} - outputs: - - {name: out1} - implementation: - container: - image: busybox - command: [echo, {executorInput: a_bad_key}] - """) - with self.assertRaises(TypeError): - _ = load_component_from_text(test_component) diff --git a/sdk/python/tests/dsl/component_tests.py b/sdk/python/tests/dsl/component_tests.py deleted file mode 100644 index 057d306efa0..00000000000 --- a/sdk/python/tests/dsl/component_tests.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from kfp.deprecated.dsl import graph_component -from kfp.deprecated.dsl import Pipeline -from kfp.deprecated.dsl import PipelineParam -import kfp.deprecated.dsl as dsl - - -@unittest.skip("deprecated") -class TestGraphComponent(unittest.TestCase): - - def test_graphcomponent_basic(self): - """Test graph_component decorator metadata.""" - - @graph_component - def flip_component(flip_result): - with dsl.Condition(flip_result == 'heads'): - flip_component(flip_result) - - with Pipeline('pipeline') as p: - param = PipelineParam(name='param') - flip_component(param) - self.assertEqual(1, len(p.groups)) - self.assertEqual(1, len(p.groups[0].groups)) # pipeline - self.assertEqual(1, len( - p.groups[0].groups[0].groups)) # flip_component - self.assertEqual(1, len( - p.groups[0].groups[0].groups[0].groups)) # condition - self.assertEqual(0, - len(p.groups[0].groups[0].groups[0].groups[0] - .groups)) # recursive flip_component - recursive_group = p.groups[0].groups[0].groups[0].groups[0] - self.assertTrue(recursive_group.recursive_ref is not None) - self.assertEqual(1, len(recursive_group.inputs)) - self.assertEqual('param', recursive_group.inputs[0].name) diff --git a/sdk/python/tests/dsl/extensions/__init__.py b/sdk/python/tests/dsl/extensions/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/sdk/python/tests/dsl/extensions/test_kubernetes.py b/sdk/python/tests/dsl/extensions/test_kubernetes.py deleted file mode 100644 index 3880d6746e1..00000000000 --- a/sdk/python/tests/dsl/extensions/test_kubernetes.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from kfp.deprecated.dsl import ContainerOp -from kfp.deprecated.dsl.extensions.kubernetes import use_secret - - -class TestAddSecrets(unittest.TestCase): - - def test_use_default_use_secret(self): - op1 = ContainerOp(name="op1", image="image") - secret_name = "my-secret" - secret_path = "/here/are/my/secret" - op1 = op1.apply( - use_secret( - secret_name=secret_name, secret_volume_mount_path=secret_path)) - self.assertEqual(type(op1.container.env), type(None)) - container_dict = op1.container.to_dict() - volume_mounts = container_dict["volume_mounts"][0] - self.assertEqual(volume_mounts["name"], secret_name) - self.assertEqual(type(volume_mounts), dict) - self.assertEqual(volume_mounts["mount_path"], secret_path) - - def test_use_set_volume_use_secret(self): - op1 = ContainerOp(name="op1", image="image") - secret_name = "my-secret" - secret_path = "/here/are/my/secret" - op1 = op1.apply( - use_secret( - secret_name=secret_name, secret_volume_mount_path=secret_path)) - self.assertEqual(type(op1.container.env), type(None)) - container_dict = op1.container.to_dict() - volume_mounts = container_dict["volume_mounts"][0] - self.assertEqual(type(volume_mounts), dict) - self.assertEqual(volume_mounts["mount_path"], secret_path) - - def test_use_set_env_use_secret(self): - op1 = ContainerOp(name="op1", image="image") - secret_name = "my-secret" - secret_path = "/here/are/my/secret/" - env_variable = "MY_SECRET" - secret_file_path_in_volume = "secret.json" - op1 = op1.apply( - use_secret( - secret_name=secret_name, - secret_volume_mount_path=secret_path, - env_variable=env_variable, - secret_file_path_in_volume=secret_file_path_in_volume)) - self.assertEqual(len(op1.container.env), 1) - container_dict = op1.container.to_dict() - volume_mounts = container_dict["volume_mounts"][0] - self.assertEqual(type(volume_mounts), dict) - self.assertEqual(volume_mounts["mount_path"], secret_path) - env_dict = op1.container.env[0].to_dict() - self.assertEqual(env_dict["name"], env_variable) - self.assertEqual(env_dict["value"], - secret_path + secret_file_path_in_volume) diff --git a/sdk/python/tests/dsl/main.py b/sdk/python/tests/dsl/main.py deleted file mode 100644 index eda3d62ac8a..00000000000 --- a/sdk/python/tests/dsl/main.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2018-2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import unittest - -import aws_extensions_tests -import component_bridge_tests -import component_tests -import container_op_tests -import extensions.test_kubernetes as test_kubernetes -import metadata_tests -import ops_group_tests -import pipeline_param_tests -import pipeline_tests -import pipeline_volume_tests -import resource_op_tests -import type_tests -import volume_op_tests -import volume_snapshotop_tests - -if __name__ == '__main__': - suite = unittest.TestSuite() - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(aws_extensions_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(pipeline_param_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(pipeline_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(container_op_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(ops_group_tests)) - suite.addTests(unittest.defaultTestLoader.loadTestsFromModule(type_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(component_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(component_bridge_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(metadata_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(resource_op_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(volume_op_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(pipeline_volume_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(volume_snapshotop_tests)) - suite.addTests( - unittest.defaultTestLoader.loadTestsFromModule(test_kubernetes)) - - runner = unittest.TextTestRunner() - if not runner.run(suite).wasSuccessful(): - sys.exit(1) diff --git a/sdk/python/tests/dsl/metadata_tests.py b/sdk/python/tests/dsl/metadata_tests.py deleted file mode 100644 index f8f14471c8c..00000000000 --- a/sdk/python/tests/dsl/metadata_tests.py +++ /dev/null @@ -1,105 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from kfp.deprecated.components.structures import ComponentSpec -from kfp.deprecated.components.structures import InputSpec -from kfp.deprecated.components.structures import OutputSpec - - -class TestComponentMeta(unittest.TestCase): - - def test_to_dict(self): - component_meta = ComponentSpec( - name='foobar', - description='foobar example', - inputs=[ - InputSpec( - name='input1', - description='input1 desc', - type={ - 'GCSPath': { - 'bucket_type': 'directory', - 'file_type': 'csv' - } - }, - default='default1'), - InputSpec( - name='input2', - description='input2 desc', - type={ - 'TFModel': { - 'input_data': 'tensor', - 'version': '1.8.0' - } - }, - default='default2'), - InputSpec( - name='input3', - description='input3 desc', - type='Integer', - default='default3'), - ], - outputs=[ - OutputSpec( - name='output1', - description='output1 desc', - type={'Schema': { - 'file_type': 'tsv' - }}, - ) - ]) - golden_meta = { - 'name': - 'foobar', - 'description': - 'foobar example', - 'inputs': [{ - 'name': 'input1', - 'description': 'input1 desc', - 'type': { - 'GCSPath': { - 'bucket_type': 'directory', - 'file_type': 'csv' - } - }, - 'default': 'default1' - }, { - 'name': 'input2', - 'description': 'input2 desc', - 'type': { - 'TFModel': { - 'input_data': 'tensor', - 'version': '1.8.0' - } - }, - 'default': 'default2' - }, { - 'name': 'input3', - 'description': 'input3 desc', - 'type': 'Integer', - 'default': 'default3' - }], - 'outputs': [{ - 'name': 'output1', - 'description': 'output1 desc', - 'type': { - 'Schema': { - 'file_type': 'tsv' - } - }, - }] - } - self.assertEqual(component_meta.to_dict(), golden_meta) diff --git a/sdk/python/tests/dsl/ops_group_tests.py b/sdk/python/tests/dsl/ops_group_tests.py deleted file mode 100644 index 702bf4b6100..00000000000 --- a/sdk/python/tests/dsl/ops_group_tests.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from kfp.deprecated.dsl import Condition -from kfp.deprecated.dsl import ContainerOp -from kfp.deprecated.dsl import ExitHandler -from kfp.deprecated.dsl import OpsGroup -from kfp.deprecated.dsl import Pipeline -import kfp.deprecated.dsl as dsl - - -class TestOpsGroup(unittest.TestCase): - - def test_basic(self): - """Test basic usage.""" - with Pipeline('somename') as p: - self.assertEqual(1, len(p.groups)) - with OpsGroup(group_type='exit_handler'): - op1 = ContainerOp(name='op1', image='image') - with OpsGroup(group_type='branch'): - op2 = ContainerOp(name='op2', image='image') - op3 = ContainerOp(name='op3', image='image') - with OpsGroup(group_type='loop'): - op4 = ContainerOp(name='op4', image='image') - - self.assertEqual(1, len(p.groups)) - self.assertEqual(1, len(p.groups[0].groups)) - exit_handler_group = p.groups[0].groups[0] - self.assertEqual('exit_handler', exit_handler_group.type) - self.assertEqual(2, len(exit_handler_group.groups)) - self.assertEqual(1, len(exit_handler_group.ops)) - self.assertEqual('op1', exit_handler_group.ops[0].name) - - branch_group = exit_handler_group.groups[0] - self.assertFalse(branch_group.groups) - self.assertCountEqual([x.name for x in branch_group.ops], - ['op2', 'op3']) - - loop_group = exit_handler_group.groups[1] - self.assertFalse(loop_group.groups) - self.assertCountEqual([x.name for x in loop_group.ops], ['op4']) - - def test_basic_recursive_opsgroups(self): - """Test recursive opsgroups.""" - with Pipeline('somename') as p: - self.assertEqual(1, len(p.groups)) - - # When a graph opsgraph is called. - graph_ops_group_one = dsl._ops_group.Graph('hello') - graph_ops_group_one.__enter__() - self.assertFalse(graph_ops_group_one.recursive_ref) - self.assertEqual('graph-hello-1', graph_ops_group_one.name) - - # Another graph opsgraph is called with the same name - # when the previous graph opsgraphs is not finished. - graph_ops_group_two = dsl._ops_group.Graph('hello') - graph_ops_group_two.__enter__() - self.assertTrue(graph_ops_group_two.recursive_ref) - self.assertEqual(graph_ops_group_one, - graph_ops_group_two.recursive_ref) - - def test_recursive_opsgroups_with_prefix_names(self): - """Test recursive opsgroups.""" - with Pipeline('somename') as p: - self.assertEqual(1, len(p.groups)) - - # When a graph opsgraph is called. - graph_ops_group_one = dsl._ops_group.Graph('foo_bar') - graph_ops_group_one.__enter__() - self.assertFalse(graph_ops_group_one.recursive_ref) - self.assertEqual('graph-foo-bar-1', graph_ops_group_one.name) - - # Another graph opsgraph is called with the name as the prefix of the ops_group_one - # when the previous graph opsgraphs is not finished. - graph_ops_group_two = dsl._ops_group.Graph('foo') - graph_ops_group_two.__enter__() - self.assertFalse(graph_ops_group_two.recursive_ref) - - -class TestExitHandler(unittest.TestCase): - - def test_basic(self): - """Test basic usage.""" - with Pipeline('somename') as p: - exit_op = ContainerOp(name='exit', image='image') - with ExitHandler(exit_op=exit_op): - op1 = ContainerOp(name='op1', image='image') - - exit_handler = p.groups[0].groups[0] - self.assertEqual('exit_handler', exit_handler.type) - self.assertEqual('exit', exit_handler.exit_op.name) - self.assertEqual(1, len(exit_handler.ops)) - self.assertEqual('op1', exit_handler.ops[0].name) - - def test_invalid_exit_op(self): - with self.assertRaises(ValueError): - with Pipeline('somename') as p: - op1 = ContainerOp(name='op1', image='image') - exit_op = ContainerOp(name='exit', image='image') - exit_op.after(op1) - with ExitHandler(exit_op=exit_op): - pass - - -class TestConditionOp(unittest.TestCase): - - def test_basic(self): - with Pipeline('somename') as p: - param1 = 'pizza' - condition1 = Condition(param1 == 'pizza') - self.assertEqual(condition1.name, None) - with condition1: - pass - self.assertEqual(condition1.name, 'condition-1') - - condition2 = Condition(param1 == 'pizza', '[param1 is pizza]') - self.assertEqual(condition2.name, '[param1 is pizza]') - with condition2: - pass - self.assertEqual(condition2.name, 'condition-[param1 is pizza]-2') diff --git a/sdk/python/tests/dsl/pipeline_param_tests.py b/sdk/python/tests/dsl/pipeline_param_tests.py deleted file mode 100644 index 8cdaf1badbf..00000000000 --- a/sdk/python/tests/dsl/pipeline_param_tests.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from kfp.deprecated.dsl import PipelineParam -from kfp.deprecated.dsl._pipeline_param import _extract_pipelineparams -from kfp.deprecated.dsl._pipeline_param import extract_pipelineparams_from_any -from kubernetes.client.models import V1ConfigMap -from kubernetes.client.models import V1Container -from kubernetes.client.models import V1EnvVar - - -class TestPipelineParam(unittest.TestCase): - - def test_invalid(self): - """Invalid pipeline param name and op_name.""" - with self.assertRaises(ValueError): - p = PipelineParam(name='123_abc') - - def test_str_repr(self): - """Test string representation.""" - - p = PipelineParam(name='param1', op_name='op1') - self.assertEqual('{{pipelineparam:op=op1;name=param1}}', str(p)) - - p = PipelineParam(name='param2') - self.assertEqual('{{pipelineparam:op=;name=param2}}', str(p)) - - p = PipelineParam(name='param3', value='value3') - self.assertEqual('{{pipelineparam:op=;name=param3}}', str(p)) - - def test_extract_pipelineparams(self): - """Test _extract_pipeleineparams.""" - - p1 = PipelineParam(name='param1', op_name='op1') - p2 = PipelineParam(name='param2') - p3 = PipelineParam(name='param3', value='value3') - stuff_chars = ' between ' - payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3) - params = _extract_pipelineparams(payload) - self.assertListEqual([p1, p2, p3], params) - payload = [ - str(p1) + stuff_chars + str(p2), - str(p2) + stuff_chars + str(p3) - ] - params = _extract_pipelineparams(payload) - self.assertListEqual([p1, p2, p3], params) - - def test_extract_pipelineparams_from_any(self): - """Test extract_pipeleineparams.""" - p1 = PipelineParam(name='param1', op_name='op1') - p2 = PipelineParam(name='param2') - p3 = PipelineParam(name='param3', value='value3') - stuff_chars = ' between ' - payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3) - - container = V1Container( - name=p1, image=p2, env=[V1EnvVar(name="foo", value=payload)]) - - params = extract_pipelineparams_from_any(container) - self.assertListEqual(sorted([p1, p2, p3]), sorted(params)) - - def test_extract_pipelineparams_from_dict(self): - """Test extract_pipeleineparams.""" - p1 = PipelineParam(name='param1', op_name='op1') - p2 = PipelineParam(name='param2') - - configmap = V1ConfigMap(data={str(p1): str(p2)}) - - params = extract_pipelineparams_from_any(configmap) - self.assertListEqual(sorted([p1, p2]), sorted(params)) - - def test_extract_pipelineparam_with_types(self): - """Test _extract_pipelineparams.""" - p1 = PipelineParam( - name='param1', - op_name='op1', - param_type={'customized_type_a': { - 'property_a': 'value_a' - }}) - p2 = PipelineParam(name='param2', param_type='customized_type_b') - p3 = PipelineParam( - name='param3', - value='value3', - param_type={'customized_type_c': { - 'property_c': 'value_c' - }}) - stuff_chars = ' between ' - payload = str(p1) + stuff_chars + str(p2) + stuff_chars + str(p3) - params = _extract_pipelineparams(payload) - self.assertListEqual([p1, p2, p3], params) - # Expecting the _extract_pipelineparam to dedup the pipelineparams among all the payloads. - payload = [ - str(p1) + stuff_chars + str(p2), - str(p2) + stuff_chars + str(p3) - ] - params = _extract_pipelineparams(payload) - self.assertListEqual([p1, p2, p3], params) diff --git a/sdk/python/tests/dsl/pipeline_tests.py b/sdk/python/tests/dsl/pipeline_tests.py deleted file mode 100644 index 68603d0c4c3..00000000000 --- a/sdk/python/tests/dsl/pipeline_tests.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from kfp.deprecated.components.structures import ComponentSpec -from kfp.deprecated.components.structures import InputSpec -from kfp.deprecated.dsl import ContainerOp -from kfp.deprecated.dsl import Pipeline -from kfp.deprecated.dsl import pipeline -from kfp.deprecated.dsl._metadata import _extract_pipeline_metadata -from kfp.deprecated.dsl.types import Integer - - -class TestPipeline(unittest.TestCase): - - def test_basic(self): - """Test basic usage.""" - with Pipeline('somename') as p: - self.assertTrue(Pipeline.get_default_pipeline() is not None) - op1 = ContainerOp(name='op1', image='image') - op2 = ContainerOp(name='op2', image='image') - - self.assertTrue(Pipeline.get_default_pipeline() is None) - self.assertEqual(p.ops['op1'].name, 'op1') - self.assertEqual(p.ops['op2'].name, 'op2') - - def test_nested_pipelines(self): - """Test nested pipelines.""" - with self.assertRaises(Exception): - with Pipeline('somename1') as p1: - with Pipeline('somename2') as p2: - pass - - def test_decorator(self): - """Test @pipeline decorator.""" - - @pipeline(name='p1', description='description1') - def my_pipeline1(): - pass - - @pipeline(name='p2', description='description2') - def my_pipeline2(): - pass - - self.assertEqual(my_pipeline1._component_human_name, 'p1') - self.assertEqual(my_pipeline2._component_human_name, 'p2') - self.assertEqual(my_pipeline1._component_description, 'description1') - self.assertEqual(my_pipeline2._component_description, 'description2') - - def test_decorator_metadata(self): - """Test @pipeline decorator with metadata.""" - - @pipeline(name='p1', description='description1') - def my_pipeline1( - a: {'Schema': { - 'file_type': 'csv' - }} = 'good', - b: Integer() = 12): - pass - - golden_meta = ComponentSpec( - name='p1', description='description1', inputs=[]) - golden_meta.inputs.append( - InputSpec( - name='a', - type={'Schema': { - 'file_type': 'csv' - }}, - default='good', - optional=True)) - golden_meta.inputs.append( - InputSpec( - name='b', - type={ - 'Integer': { - 'openapi_schema_validator': { - "type": "integer" - } - } - }, - default="12", - optional=True)) - - pipeline_meta = _extract_pipeline_metadata(my_pipeline1) - self.assertEqual(pipeline_meta, golden_meta) diff --git a/sdk/python/tests/dsl/test_azure_extensions.py b/sdk/python/tests/dsl/test_azure_extensions.py deleted file mode 100644 index 734df600d09..00000000000 --- a/sdk/python/tests/dsl/test_azure_extensions.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2019 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -import unittest - -from kfp.deprecated.azure import use_azure_secret -from kfp.deprecated.dsl import ContainerOp - - -class AzExtensionTests(unittest.TestCase): - - def test_default_secret_name(self): - spec = inspect.getfullargspec(use_azure_secret) - assert len(spec.defaults) == 1 - assert spec.defaults[0] == 'azcreds' - - def test_use_azure_secret(self): - op1 = ContainerOp(name='op1', image='image') - op1 = op1.apply(use_azure_secret('foo')) - assert len(op1.container.env) == 4 - - index = 0 - for expected in [ - 'AZ_SUBSCRIPTION_ID', 'AZ_TENANT_ID', 'AZ_CLIENT_ID', - 'AZ_CLIENT_SECRET' - ]: - assert op1.container.env[index].name == expected - assert op1.container.env[ - index].value_from.secret_key_ref.name == 'foo' - assert op1.container.env[ - index].value_from.secret_key_ref.key == expected - index += 1 - - -if __name__ == '__main__': - unittest.main() diff --git a/sdk/python/tests/dsl/type_tests.py b/sdk/python/tests/dsl/type_tests.py deleted file mode 100644 index 8b3dc188562..00000000000 --- a/sdk/python/tests/dsl/type_tests.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from kfp.deprecated.dsl.types import check_types -from kfp.deprecated.dsl.types import GCSPath - - -class TestTypes(unittest.TestCase): - - def test_class_to_dict(self): - """Test _class_to_dict function.""" - gcspath_dict = GCSPath().to_dict() - golden_dict = { - 'GCSPath': { - 'openapi_schema_validator': { - "type": "string", - "pattern": "^gs://.*$" - } - } - } - self.assertEqual(golden_dict, gcspath_dict) - - def test_check_types(self): - #Core types - typeA = {'ArtifactA': {'path_type': 'file', 'file_type': 'csv'}} - typeB = {'ArtifactA': {'path_type': 'file', 'file_type': 'csv'}} - self.assertTrue(check_types(typeA, typeB)) - typeC = {'ArtifactA': {'path_type': 'file', 'file_type': 'tsv'}} - self.assertFalse(check_types(typeA, typeC)) - - # Custom types - typeA = {'A': {'X': 'value1', 'Y': 'value2'}} - typeB = {'B': {'X': 'value1', 'Y': 'value2'}} - typeC = {'A': {'X': 'value1'}} - typeD = {'A': {'X': 'value1', 'Y': 'value3'}} - self.assertFalse(check_types(typeA, typeB)) - self.assertFalse(check_types(typeA, typeC)) - self.assertTrue(check_types(typeC, typeA)) - self.assertFalse(check_types(typeA, typeD)) diff --git a/sdk/python/tests/local_runner_test.py b/sdk/python/tests/local_runner_test.py deleted file mode 100644 index 9522004ed51..00000000000 --- a/sdk/python/tests/local_runner_test.py +++ /dev/null @@ -1,240 +0,0 @@ -# Copyright 2021 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from typing import Callable -import unittest - -from kfp.deprecated import LocalClient -from kfp.deprecated import run_pipeline_func_locally -import kfp.deprecated as kfp - -InputPath = kfp.components.InputPath() -OutputPath = kfp.components.OutputPath() - -BASE_IMAGE = "python:3.9" - - -def light_component(base_image: str = BASE_IMAGE,): - """Decorator of kfp light component with customized parameters. - - Usage: - ```python - @light_component(base_image="python:3.9") - def a_component(src: kfp.components.InputPath(), ...): - ... - ``` - """ - - def wrapper(func: Callable): - return kfp.components.create_component_from_func( - func=func, - base_image=base_image, - ) - - return wrapper - - -@light_component() -def hello(name: str): - print(f"hello {name}") - - -@light_component() -def local_loader(src: str, dst: kfp.components.OutputPath()): - import os - import shutil - - if os.path.exists(src): - shutil.copyfile(src, dst) - - -@light_component() -def flip_coin(dst: kfp.components.OutputPath()): - import random - - result = "head" if random.randint(0, 1) == 0 else "tail" - with open(dst, "w") as f: - f.write(result) - - -@light_component() -def list(dst: kfp.components.OutputPath()): - import json - - with open(dst, "w") as f: - json.dump(["hello", "world", "kfp"], f) - - -@light_component() -def component_connect_demo(src: kfp.components.InputPath(), - dst: kfp.components.OutputPath()): - with open(src, "r") as f: - line = f.readline() - print(f"read first line: {line}") - with open(dst, "w") as fw: - fw.write(f"{line} copied") - - -class LocalRunnerTest(unittest.TestCase): - - def setUp(self): - import tempfile - - with tempfile.NamedTemporaryFile('w', delete=False) as f: - self.temp_file_path = f.name - f.write("hello world") - - def test_run_local(self): - - def _pipeline(name: str): - hello(name) - - run_pipeline_func_locally( - _pipeline, - {"name": "world"}, - execution_mode=LocalClient.ExecutionMode("local"), - ) - - def test_local_file(self): - - def _pipeline(file_path: str): - local_loader(file_path) - - run_result = run_pipeline_func_locally( - _pipeline, - {"file_path": self.temp_file_path}, - execution_mode=LocalClient.ExecutionMode("local"), - ) - output_file_path = run_result.get_output_file("local-loader") - - with open(output_file_path, "r") as f: - line = f.readline() - assert "hello" in line - - def test_condition(self): - - def _pipeline(): - _flip = flip_coin() - with kfp.dsl.Condition(_flip.output == "head"): - hello("head") - - with kfp.dsl.Condition(_flip.output == "tail"): - hello("tail") - - run_pipeline_func_locally( - _pipeline, {}, execution_mode=LocalClient.ExecutionMode("local")) - - def test_for(self): - - @light_component() - def cat(item, dst: OutputPath): - with open(dst, "w") as f: - f.write(item) - - def _pipeline(): - with kfp.dsl.ParallelFor(list().output) as item: - cat(item) - - run_pipeline_func_locally( - _pipeline, {}, execution_mode=LocalClient.ExecutionMode("local")) - - def test_connect(self): - - def _pipeline(): - _local_loader = local_loader(self.temp_file_path) - component_connect_demo(_local_loader.output) - - run_result = run_pipeline_func_locally( - _pipeline, {}, execution_mode=LocalClient.ExecutionMode("local")) - output_file_path = run_result.get_output_file("component-connect-demo") - - with open(output_file_path, "r") as f: - line = f.readline() - assert "copied" in line - - def test_command_argument_in_any_format(self): - - def echo(): - return kfp.dsl.ContainerOp( - name="echo", - image=BASE_IMAGE, - command=[ - "echo", "hello world", ">", "/tmp/outputs/output_file" - ], - arguments=[], - file_outputs={"output": "/tmp/outputs/output_file"}, - ) - - def _pipeline(): - _echo = echo() - component_connect_demo(_echo.output) - - run_pipeline_func_locally( - _pipeline, {}, execution_mode=LocalClient.ExecutionMode("local")) - - @unittest.skip('docker is not installed in CI environment.') - def test_execution_mode_exclude_op(self): - - @light_component(base_image="image_not_exist") - def cat_on_image_not_exist(name: str, dst: OutputPath): - with open(dst, "w") as f: - f.write(name) - - def _pipeline(): - cat_on_image_not_exist("exclude ops") - - run_result = run_pipeline_func_locally( - _pipeline, - {}, - execution_mode=LocalClient.ExecutionMode(mode="docker"), - ) - output_file_path = run_result.get_output_file("cat-on-image-not-exist") - import os - - assert not os.path.exists(output_file_path) - - run_result = run_pipeline_func_locally( - _pipeline, - {}, - execution_mode=LocalClient.ExecutionMode( - mode="docker", ops_to_exclude=["cat-on-image-not-exist"]), - ) - output_file_path = run_result.get_output_file("cat-on-image-not-exist") - - with open(output_file_path, "r") as f: - line = f.readline() - assert "exclude ops" in line - - @unittest.skip('docker is not installed in CI environment.') - def test_docker_options(self): - - @light_component() - def check_option(dst: OutputPath): - import os - with open(dst, "w") as f: - f.write(os.environ["foo"]) - - def _pipeline(): - check_option() - - run_result = run_pipeline_func_locally( - _pipeline, {}, - execution_mode=LocalClient.ExecutionMode( - mode="docker", docker_options=["-e", "foo=bar"])) - assert run_result.success - output_file_path = run_result.get_output_file("check-option") - - with open(output_file_path, "r") as f: - line = f.readline() - assert "bar" in line diff --git a/sdk/python/tests/run_tests.sh b/sdk/python/tests/run_tests.sh deleted file mode 100755 index ddb0845d8ce..00000000000 --- a/sdk/python/tests/run_tests.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -e -# Copyright 2018 The Kubeflow Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cd "$(dirname "$0")/.." -python3 -m unittest discover --verbose --top-level-directory=. -p "*test*.py" diff --git a/sdk/python/tests/test_kfp.py b/sdk/python/tests/test_kfp.py deleted file mode 100644 index a56d3ae182a..00000000000 --- a/sdk/python/tests/test_kfp.py +++ /dev/null @@ -1,8 +0,0 @@ -import unittest - - -class KfpTestCase(unittest.TestCase): - - def test_kfp_version(self): - import kfp - self.assertTrue(len(kfp.__version__) > 0) diff --git a/test/presubmit-isort-sdk.sh b/test/presubmit-isort-sdk.sh index c061a10aea9..5230073045c 100755 --- a/test/presubmit-isort-sdk.sh +++ b/test/presubmit-isort-sdk.sh @@ -19,5 +19,5 @@ python3 -m pip install --upgrade pip python3 -m pip install $(grep 'isort==' sdk/python/requirements-dev.txt) python3 -m pip install $(grep 'pycln==' sdk/python/requirements-dev.txt) -pycln --check "${source_root}/sdk/python" --exclude "${source_root}/sdk/python/kfp/deprecated" --extend-exclude "${source_root}/sdk/python/tests" -isort --check --profile google "${source_root}/sdk/python" --skip "${source_root}/sdk/python/kfp/deprecated" +pycln --check "${source_root}/sdk/python" +isort --check --profile google "${source_root}/sdk/python" diff --git a/test/presubmit-tests-sdk.sh b/test/presubmit-tests-sdk.sh index 894e3d7a306..3d283671d33 100755 --- a/test/presubmit-tests-sdk.sh +++ b/test/presubmit-tests-sdk.sh @@ -30,8 +30,7 @@ python3 -m pip install --upgrade protobuf python3 -m pip install sdk/python -# TODO: remove deprecated dependency; then remove --ignore arg -pytest sdk/python/kfp --ignore=sdk/python/kfp/deprecated --cov=kfp +pytest sdk/python/kfp --cov=kfp set +x # export COVERALLS_REPO_TOKEN=$(gsutil cat gs://ml-pipeline-test-keys/coveralls_repo_token) diff --git a/test/presubmit-yapf-sdk.sh b/test/presubmit-yapf-sdk.sh index 6166843a95e..76bd7cf96e7 100755 --- a/test/presubmit-yapf-sdk.sh +++ b/test/presubmit-yapf-sdk.sh @@ -18,5 +18,5 @@ source_root=$(pwd) python3 -m pip install --upgrade pip python3 -m pip install $(grep 'yapf==' sdk/python/requirements-dev.txt) python3 -m pip install pre_commit_hooks -python3 -m pre_commit_hooks.string_fixer $(find sdk/python/kfp/**/*.py -not -path 'sdk/python/kfp/deprecated/*' -type f) -yapf --recursive --diff "${source_root}/sdk/python/" --exclude "${source_root}/sdk/python/kfp/deprecated/" +python3 -m pre_commit_hooks.string_fixer $(find sdk/python/kfp/**/*.py -type f) +yapf --recursive --diff "${source_root}/sdk/python/"