From 957703f9b9c953ee1f67740a652f68279907b104 Mon Sep 17 00:00:00 2001 From: Matthew Tang Date: Wed, 30 Nov 2022 16:49:02 -0800 Subject: [PATCH] feat: add explanationSpec to TrainingPipeline-based custom jobs PiperOrigin-RevId: 492054553 --- google/cloud/aiplatform/explain/__init__.py | 3 + google/cloud/aiplatform/models.py | 120 +++------- google/cloud/aiplatform/training_jobs.py | 71 +++++- .../aiplatform/utils/_explanation_utils.py | 63 ++++++ tests/unit/aiplatform/test_training_jobs.py | 214 ++++++++++++++++++ 5 files changed, 381 insertions(+), 90 deletions(-) create mode 100644 google/cloud/aiplatform/utils/_explanation_utils.py diff --git a/google/cloud/aiplatform/explain/__init__.py b/google/cloud/aiplatform/explain/__init__.py index 4701d709b5..8167e80a4a 100644 --- a/google/cloud/aiplatform/explain/__init__.py +++ b/google/cloud/aiplatform/explain/__init__.py @@ -35,6 +35,8 @@ ExplanationParameters = explanation_compat.ExplanationParameters FeatureNoiseSigma = explanation_compat.FeatureNoiseSigma +ExplanationSpec = explanation_compat.ExplanationSpec + # Classes used by ExplanationParameters IntegratedGradientsAttribution = explanation_compat.IntegratedGradientsAttribution SampledShapleyAttribution = explanation_compat.SampledShapleyAttribution @@ -44,6 +46,7 @@ __all__ = ( "Encoding", + "ExplanationSpec", "ExplanationMetadata", "ExplanationParameters", "FeatureNoiseSigma", diff --git a/google/cloud/aiplatform/models.py b/google/cloud/aiplatform/models.py index 222f7df1a8..8ee0f400f1 100644 --- a/google/cloud/aiplatform/models.py +++ b/google/cloud/aiplatform/models.py @@ -47,8 +47,8 @@ from google.cloud.aiplatform import models from google.cloud.aiplatform import utils from google.cloud.aiplatform.utils import gcs_utils +from google.cloud.aiplatform.utils import _explanation_utils from google.cloud.aiplatform import model_evaluation - from google.cloud.aiplatform.compat.services import endpoint_service_client from google.cloud.aiplatform.compat.types import ( @@ -617,10 +617,6 @@ def _validate_deploy_args( deployed_model_display_name: Optional[str], traffic_split: Optional[Dict[str, int]], traffic_percentage: Optional[int], - explanation_metadata: Optional[aiplatform.explain.ExplanationMetadata] = None, - explanation_parameters: Optional[ - aiplatform.explain.ExplanationParameters - ] = None, ): """Helper method to validate deploy arguments. @@ -663,20 +659,10 @@ def _validate_deploy_args( not be provided. Traffic of previously deployed models at the endpoint will be scaled down to accommodate new deployed model's traffic. Should not be provided if traffic_split is provided. - explanation_metadata (aiplatform.explain.ExplanationMetadata): - Optional. Metadata describing the Model's input and output for explanation. - `explanation_metadata` is optional while `explanation_parameters` must be - specified when used. - For more details, see `Ref docs ` - explanation_parameters (aiplatform.explain.ExplanationParameters): - Optional. Parameters to configure explaining for Model's predictions. - For more details, see `Ref docs ` Raises: ValueError: if Min or Max replica is negative. Traffic percentage > 100 or < 0. Or if traffic_split does not sum to 100. - ValueError: if explanation_metadata is specified while explanation_parameters - is not. """ if min_replica_count < 0: raise ValueError("Min replica cannot be negative.") @@ -697,11 +683,6 @@ def _validate_deploy_args( "Sum of all traffic within traffic split needs to be 100." ) - if bool(explanation_metadata) and not bool(explanation_parameters): - raise ValueError( - "To get model explanation, `explanation_parameters` must be specified." - ) - # Raises ValueError if invalid accelerator if accelerator_type: utils.validate_accelerator_type(accelerator_type) @@ -817,6 +798,9 @@ def deploy( deployed_model_display_name=deployed_model_display_name, traffic_split=traffic_split, traffic_percentage=traffic_percentage, + ) + + explanation_spec = _explanation_utils.create_and_validate_explanation_spec( explanation_metadata=explanation_metadata, explanation_parameters=explanation_parameters, ) @@ -832,8 +816,7 @@ def deploy( accelerator_type=accelerator_type, accelerator_count=accelerator_count, service_account=service_account, - explanation_metadata=explanation_metadata, - explanation_parameters=explanation_parameters, + explanation_spec=explanation_spec, metadata=metadata, sync=sync, deploy_request_timeout=deploy_request_timeout, @@ -854,10 +837,7 @@ def _deploy( accelerator_type: Optional[str] = None, accelerator_count: Optional[int] = None, service_account: Optional[str] = None, - explanation_metadata: Optional[aiplatform.explain.ExplanationMetadata] = None, - explanation_parameters: Optional[ - aiplatform.explain.ExplanationParameters - ] = None, + explanation_spec: Optional[aiplatform.explain.ExplanationSpec] = None, metadata: Optional[Sequence[Tuple[str, str]]] = (), sync=True, deploy_request_timeout: Optional[float] = None, @@ -919,14 +899,8 @@ def _deploy( to the resource project. Users deploying the Model must have the `iam.serviceAccounts.actAs` permission on this service account. - explanation_metadata (aiplatform.explain.ExplanationMetadata): - Optional. Metadata describing the Model's input and output for explanation. - `explanation_metadata` is optional while `explanation_parameters` must be - specified when used. - For more details, see `Ref docs ` - explanation_parameters (aiplatform.explain.ExplanationParameters): - Optional. Parameters to configure explaining for Model's predictions. - For more details, see `Ref docs ` + explanation_spec (aiplatform.explain.ExplanationSpec): + Optional. Specification of Model explanation. metadata (Sequence[Tuple[str, str]]): Optional. Strings which should be sent along with the request as metadata. @@ -963,8 +937,7 @@ def _deploy( accelerator_type=accelerator_type, accelerator_count=accelerator_count, service_account=service_account, - explanation_metadata=explanation_metadata, - explanation_parameters=explanation_parameters, + explanation_spec=explanation_spec, metadata=metadata, deploy_request_timeout=deploy_request_timeout, autoscaling_target_cpu_utilization=autoscaling_target_cpu_utilization, @@ -992,10 +965,7 @@ def _deploy_call( accelerator_type: Optional[str] = None, accelerator_count: Optional[int] = None, service_account: Optional[str] = None, - explanation_metadata: Optional[aiplatform.explain.ExplanationMetadata] = None, - explanation_parameters: Optional[ - aiplatform.explain.ExplanationParameters - ] = None, + explanation_spec: Optional[aiplatform.explain.ExplanationSpec] = None, metadata: Optional[Sequence[Tuple[str, str]]] = (), deploy_request_timeout: Optional[float] = None, autoscaling_target_cpu_utilization: Optional[int] = None, @@ -1066,14 +1036,8 @@ def _deploy_call( to the resource project. Users deploying the Model must have the `iam.serviceAccounts.actAs` permission on this service account. - explanation_metadata (aiplatform.explain.ExplanationMetadata): - Optional. Metadata describing the Model's input and output for explanation. - `explanation_metadata` is optional while `explanation_parameters` must be - specified when used. - For more details, see `Ref docs ` - explanation_parameters (aiplatform.explain.ExplanationParameters): - Optional. Parameters to configure explaining for Model's predictions. - For more details, see `Ref docs ` + explanation_spec (aiplatform.explain.ExplanationSpec): + Optional. Specification of Model explanation. metadata (Sequence[Tuple[str, str]]): Optional. Strings which should be sent along with the request as metadata. @@ -1199,13 +1163,7 @@ def _deploy_call( "See https://cloud.google.com/vertex-ai/docs/reference/rpc/google.cloud.aiplatform.v1#google.cloud.aiplatform.v1.Model.FIELDS.repeated.google.cloud.aiplatform.v1.Model.DeploymentResourcesType.google.cloud.aiplatform.v1.Model.supported_deployment_resources_types" ) - # Service will throw error if explanation_parameters is not provided - if explanation_parameters: - explanation_spec = gca_endpoint_compat.explanation.ExplanationSpec() - explanation_spec.parameters = explanation_parameters - if explanation_metadata: - explanation_spec.metadata = explanation_metadata - deployed_model.explanation_spec = explanation_spec + deployed_model.explanation_spec = explanation_spec # Checking if traffic percentage is valid # TODO(b/221059294) PrivateEndpoint should support traffic split @@ -2332,6 +2290,9 @@ def deploy( deployed_model_display_name=deployed_model_display_name, traffic_split=None, traffic_percentage=100, + ) + + explanation_spec = _explanation_utils.create_and_validate_explanation_spec( explanation_metadata=explanation_metadata, explanation_parameters=explanation_parameters, ) @@ -2347,8 +2308,7 @@ def deploy( accelerator_type=accelerator_type, accelerator_count=accelerator_count, service_account=service_account, - explanation_metadata=explanation_metadata, - explanation_parameters=explanation_parameters, + explanation_spec=explanation_spec, metadata=metadata, sync=sync, ) @@ -3004,11 +2964,6 @@ def upload( if labels: utils.validate_labels(labels) - if bool(explanation_metadata) and not bool(explanation_parameters): - raise ValueError( - "To get model explanation, `explanation_parameters` must be specified." - ) - appended_user_agent = None if local_model: container_spec = local_model.get_serving_container_spec() @@ -3109,13 +3064,12 @@ def upload( if artifact_uri: managed_model.artifact_uri = artifact_uri - # Override explanation_spec if required field is provided - if explanation_parameters: - explanation_spec = gca_endpoint_compat.explanation.ExplanationSpec() - explanation_spec.parameters = explanation_parameters - if explanation_metadata: - explanation_spec.metadata = explanation_metadata - managed_model.explanation_spec = explanation_spec + managed_model.explanation_spec = ( + _explanation_utils.create_and_validate_explanation_spec( + explanation_metadata=explanation_metadata, + explanation_parameters=explanation_parameters, + ) + ) request = gca_model_service_compat.UploadModelRequest( parent=initializer.global_config.common_location_path(project, location), @@ -3283,8 +3237,6 @@ def deploy( deployed_model_display_name=deployed_model_display_name, traffic_split=traffic_split, traffic_percentage=traffic_percentage, - explanation_metadata=explanation_metadata, - explanation_parameters=explanation_parameters, ) if isinstance(endpoint, PrivateEndpoint): @@ -3295,6 +3247,11 @@ def deploy( "A maximum of one model can be deployed to each private Endpoint." ) + explanation_spec = _explanation_utils.create_and_validate_explanation_spec( + explanation_metadata=explanation_metadata, + explanation_parameters=explanation_parameters, + ) + return self._deploy( endpoint=endpoint, deployed_model_display_name=deployed_model_display_name, @@ -3306,8 +3263,7 @@ def deploy( accelerator_type=accelerator_type, accelerator_count=accelerator_count, service_account=service_account, - explanation_metadata=explanation_metadata, - explanation_parameters=explanation_parameters, + explanation_spec=explanation_spec, metadata=metadata, encryption_spec_key_name=encryption_spec_key_name or initializer.global_config.encryption_spec_key_name, @@ -3331,10 +3287,7 @@ def _deploy( accelerator_type: Optional[str] = None, accelerator_count: Optional[int] = None, service_account: Optional[str] = None, - explanation_metadata: Optional[aiplatform.explain.ExplanationMetadata] = None, - explanation_parameters: Optional[ - aiplatform.explain.ExplanationParameters - ] = None, + explanation_spec: Optional[aiplatform.explain.ExplanationSpec] = None, metadata: Optional[Sequence[Tuple[str, str]]] = (), encryption_spec_key_name: Optional[str] = None, network: Optional[str] = None, @@ -3398,14 +3351,8 @@ def _deploy( to the resource project. Users deploying the Model must have the `iam.serviceAccounts.actAs` permission on this service account. - explanation_metadata (aiplatform.explain.ExplanationMetadata): - Optional. Metadata describing the Model's input and output for explanation. - `explanation_metadata` is optional while `explanation_parameters` must be - specified when used. - For more details, see `Ref docs ` - explanation_parameters (aiplatform.explain.ExplanationParameters): - Optional. Parameters to configure explaining for Model's predictions. - For more details, see `Ref docs ` + explanation_spec (aiplatform.explain.ExplanationSpec): + Optional. Specification of Model explanation. metadata (Sequence[Tuple[str, str]]): Optional. Strings which should be sent along with the request as metadata. @@ -3483,8 +3430,7 @@ def _deploy( accelerator_type=accelerator_type, accelerator_count=accelerator_count, service_account=service_account, - explanation_metadata=explanation_metadata, - explanation_parameters=explanation_parameters, + explanation_spec=explanation_spec, metadata=metadata, deploy_request_timeout=deploy_request_timeout, autoscaling_target_cpu_utilization=autoscaling_target_cpu_utilization, diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 4b394dabdc..e8aa9c0f3d 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -25,6 +25,7 @@ from google.cloud.aiplatform import base from google.cloud.aiplatform.constants import base as constants from google.cloud.aiplatform import datasets +from google.cloud.aiplatform import explain from google.cloud.aiplatform import initializer from google.cloud.aiplatform import models from google.cloud.aiplatform import jobs @@ -32,17 +33,21 @@ from google.cloud.aiplatform import utils from google.cloud.aiplatform.utils import console_utils +from google.cloud.aiplatform.compat.types import env_var as gca_env_var +from google.cloud.aiplatform.compat.types import io as gca_io +from google.cloud.aiplatform.compat.types import model as gca_model from google.cloud.aiplatform.compat.types import ( - env_var as gca_env_var, - io as gca_io, - model as gca_model, pipeline_state as gca_pipeline_state, +) +from google.cloud.aiplatform.compat.types import ( training_pipeline as gca_training_pipeline, ) + from google.cloud.aiplatform.utils import _timestamped_gcs_dir from google.cloud.aiplatform.utils import source_utils from google.cloud.aiplatform.utils import worker_spec_utils from google.cloud.aiplatform.utils import column_transformations_utils +from google.cloud.aiplatform.utils import _explanation_utils from google.cloud.aiplatform.v1.schema.trainingjob import ( definition_v1 as training_job_inputs, @@ -1093,6 +1098,8 @@ def __init__( model_instance_schema_uri: Optional[str] = None, model_parameters_schema_uri: Optional[str] = None, model_prediction_schema_uri: Optional[str] = None, + explanation_metadata: Optional[explain.ExplanationMetadata] = None, + explanation_parameters: Optional[explain.ExplanationParameters] = None, project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, @@ -1194,6 +1201,15 @@ def __init__( and probably different, including the URI scheme, than the one given on input. The output URI will point to a location where the user only has a read access. + explanation_metadata (explain.ExplanationMetadata): + Optional. Metadata describing the Model's input and output for + explanation. `explanation_metadata` is optional while + `explanation_parameters` must be specified when used. + For more details, see `Ref docs ` + explanation_parameters (explain.ExplanationParameters): + Optional. Parameters to configure explaining for Model's + predictions. + For more details, see `Ref docs ` project (str): Project to run training in. Overrides project set in aiplatform.init. location (str): @@ -1312,6 +1328,10 @@ def __init__( "set using aiplatform.init(staging_bucket='gs://my-bucket')" ) + # Save explanationSpec as instance attributes + self._explanation_metadata = explanation_metadata + self._explanation_parameters = explanation_parameters + # Backing Custom Job resource is not known until after data preprocessing # once Custom Job is known we log the console uri and the tensorboard uri # this flags keeps that state so we don't log it multiple times @@ -1439,6 +1459,12 @@ def _prepare_and_validate_run( managed_model.labels = model_labels else: managed_model.labels = self._labels + managed_model.explanation_spec = ( + _explanation_utils.create_and_validate_explanation_spec( + explanation_metadata=self._explanation_metadata, + explanation_parameters=self._explanation_parameters, + ) + ) else: managed_model = None @@ -2608,6 +2634,8 @@ def __init__( model_instance_schema_uri: Optional[str] = None, model_parameters_schema_uri: Optional[str] = None, model_prediction_schema_uri: Optional[str] = None, + explanation_metadata: Optional[explain.ExplanationMetadata] = None, + explanation_parameters: Optional[explain.ExplanationParameters] = None, project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, @@ -2745,6 +2773,15 @@ def __init__( and probably different, including the URI scheme, than the one given on input. The output URI will point to a location where the user only has a read access. + explanation_metadata (explain.ExplanationMetadata): + Optional. Metadata describing the Model's input and output for + explanation. `explanation_metadata` is optional while + `explanation_parameters` must be specified when used. + For more details, see `Ref docs ` + explanation_parameters (explain.ExplanationParameters): + Optional. Parameters to configure explaining for Model's + predictions. + For more details, see `Ref docs ` project (str): Project to run training in. Overrides project set in aiplatform.init. location (str): @@ -2813,6 +2850,8 @@ def __init__( model_serving_container_predict_route=model_serving_container_predict_route, model_serving_container_health_route=model_serving_container_health_route, model_description=model_description, + explanation_metadata=explanation_metadata, + explanation_parameters=explanation_parameters, staging_bucket=staging_bucket, ) @@ -3529,6 +3568,8 @@ def __init__( model_instance_schema_uri: Optional[str] = None, model_parameters_schema_uri: Optional[str] = None, model_prediction_schema_uri: Optional[str] = None, + explanation_metadata: Optional[explain.ExplanationMetadata] = None, + explanation_parameters: Optional[explain.ExplanationParameters] = None, project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, @@ -3665,6 +3706,15 @@ def __init__( and probably different, including the URI scheme, than the one given on input. The output URI will point to a location where the user only has a read access. + explanation_metadata (explain.ExplanationMetadata): + Optional. Metadata describing the Model's input and output for + explanation. `explanation_metadata` is optional while + `explanation_parameters` must be specified when used. + For more details, see `Ref docs ` + explanation_parameters (explain.ExplanationParameters): + Optional. Parameters to configure explaining for Model's + predictions. + For more details, see `Ref docs ` project (str): Project to run training in. Overrides project set in aiplatform.init. location (str): @@ -3733,6 +3783,8 @@ def __init__( model_serving_container_predict_route=model_serving_container_predict_route, model_serving_container_health_route=model_serving_container_health_route, model_description=model_description, + explanation_metadata=explanation_metadata, + explanation_parameters=explanation_parameters, staging_bucket=staging_bucket, ) @@ -5777,6 +5829,8 @@ def __init__( model_instance_schema_uri: Optional[str] = None, model_parameters_schema_uri: Optional[str] = None, model_prediction_schema_uri: Optional[str] = None, + explanation_metadata: Optional[explain.ExplanationMetadata] = None, + explanation_parameters: Optional[explain.ExplanationParameters] = None, project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, @@ -5918,6 +5972,15 @@ def __init__( and probably different, including the URI scheme, than the one given on input. The output URI will point to a location where the user only has a read access. + explanation_metadata (explain.ExplanationMetadata): + Optional. Metadata describing the Model's input and output for + explanation. `explanation_metadata` is optional while + `explanation_parameters` must be specified when used. + For more details, see `Ref docs ` + explanation_parameters (explain.ExplanationParameters): + Optional. Parameters to configure explaining for Model's + predictions. + For more details, see `Ref docs ` project (str): Project to run training in. Overrides project set in aiplatform.init. location (str): @@ -5986,6 +6049,8 @@ def __init__( model_serving_container_predict_route=model_serving_container_predict_route, model_serving_container_health_route=model_serving_container_health_route, model_description=model_description, + explanation_metadata=explanation_metadata, + explanation_parameters=explanation_parameters, staging_bucket=staging_bucket, ) diff --git a/google/cloud/aiplatform/utils/_explanation_utils.py b/google/cloud/aiplatform/utils/_explanation_utils.py new file mode 100644 index 0000000000..6dd87b30e5 --- /dev/null +++ b/google/cloud/aiplatform/utils/_explanation_utils.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional + +from google.cloud.aiplatform import explain +from google.cloud.aiplatform.compat.types import ( + endpoint as gca_endpoint_compat, +) + + +def create_and_validate_explanation_spec( + explanation_metadata: Optional[explain.ExplanationMetadata] = None, + explanation_parameters: Optional[explain.ExplanationParameters] = None, +) -> Optional[explain.ExplanationSpec]: + """Validates the parameters needed to create explanation_spec and creates it. + + Args: + explanation_metadata (explain.ExplanationMetadata): + Optional. Metadata describing the Model's input and output for + explanation. `explanation_metadata` is optional while + `explanation_parameters` must be specified when used. + For more details, see `Ref docs ` + explanation_parameters (explain.ExplanationParameters): + Optional. Parameters to configure explaining for Model's + predictions. + For more details, see `Ref docs ` + + Returns: + explanation_spec: Specification of Model explanation. + + Raises: + ValueError: If `explanation_metadata` is given, but + `explanation_parameters` is omitted. `explanation_metadata` is optional + while `explanation_parameters` must be specified when used. + """ + if bool(explanation_metadata) and not bool(explanation_parameters): + raise ValueError( + "To get model explanation, `explanation_parameters` must be specified." + ) + + if explanation_parameters: + explanation_spec = gca_endpoint_compat.explanation.ExplanationSpec() + explanation_spec.parameters = explanation_parameters + if explanation_metadata: + explanation_spec.metadata = explanation_metadata + return explanation_spec + + return None diff --git a/tests/unit/aiplatform/test_training_jobs.py b/tests/unit/aiplatform/test_training_jobs.py index fdadb2a1b1..326e83983f 100644 --- a/tests/unit/aiplatform/test_training_jobs.py +++ b/tests/unit/aiplatform/test_training_jobs.py @@ -39,6 +39,7 @@ from google.cloud import aiplatform from google.cloud.aiplatform import base from google.cloud.aiplatform import datasets +from google.cloud.aiplatform import explain from google.cloud.aiplatform import initializer from google.cloud.aiplatform import schema from google.cloud.aiplatform import training_jobs @@ -169,6 +170,23 @@ ) _TEST_CREDENTIALS = mock.Mock(spec=auth_credentials.AnonymousCredentials()) + +# Explanation Spec +_TEST_EXPLANATION_METADATA = explain.ExplanationMetadata( + inputs={ + "features": { + "input_tensor_name": "dense_input", + "encoding": "BAG_OF_FEATURES", + "modality": "numeric", + "index_feature_mapping": ["abc", "def", "ghj"], + } + }, + outputs={"medv": {"output_tensor_name": "dense_2"}}, +) +_TEST_EXPLANATION_PARAMETERS = explain.ExplanationParameters( + {"sampled_shapley_attribution": {"path_count": 10}} +) + # CMEK encryption _TEST_DEFAULT_ENCRYPTION_KEY_NAME = "key_default" _TEST_DEFAULT_ENCRYPTION_SPEC = gca_encryption_spec.EncryptionSpec( @@ -923,6 +941,8 @@ def test_run_call_pipeline_service_create_with_tabular_dataset( model_serving_container_environment_variables=_TEST_MODEL_SERVING_CONTAINER_ENVIRONMENT_VARIABLES, model_serving_container_ports=_TEST_MODEL_SERVING_CONTAINER_PORTS, model_description=_TEST_MODEL_DESCRIPTION, + explanation_metadata=_TEST_EXPLANATION_METADATA, + explanation_parameters=_TEST_EXPLANATION_PARAMETERS, ) model_from_job = job.run( @@ -1018,6 +1038,10 @@ def test_run_call_pipeline_service_create_with_tabular_dataset( parameters_schema_uri=_TEST_MODEL_PARAMETERS_SCHEMA_URI, prediction_schema_uri=_TEST_MODEL_PREDICTION_SCHEMA_URI, ), + explanation_spec=gca_model.explanation.ExplanationSpec( + metadata=_TEST_EXPLANATION_METADATA, + parameters=_TEST_EXPLANATION_PARAMETERS, + ), encryption_spec=_TEST_DEFAULT_ENCRYPTION_SPEC, version_aliases=["default"], ) @@ -1075,6 +1099,70 @@ def test_run_call_pipeline_service_create_with_tabular_dataset( assert job._has_logged_custom_job + @mock.patch.object(training_jobs, "_JOB_WAIT_TIME", 1) + @mock.patch.object(training_jobs, "_LOG_WAIT_TIME", 1) + def test_custom_training_job_run_raises_with_impartial_explanation_spec( + self, + mock_pipeline_service_create, + mock_pipeline_service_get, + mock_python_package_to_gcs, + mock_tabular_dataset, + mock_model_service_get, + ): + aiplatform.init( + project=_TEST_PROJECT, + staging_bucket=_TEST_BUCKET_NAME, + credentials=_TEST_CREDENTIALS, + encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME, + ) + + job = training_jobs.CustomTrainingJob( + display_name=_TEST_DISPLAY_NAME, + labels=_TEST_LABELS, + script_path=_TEST_LOCAL_SCRIPT_FILE_NAME, + container_uri=_TEST_TRAINING_CONTAINER_IMAGE, + model_serving_container_image_uri=_TEST_SERVING_CONTAINER_IMAGE, + model_serving_container_predict_route=_TEST_SERVING_CONTAINER_PREDICTION_ROUTE, + model_serving_container_health_route=_TEST_SERVING_CONTAINER_HEALTH_ROUTE, + model_instance_schema_uri=_TEST_MODEL_INSTANCE_SCHEMA_URI, + model_parameters_schema_uri=_TEST_MODEL_PARAMETERS_SCHEMA_URI, + model_prediction_schema_uri=_TEST_MODEL_PREDICTION_SCHEMA_URI, + model_serving_container_command=_TEST_MODEL_SERVING_CONTAINER_COMMAND, + model_serving_container_args=_TEST_MODEL_SERVING_CONTAINER_ARGS, + model_serving_container_environment_variables=_TEST_MODEL_SERVING_CONTAINER_ENVIRONMENT_VARIABLES, + model_serving_container_ports=_TEST_MODEL_SERVING_CONTAINER_PORTS, + model_description=_TEST_MODEL_DESCRIPTION, + explanation_metadata=_TEST_EXPLANATION_METADATA, + # Missing the required explanations_parameters field + ) + + with pytest.raises(ValueError) as e: + job.run( + dataset=mock_tabular_dataset, + base_output_dir=_TEST_BASE_OUTPUT_DIR, + service_account=_TEST_SERVICE_ACCOUNT, + network=_TEST_NETWORK, + args=_TEST_RUN_ARGS, + environment_variables=_TEST_ENVIRONMENT_VARIABLES, + machine_type=_TEST_MACHINE_TYPE, + accelerator_type=_TEST_ACCELERATOR_TYPE, + accelerator_count=_TEST_ACCELERATOR_COUNT, + model_display_name=_TEST_MODEL_DISPLAY_NAME, + model_labels=_TEST_MODEL_LABELS, + training_fraction_split=_TEST_TRAINING_FRACTION_SPLIT, + validation_fraction_split=_TEST_VALIDATION_FRACTION_SPLIT, + test_fraction_split=_TEST_TEST_FRACTION_SPLIT, + timestamp_split_column_name=_TEST_TIMESTAMP_SPLIT_COLUMN_NAME, + tensorboard=_TEST_TENSORBOARD_RESOURCE_NAME, + sync=False, + create_request_timeout=None, + ) + + assert e.match( + regexp=r"To get model explanation, `explanation_parameters` " + "must be specified." + ) + @mock.patch.object(training_jobs, "_JOB_WAIT_TIME", 1) @mock.patch.object(training_jobs, "_LOG_WAIT_TIME", 1) def test_custom_training_tabular_done( @@ -2925,6 +3013,8 @@ def test_run_call_pipeline_service_create_with_tabular_dataset( model_serving_container_environment_variables=_TEST_MODEL_SERVING_CONTAINER_ENVIRONMENT_VARIABLES, model_serving_container_ports=_TEST_MODEL_SERVING_CONTAINER_PORTS, model_description=_TEST_MODEL_DESCRIPTION, + explanation_metadata=_TEST_EXPLANATION_METADATA, + explanation_parameters=_TEST_EXPLANATION_PARAMETERS, ) model_from_job = job.run( @@ -3002,6 +3092,10 @@ def test_run_call_pipeline_service_create_with_tabular_dataset( parameters_schema_uri=_TEST_MODEL_PARAMETERS_SCHEMA_URI, prediction_schema_uri=_TEST_MODEL_PREDICTION_SCHEMA_URI, ), + explanation_spec=gca_model.explanation.ExplanationSpec( + metadata=_TEST_EXPLANATION_METADATA, + parameters=_TEST_EXPLANATION_PARAMETERS, + ), encryption_spec=_TEST_DEFAULT_ENCRYPTION_SPEC, version_aliases=["default"], ) @@ -3060,6 +3154,62 @@ def test_run_call_pipeline_service_create_with_tabular_dataset( assert job._has_logged_custom_job + @mock.patch.object(training_jobs, "_JOB_WAIT_TIME", 1) + @mock.patch.object(training_jobs, "_LOG_WAIT_TIME", 1) + def test_custom_container_training_job_run_raises_with_impartial_explanation_spec( + self, + mock_pipeline_service_create, + mock_pipeline_service_get, + mock_tabular_dataset, + mock_model_service_get, + ): + aiplatform.init( + project=_TEST_PROJECT, + staging_bucket=_TEST_BUCKET_NAME, + encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME, + ) + + job = training_jobs.CustomContainerTrainingJob( + display_name=_TEST_DISPLAY_NAME, + labels=_TEST_LABELS, + container_uri=_TEST_TRAINING_CONTAINER_IMAGE, + command=_TEST_TRAINING_CONTAINER_CMD, + model_serving_container_image_uri=_TEST_SERVING_CONTAINER_IMAGE, + model_serving_container_predict_route=_TEST_SERVING_CONTAINER_PREDICTION_ROUTE, + model_serving_container_health_route=_TEST_SERVING_CONTAINER_HEALTH_ROUTE, + model_instance_schema_uri=_TEST_MODEL_INSTANCE_SCHEMA_URI, + model_parameters_schema_uri=_TEST_MODEL_PARAMETERS_SCHEMA_URI, + model_prediction_schema_uri=_TEST_MODEL_PREDICTION_SCHEMA_URI, + model_serving_container_command=_TEST_MODEL_SERVING_CONTAINER_COMMAND, + model_serving_container_args=_TEST_MODEL_SERVING_CONTAINER_ARGS, + model_serving_container_environment_variables=_TEST_MODEL_SERVING_CONTAINER_ENVIRONMENT_VARIABLES, + model_serving_container_ports=_TEST_MODEL_SERVING_CONTAINER_PORTS, + model_description=_TEST_MODEL_DESCRIPTION, + explanation_metadata=_TEST_EXPLANATION_METADATA, + # Missing the required explanations_parameters field + ) + + with pytest.raises(ValueError) as e: + job.run( + dataset=mock_tabular_dataset, + base_output_dir=_TEST_BASE_OUTPUT_DIR, + args=_TEST_RUN_ARGS, + environment_variables=_TEST_ENVIRONMENT_VARIABLES, + machine_type=_TEST_MACHINE_TYPE, + accelerator_type=_TEST_ACCELERATOR_TYPE, + accelerator_count=_TEST_ACCELERATOR_COUNT, + model_display_name=_TEST_MODEL_DISPLAY_NAME, + model_labels=_TEST_MODEL_LABELS, + predefined_split_column_name=_TEST_PREDEFINED_SPLIT_COLUMN_NAME, + service_account=_TEST_SERVICE_ACCOUNT, + tensorboard=_TEST_TENSORBOARD_RESOURCE_NAME, + create_request_timeout=None, + ) + assert e.match( + regexp=r"To get model explanation, `explanation_parameters` " + "must be specified." + ) + @mock.patch.object(training_jobs, "_JOB_WAIT_TIME", 1) @mock.patch.object(training_jobs, "_LOG_WAIT_TIME", 1) @pytest.mark.parametrize("sync", [True, False]) @@ -4868,6 +5018,8 @@ def test_run_call_pipeline_service_create_with_tabular_dataset( model_instance_schema_uri=_TEST_MODEL_INSTANCE_SCHEMA_URI, model_parameters_schema_uri=_TEST_MODEL_PARAMETERS_SCHEMA_URI, model_prediction_schema_uri=_TEST_MODEL_PREDICTION_SCHEMA_URI, + explanation_metadata=_TEST_EXPLANATION_METADATA, + explanation_parameters=_TEST_EXPLANATION_PARAMETERS, ) model_from_job = job.run( @@ -4955,6 +5107,10 @@ def test_run_call_pipeline_service_create_with_tabular_dataset( prediction_schema_uri=_TEST_MODEL_PREDICTION_SCHEMA_URI, ), encryption_spec=_TEST_DEFAULT_ENCRYPTION_SPEC, + explanation_spec=gca_model.explanation.ExplanationSpec( + metadata=_TEST_EXPLANATION_METADATA, + parameters=_TEST_EXPLANATION_PARAMETERS, + ), version_aliases=["default"], ) @@ -5008,6 +5164,64 @@ def test_run_call_pipeline_service_create_with_tabular_dataset( assert job.state == gca_pipeline_state.PipelineState.PIPELINE_STATE_SUCCEEDED + @mock.patch.object(training_jobs, "_JOB_WAIT_TIME", 1) + @mock.patch.object(training_jobs, "_LOG_WAIT_TIME", 1) + def test_custom_python_package_training_job_run_raises_with_impartial_explanation_spec( + self, + mock_pipeline_service_create, + mock_pipeline_service_get, + mock_tabular_dataset, + mock_model_service_get, + ): + aiplatform.init( + project=_TEST_PROJECT, + staging_bucket=_TEST_BUCKET_NAME, + encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME, + ) + + job = training_jobs.CustomContainerTrainingJob( + display_name=_TEST_DISPLAY_NAME, + labels=_TEST_LABELS, + container_uri=_TEST_TRAINING_CONTAINER_IMAGE, + command=_TEST_TRAINING_CONTAINER_CMD, + model_serving_container_image_uri=_TEST_SERVING_CONTAINER_IMAGE, + model_serving_container_predict_route=_TEST_SERVING_CONTAINER_PREDICTION_ROUTE, + model_serving_container_health_route=_TEST_SERVING_CONTAINER_HEALTH_ROUTE, + model_instance_schema_uri=_TEST_MODEL_INSTANCE_SCHEMA_URI, + model_parameters_schema_uri=_TEST_MODEL_PARAMETERS_SCHEMA_URI, + model_prediction_schema_uri=_TEST_MODEL_PREDICTION_SCHEMA_URI, + model_serving_container_command=_TEST_MODEL_SERVING_CONTAINER_COMMAND, + model_serving_container_args=_TEST_MODEL_SERVING_CONTAINER_ARGS, + model_serving_container_environment_variables=_TEST_MODEL_SERVING_CONTAINER_ENVIRONMENT_VARIABLES, + model_serving_container_ports=_TEST_MODEL_SERVING_CONTAINER_PORTS, + model_description=_TEST_MODEL_DESCRIPTION, + explanation_metadata=_TEST_EXPLANATION_METADATA, + # Missing the required explanations_parameters field + ) + + with pytest.raises(ValueError) as e: + job.run( + dataset=mock_tabular_dataset, + model_display_name=_TEST_MODEL_DISPLAY_NAME, + model_labels=_TEST_MODEL_LABELS, + base_output_dir=_TEST_BASE_OUTPUT_DIR, + service_account=_TEST_SERVICE_ACCOUNT, + network=_TEST_NETWORK, + args=_TEST_RUN_ARGS, + environment_variables=_TEST_ENVIRONMENT_VARIABLES, + machine_type=_TEST_MACHINE_TYPE, + accelerator_type=_TEST_ACCELERATOR_TYPE, + accelerator_count=_TEST_ACCELERATOR_COUNT, + training_fraction_split=_TEST_TRAINING_FRACTION_SPLIT, + validation_fraction_split=_TEST_VALIDATION_FRACTION_SPLIT, + test_fraction_split=_TEST_TEST_FRACTION_SPLIT, + create_request_timeout=None, + ) + assert e.match( + regexp=r"To get model explanation, `explanation_parameters` " + "must be specified." + ) + @mock.patch.object(training_jobs, "_JOB_WAIT_TIME", 1) @mock.patch.object(training_jobs, "_LOG_WAIT_TIME", 1) @pytest.mark.parametrize("sync", [True, False])