From 5376092fb9b79bdfbebe98d256017fc9ad3f6a57 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Thu, 15 Sep 2022 21:48:22 -0700 Subject: [PATCH 01/22] fix: fix endpoint parsing in ModelDeploymentMonitoringJob.update() function --- google/cloud/aiplatform/jobs.py | 2 +- .../aiplatform/test_model_monitoring.py | 75 ++++++++++++++++--- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 203362d7a1..1f9a942e88 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2485,7 +2485,7 @@ def update( current_job.model_deployment_monitoring_objective_configs = ( ModelDeploymentMonitoringJob._parse_configs( objective_configs, - current_job.endpoint, + aiplatform.Endpoint(current_job.endpoint), deployed_model_ids, ) ) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index ab59e9e3de..af09d527b8 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -33,6 +33,26 @@ USER_EMAIL = "" PERMANENT_CHURN_ENDPOINT_ID = "8289570005524152320" CHURN_MODEL_PATH = "gs://mco-mm/churn" +DEFAULT_INPUT = { + "cnt_ad_reward": 0, + "cnt_challenge_a_friend": 0, + "cnt_completed_5_levels": 1, + "cnt_level_complete_quickplay": 3, + "cnt_level_end_quickplay": 5, + "cnt_level_reset_quickplay": 2, + "cnt_level_start_quickplay": 6, + "cnt_post_score": 34, + "cnt_spend_virtual_currency": 0, + "cnt_use_extra_steps": 0, + "cnt_user_engagement": 120, + "country": "Denmark", + "dayofweek": 3, + "julianday": 254, + "language": "da-dk", + "month": 9, + "operating_system": "IOS", + "user_pseudo_id": "104B0770BAE16E8B53DF330C95881893", +} JOB_NAME = "churn" @@ -156,22 +176,57 @@ def test_mdm_two_models_one_valid_config(self): gca_obj_config.prediction_drift_detection_config == drift_config.as_proto() ) + # delete this job and re-configure it to only enable drift detection for faster testing + job.delete() job_resource = job._gca_resource.name - # test job update and delete() - timeout = time.time() + 3600 - new_obj_config = model_monitoring.ObjectiveConfig(skew_config) + # test job delete + with pytest.raises(core_exceptions.NotFound): + job.api_client.get_model_deployment_monitoring_job(name=job_resource) + + new_job = aiplatform.ModelDeploymentMonitoringJob.create( + display_name=self._make_display_name(key=JOB_NAME), + logging_sampling_strategy=sampling_strategy, + schedule_config=schedule_config, + alert_config=alert_config, + objective_configs=model_monitoring.ObjectiveConfig(drift_detection_config = drift_config), + create_request_timeout=3600, + project=e2e_base._PROJECT, + location=e2e_base._LOCATION, + endpoint=self.endpoint, + predict_instance_schema_uri="", + analysis_instance_schema_uri="", + ) + assert new_job is not None + + # generate traffic to force MDM job to come online + for i in range(1100): + DEFAULT_INPUT['cnt_user_engagement'] += i + self.endpoint.predict([DEFAULT_INPUT], use_raw_predict = True) + + # test job update + new_obj_config = model_monitoring.ObjectiveConfig(skew_detection_config = skew_config) + + gca_obj_config = new_job._gca_resource.model_deployment_monitoring_objective_configs[ + 0 + ].objective_config - while time.time() < timeout: - if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: - job.update(objective_configs=new_obj_config) - assert str(job._gca_resource.prediction_drift_detection_config) == "" + while new_job.state != gca_job_state.JobState.JOB_STATE_RUNNING: + time.sleep(1) + if new_job.state == gca_job_state.JobState.JOB_STATE_RUNNING: + new_job.update(objective_configs=new_obj_config) + assert str(gca_obj_config.prediction_drift_detection_config) == "" + assert gca_obj_config.training_prediction_skew_detection_config == skew_config.as_proto() + break + new_job.pause() + while new_job.state != gca_job_state.JobState.JOB_STATE_PAUSED: + if new_job.state == gca_job_state.JobState.JOB_STATE_PAUSED: break - time.sleep(5) + new_job.delete() - job.delete() + # confirm deletion with pytest.raises(core_exceptions.NotFound): - job.api_client.get_model_deployment_monitoring_job(name=job_resource) + new_job.state def test_mdm_two_models_two_valid_configs(self): [deployed_model1, deployed_model2] = list( From 5fe5ae5ccc58a547060cbe04539be963930d5828 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 16 Sep 2022 04:53:06 +0000 Subject: [PATCH 02/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../aiplatform/test_model_monitoring.py | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index af09d527b8..f9517f0146 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -183,13 +183,15 @@ def test_mdm_two_models_one_valid_config(self): # test job delete with pytest.raises(core_exceptions.NotFound): job.api_client.get_model_deployment_monitoring_job(name=job_resource) - + new_job = aiplatform.ModelDeploymentMonitoringJob.create( display_name=self._make_display_name(key=JOB_NAME), logging_sampling_strategy=sampling_strategy, schedule_config=schedule_config, alert_config=alert_config, - objective_configs=model_monitoring.ObjectiveConfig(drift_detection_config = drift_config), + objective_configs=model_monitoring.ObjectiveConfig( + drift_detection_config=drift_config + ), create_request_timeout=3600, project=e2e_base._PROJECT, location=e2e_base._LOCATION, @@ -198,25 +200,32 @@ def test_mdm_two_models_one_valid_config(self): analysis_instance_schema_uri="", ) assert new_job is not None - + # generate traffic to force MDM job to come online for i in range(1100): - DEFAULT_INPUT['cnt_user_engagement'] += i - self.endpoint.predict([DEFAULT_INPUT], use_raw_predict = True) + DEFAULT_INPUT["cnt_user_engagement"] += i + self.endpoint.predict([DEFAULT_INPUT], use_raw_predict=True) # test job update - new_obj_config = model_monitoring.ObjectiveConfig(skew_detection_config = skew_config) - - gca_obj_config = new_job._gca_resource.model_deployment_monitoring_objective_configs[ - 0 - ].objective_config + new_obj_config = model_monitoring.ObjectiveConfig( + skew_detection_config=skew_config + ) + + gca_obj_config = ( + new_job._gca_resource.model_deployment_monitoring_objective_configs[ + 0 + ].objective_config + ) while new_job.state != gca_job_state.JobState.JOB_STATE_RUNNING: time.sleep(1) if new_job.state == gca_job_state.JobState.JOB_STATE_RUNNING: new_job.update(objective_configs=new_obj_config) assert str(gca_obj_config.prediction_drift_detection_config) == "" - assert gca_obj_config.training_prediction_skew_detection_config == skew_config.as_proto() + assert ( + gca_obj_config.training_prediction_skew_detection_config + == skew_config.as_proto() + ) break new_job.pause() while new_job.state != gca_job_state.JobState.JOB_STATE_PAUSED: From 96ea4925efa1549bee6a7adb1de5cf2a2bf82b10 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Fri, 16 Sep 2022 17:03:03 -0700 Subject: [PATCH 03/22] addressed PR feedback --- .../aiplatform/test_model_monitoring.py | 30 ++++--- tests/unit/aiplatform/test_jobs.py | 83 +++++++++++++++++++ 2 files changed, 101 insertions(+), 12 deletions(-) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index f9517f0146..9b9d526db2 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -127,6 +127,7 @@ def test_mdm_two_models_one_valid_config(self): Enable model monitoring on two existing models deployed to the same endpoint. """ # test model monitoring configurations + job = None job = aiplatform.ModelDeploymentMonitoringJob.create( display_name=self._make_display_name(key=JOB_NAME), logging_sampling_strategy=sampling_strategy, @@ -183,8 +184,11 @@ def test_mdm_two_models_one_valid_config(self): # test job delete with pytest.raises(core_exceptions.NotFound): job.api_client.get_model_deployment_monitoring_job(name=job_resource) - - new_job = aiplatform.ModelDeploymentMonitoringJob.create( + + def test_mdm_pause_and_update_config(self): + """Test objective config updates for existing MDM job""" + job = None + job = aiplatform.ModelDeploymentMonitoringJob.create( display_name=self._make_display_name(key=JOB_NAME), logging_sampling_strategy=sampling_strategy, schedule_config=schedule_config, @@ -199,7 +203,7 @@ def test_mdm_two_models_one_valid_config(self): predict_instance_schema_uri="", analysis_instance_schema_uri="", ) - assert new_job is not None + assert job is not None # generate traffic to force MDM job to come online for i in range(1100): @@ -212,30 +216,32 @@ def test_mdm_two_models_one_valid_config(self): ) gca_obj_config = ( - new_job._gca_resource.model_deployment_monitoring_objective_configs[ + job._gca_resource.model_deployment_monitoring_objective_configs[ 0 ].objective_config ) - while new_job.state != gca_job_state.JobState.JOB_STATE_RUNNING: + while job.state != gca_job_state.JobState.JOB_STATE_RUNNING: time.sleep(1) - if new_job.state == gca_job_state.JobState.JOB_STATE_RUNNING: - new_job.update(objective_configs=new_obj_config) + if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: + job.update(objective_configs=new_obj_config) assert str(gca_obj_config.prediction_drift_detection_config) == "" assert ( gca_obj_config.training_prediction_skew_detection_config == skew_config.as_proto() ) break - new_job.pause() - while new_job.state != gca_job_state.JobState.JOB_STATE_PAUSED: - if new_job.state == gca_job_state.JobState.JOB_STATE_PAUSED: + # test pause + job.pause() + while job.state != gca_job_state.JobState.JOB_STATE_PAUSED: + time.sleep(1) + if job.state == gca_job_state.JobState.JOB_STATE_PAUSED: break - new_job.delete() + job.delete() # confirm deletion with pytest.raises(core_exceptions.NotFound): - new_job.state + job.state def test_mdm_two_models_two_valid_configs(self): [deployed_model1, deployed_model2] = list( diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 7381481b30..ad9a4ade89 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -38,11 +38,14 @@ job_state as gca_job_state_compat, machine_resources as gca_machine_resources_compat, manual_batch_tuning_parameters as gca_manual_batch_tuning_parameters_compat, + model_deployment_monitoring_job as gca_model_deployment_monitoring_job_compat, + endpoint as gca_endpoint_compat ) from google.cloud.aiplatform.compat.services import ( job_service_client, ) +from google.protobuf import field_mask_pb2 # type: ignore _TEST_API_CLIENT = job_service_client.JobServiceClient @@ -84,6 +87,9 @@ f"bq://{_TEST_BATCH_PREDICTION_BQ_PREFIX}" ) +_TEST_MDM_JOB_NAME = f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/modelDeploymentMonitoringJobs/{_TEST_ID}" +_TEST_ENDPOINT = f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/endpoints/{_TEST_ID}" + _TEST_JOB_STATE_SUCCESS = gca_job_state_compat.JobState(4) _TEST_JOB_STATE_RUNNING = gca_job_state_compat.JobState(3) _TEST_JOB_STATE_PENDING = gca_job_state_compat.JobState(2) @@ -969,3 +975,80 @@ def test_batch_predict_job_with_versioned_model( ].model == _TEST_VERSIONED_MODEL_NAME ) + +@pytest.fixture +def create_mdm_job_mock(): + with mock.patch.object( + _TEST_API_CLIENT, "create_model_deployment_monitoring_job" + ) as create_mdm_job_mock: + create_mdm_job_mock.return_value = ( + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( + name=_TEST_MDM_JOB_NAME, + display_name=_TEST_DISPLAY_NAME, + state=_TEST_JOB_STATE_RUNNING, + ) + ) + yield create_mdm_job_mock + +@pytest.fixture +def get_mdm_job_mock(): + with mock.patch.object( + _TEST_API_CLIENT, "get_model_deployment_monitoring_job" + ) as get_mdm_job_mock: + get_mdm_job_mock.return_value = gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( + name = _TEST_MDM_JOB_NAME, + display_name = _TEST_DISPLAY_NAME, + state = _TEST_JOB_STATE_RUNNING, + endpoint = _TEST_ENDPOINT + ) + yield get_mdm_job_mock + +@pytest.fixture +def mock_deployed_model(): + with mock.patch.object(gca_endpoint_compat, "DeployedModel") as mock_deployed_model: + mock_deployed_model.id = _TEST_ID + mock_deployed_model.explanation_spec.parameters = "TEST_VALUE" + yield mock_deployed_model + + +@pytest.fixture +def endpoint_list_models_mock(mock_deployed_model): + with mock.patch.object( + aiplatform.Endpoint, "list_models" + ) as endpoint_list_models_mock: + endpoint_list_models_mock.return_value = [mock_deployed_model] + yield endpoint_list_models_mock + +@pytest.fixture +def update_mdm_job_mock(get_mdm_job_mock, endpoint_list_models_mock): + with mock.patch.object( + job_service_client.JobServiceClient,"update_model_deployment_monitoring_job" + ) as update_mdm_job_mock: + mock_client_call = mock.Mock( + spec=job_service_client.JobServiceClient.update_model_deployment_monitoring_job + ) + update_mdm_job_mock.return_value = mock_client_call( + model_deployment_monitoring_job = get_mdm_job_mock, + update_mask = field_mask_pb2.FieldMask(paths=["model_deployment_monitoring_objective_configs"]) + ) + yield update_mdm_job_mock + +@pytest.mark.usefixtures("google_auth_mock") +class TestModelDeploymentMonitoringJob: + def setup_method(self): + reload(initializer) + reload(aiplatform) + + def teardown_method(self): + initializer.global_pool.shutdown(wait=True) + + @pytest.mark.usefixtures("create_mdm_job_mock", "get_mdm_job_mock", "update_mdm_job_mock") + def test_update_mdm_job(self): + job = jobs.ModelDeploymentMonitoringJob( + model_deployment_monitoring_job_name = _TEST_MDM_JOB_NAME + ) + new_config = aiplatform.model_monitoring.ObjectiveConfig( + explanation_config = aiplatform.model_monitoring.ExplanationConfig() + ) + job.update(objective_configs = new_config) + assert(job._gca_resource.model_deployment_monitoring_objective_configs[0].objective_config.explanation_config.enable_feature_attributes) From c51a903969ba77638a96d8609ce87f06a8acdc5d Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Sat, 17 Sep 2022 00:05:34 +0000 Subject: [PATCH 04/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../aiplatform/test_model_monitoring.py | 2 +- tests/unit/aiplatform/test_jobs.py | 45 ++++++++++++------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index 9b9d526db2..77958fb427 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -184,7 +184,7 @@ def test_mdm_two_models_one_valid_config(self): # test job delete with pytest.raises(core_exceptions.NotFound): job.api_client.get_model_deployment_monitoring_job(name=job_resource) - + def test_mdm_pause_and_update_config(self): """Test objective config updates for existing MDM job""" job = None diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index ad9a4ade89..43dda023c9 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -39,7 +39,7 @@ machine_resources as gca_machine_resources_compat, manual_batch_tuning_parameters as gca_manual_batch_tuning_parameters_compat, model_deployment_monitoring_job as gca_model_deployment_monitoring_job_compat, - endpoint as gca_endpoint_compat + endpoint as gca_endpoint_compat, ) from google.cloud.aiplatform.compat.services import ( @@ -88,7 +88,9 @@ ) _TEST_MDM_JOB_NAME = f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/modelDeploymentMonitoringJobs/{_TEST_ID}" -_TEST_ENDPOINT = f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/endpoints/{_TEST_ID}" +_TEST_ENDPOINT = ( + f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/endpoints/{_TEST_ID}" +) _TEST_JOB_STATE_SUCCESS = gca_job_state_compat.JobState(4) _TEST_JOB_STATE_RUNNING = gca_job_state_compat.JobState(3) @@ -976,6 +978,7 @@ def test_batch_predict_job_with_versioned_model( == _TEST_VERSIONED_MODEL_NAME ) + @pytest.fixture def create_mdm_job_mock(): with mock.patch.object( @@ -990,19 +993,23 @@ def create_mdm_job_mock(): ) yield create_mdm_job_mock + @pytest.fixture def get_mdm_job_mock(): with mock.patch.object( _TEST_API_CLIENT, "get_model_deployment_monitoring_job" ) as get_mdm_job_mock: - get_mdm_job_mock.return_value = gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( - name = _TEST_MDM_JOB_NAME, - display_name = _TEST_DISPLAY_NAME, - state = _TEST_JOB_STATE_RUNNING, - endpoint = _TEST_ENDPOINT + get_mdm_job_mock.return_value = ( + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( + name=_TEST_MDM_JOB_NAME, + display_name=_TEST_DISPLAY_NAME, + state=_TEST_JOB_STATE_RUNNING, + endpoint=_TEST_ENDPOINT, + ) ) yield get_mdm_job_mock + @pytest.fixture def mock_deployed_model(): with mock.patch.object(gca_endpoint_compat, "DeployedModel") as mock_deployed_model: @@ -1019,20 +1026,24 @@ def endpoint_list_models_mock(mock_deployed_model): endpoint_list_models_mock.return_value = [mock_deployed_model] yield endpoint_list_models_mock + @pytest.fixture def update_mdm_job_mock(get_mdm_job_mock, endpoint_list_models_mock): with mock.patch.object( - job_service_client.JobServiceClient,"update_model_deployment_monitoring_job" + job_service_client.JobServiceClient, "update_model_deployment_monitoring_job" ) as update_mdm_job_mock: mock_client_call = mock.Mock( spec=job_service_client.JobServiceClient.update_model_deployment_monitoring_job ) update_mdm_job_mock.return_value = mock_client_call( - model_deployment_monitoring_job = get_mdm_job_mock, - update_mask = field_mask_pb2.FieldMask(paths=["model_deployment_monitoring_objective_configs"]) + model_deployment_monitoring_job=get_mdm_job_mock, + update_mask=field_mask_pb2.FieldMask( + paths=["model_deployment_monitoring_objective_configs"] + ), ) yield update_mdm_job_mock + @pytest.mark.usefixtures("google_auth_mock") class TestModelDeploymentMonitoringJob: def setup_method(self): @@ -1042,13 +1053,17 @@ def setup_method(self): def teardown_method(self): initializer.global_pool.shutdown(wait=True) - @pytest.mark.usefixtures("create_mdm_job_mock", "get_mdm_job_mock", "update_mdm_job_mock") + @pytest.mark.usefixtures( + "create_mdm_job_mock", "get_mdm_job_mock", "update_mdm_job_mock" + ) def test_update_mdm_job(self): job = jobs.ModelDeploymentMonitoringJob( - model_deployment_monitoring_job_name = _TEST_MDM_JOB_NAME + model_deployment_monitoring_job_name=_TEST_MDM_JOB_NAME ) new_config = aiplatform.model_monitoring.ObjectiveConfig( - explanation_config = aiplatform.model_monitoring.ExplanationConfig() + explanation_config=aiplatform.model_monitoring.ExplanationConfig() ) - job.update(objective_configs = new_config) - assert(job._gca_resource.model_deployment_monitoring_objective_configs[0].objective_config.explanation_config.enable_feature_attributes) + job.update(objective_configs=new_config) + assert job._gca_resource.model_deployment_monitoring_objective_configs[ + 0 + ].objective_config.explanation_config.enable_feature_attributes From 6a12b67cf3351767dc2f75d6e44c62a9b8ae8ac1 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Sat, 17 Sep 2022 00:07:43 +0000 Subject: [PATCH 05/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- .../aiplatform/test_model_monitoring.py | 2 +- tests/unit/aiplatform/test_jobs.py | 45 ++++++++++++------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index 9b9d526db2..77958fb427 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -184,7 +184,7 @@ def test_mdm_two_models_one_valid_config(self): # test job delete with pytest.raises(core_exceptions.NotFound): job.api_client.get_model_deployment_monitoring_job(name=job_resource) - + def test_mdm_pause_and_update_config(self): """Test objective config updates for existing MDM job""" job = None diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index ad9a4ade89..43dda023c9 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -39,7 +39,7 @@ machine_resources as gca_machine_resources_compat, manual_batch_tuning_parameters as gca_manual_batch_tuning_parameters_compat, model_deployment_monitoring_job as gca_model_deployment_monitoring_job_compat, - endpoint as gca_endpoint_compat + endpoint as gca_endpoint_compat, ) from google.cloud.aiplatform.compat.services import ( @@ -88,7 +88,9 @@ ) _TEST_MDM_JOB_NAME = f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/modelDeploymentMonitoringJobs/{_TEST_ID}" -_TEST_ENDPOINT = f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/endpoints/{_TEST_ID}" +_TEST_ENDPOINT = ( + f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}/endpoints/{_TEST_ID}" +) _TEST_JOB_STATE_SUCCESS = gca_job_state_compat.JobState(4) _TEST_JOB_STATE_RUNNING = gca_job_state_compat.JobState(3) @@ -976,6 +978,7 @@ def test_batch_predict_job_with_versioned_model( == _TEST_VERSIONED_MODEL_NAME ) + @pytest.fixture def create_mdm_job_mock(): with mock.patch.object( @@ -990,19 +993,23 @@ def create_mdm_job_mock(): ) yield create_mdm_job_mock + @pytest.fixture def get_mdm_job_mock(): with mock.patch.object( _TEST_API_CLIENT, "get_model_deployment_monitoring_job" ) as get_mdm_job_mock: - get_mdm_job_mock.return_value = gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( - name = _TEST_MDM_JOB_NAME, - display_name = _TEST_DISPLAY_NAME, - state = _TEST_JOB_STATE_RUNNING, - endpoint = _TEST_ENDPOINT + get_mdm_job_mock.return_value = ( + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( + name=_TEST_MDM_JOB_NAME, + display_name=_TEST_DISPLAY_NAME, + state=_TEST_JOB_STATE_RUNNING, + endpoint=_TEST_ENDPOINT, + ) ) yield get_mdm_job_mock + @pytest.fixture def mock_deployed_model(): with mock.patch.object(gca_endpoint_compat, "DeployedModel") as mock_deployed_model: @@ -1019,20 +1026,24 @@ def endpoint_list_models_mock(mock_deployed_model): endpoint_list_models_mock.return_value = [mock_deployed_model] yield endpoint_list_models_mock + @pytest.fixture def update_mdm_job_mock(get_mdm_job_mock, endpoint_list_models_mock): with mock.patch.object( - job_service_client.JobServiceClient,"update_model_deployment_monitoring_job" + job_service_client.JobServiceClient, "update_model_deployment_monitoring_job" ) as update_mdm_job_mock: mock_client_call = mock.Mock( spec=job_service_client.JobServiceClient.update_model_deployment_monitoring_job ) update_mdm_job_mock.return_value = mock_client_call( - model_deployment_monitoring_job = get_mdm_job_mock, - update_mask = field_mask_pb2.FieldMask(paths=["model_deployment_monitoring_objective_configs"]) + model_deployment_monitoring_job=get_mdm_job_mock, + update_mask=field_mask_pb2.FieldMask( + paths=["model_deployment_monitoring_objective_configs"] + ), ) yield update_mdm_job_mock + @pytest.mark.usefixtures("google_auth_mock") class TestModelDeploymentMonitoringJob: def setup_method(self): @@ -1042,13 +1053,17 @@ def setup_method(self): def teardown_method(self): initializer.global_pool.shutdown(wait=True) - @pytest.mark.usefixtures("create_mdm_job_mock", "get_mdm_job_mock", "update_mdm_job_mock") + @pytest.mark.usefixtures( + "create_mdm_job_mock", "get_mdm_job_mock", "update_mdm_job_mock" + ) def test_update_mdm_job(self): job = jobs.ModelDeploymentMonitoringJob( - model_deployment_monitoring_job_name = _TEST_MDM_JOB_NAME + model_deployment_monitoring_job_name=_TEST_MDM_JOB_NAME ) new_config = aiplatform.model_monitoring.ObjectiveConfig( - explanation_config = aiplatform.model_monitoring.ExplanationConfig() + explanation_config=aiplatform.model_monitoring.ExplanationConfig() ) - job.update(objective_configs = new_config) - assert(job._gca_resource.model_deployment_monitoring_objective_configs[0].objective_config.explanation_config.enable_feature_attributes) + job.update(objective_configs=new_config) + assert job._gca_resource.model_deployment_monitoring_objective_configs[ + 0 + ].objective_config.explanation_config.enable_feature_attributes From d05fa0854238de4fa2b6a5964e57d034eebcd349 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 20 Sep 2022 12:19:08 -0700 Subject: [PATCH 06/22] addressed PR comments --- google/cloud/aiplatform/jobs.py | 2 +- tests/system/aiplatform/test_model_monitoring.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 1f9a942e88..dee76f1c89 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2485,7 +2485,7 @@ def update( current_job.model_deployment_monitoring_objective_configs = ( ModelDeploymentMonitoringJob._parse_configs( objective_configs, - aiplatform.Endpoint(current_job.endpoint), + aiplatform.Endpoint(current_job.endpoint, credentials=self.credentials), deployed_model_ids, ) ) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index 77958fb427..7db531f235 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -141,7 +141,6 @@ def test_mdm_two_models_one_valid_config(self): predict_instance_schema_uri="", analysis_instance_schema_uri="", ) - assert job is not None gapic_job = job._gca_resource assert ( @@ -203,7 +202,6 @@ def test_mdm_pause_and_update_config(self): predict_instance_schema_uri="", analysis_instance_schema_uri="", ) - assert job is not None # generate traffic to force MDM job to come online for i in range(1100): @@ -265,7 +263,6 @@ def test_mdm_two_models_two_valid_configs(self): predict_instance_schema_uri="", analysis_instance_schema_uri="", ) - assert job is not None gapic_job = job._gca_resource assert ( From d57d9f40a54b2b3b640690ef776a5cc0a9df3372 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 20 Sep 2022 19:22:03 +0000 Subject: [PATCH 07/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/aiplatform/jobs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index dee76f1c89..acebf5eab3 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2485,7 +2485,9 @@ def update( current_job.model_deployment_monitoring_objective_configs = ( ModelDeploymentMonitoringJob._parse_configs( objective_configs, - aiplatform.Endpoint(current_job.endpoint, credentials=self.credentials), + aiplatform.Endpoint( + current_job.endpoint, credentials=self.credentials + ), deployed_model_ids, ) ) From b5e8b8ccc41470517e39ae5c4713d68c4b440d6b Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Wed, 21 Sep 2022 17:10:06 -0700 Subject: [PATCH 08/22] addressed more PR feedback --- google/cloud/aiplatform/jobs.py | 23 +++++----- .../aiplatform/test_model_monitoring.py | 45 +++++++++++-------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index acebf5eab3..5defd73547 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2491,27 +2491,24 @@ def update( deployed_model_ids, ) ) - if self.state == gca_job_state.JobState.JOB_STATE_RUNNING: - self.api_client.update_model_deployment_monitoring_job( - model_deployment_monitoring_job=current_job, - update_mask=field_mask_pb2.FieldMask(paths=update_mask), - ) + self.api_client.update_model_deployment_monitoring_job( + model_deployment_monitoring_job=current_job, + update_mask=field_mask_pb2.FieldMask(paths=update_mask), + ) return self def pause(self) -> "ModelDeploymentMonitoringJob": """Pause a running MDM job.""" - if self.state == gca_job_state.JobState.JOB_STATE_RUNNING: - self.api_client.pause_model_deployment_monitoring_job( - name=self._gca_resource.name - ) + self.api_client.pause_model_deployment_monitoring_job( + name=self._gca_resource.name + ) return self def resume(self) -> "ModelDeploymentMonitoringJob": """Resumes a paused MDM job.""" - if self.state == gca_job_state.JobState.JOB_STATE_PAUSED: - self.api_client.resume_model_deployment_monitoring_job( - name=self._gca_resource.name - ) + self.api_client.resume_model_deployment_monitoring_job( + name=self._gca_resource.name + ) return self def delete(self) -> None: diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index 7db531f235..bf4556ea82 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -31,7 +31,7 @@ # constants used for testing USER_EMAIL = "" -PERMANENT_CHURN_ENDPOINT_ID = "8289570005524152320" +PERMANENT_CHURN_ENDPOINT_ID = "1843089351408353280" CHURN_MODEL_PATH = "gs://mco-mm/churn" DEFAULT_INPUT = { "cnt_ad_reward": 0, @@ -202,33 +202,42 @@ def test_mdm_pause_and_update_config(self): predict_instance_schema_uri="", analysis_instance_schema_uri="", ) + # test unsuccessful job update when it's pending + DRIFT_THRESHOLDS["cnt_user_engagement"] += 0.01 + new_obj_config = model_monitoring.ObjectiveConfig( + drift_detection_config = model_monitoring.DriftDetectionConfig( + drift_thresholds=DRIFT_THRESHOLDS, + attribute_drift_thresholds=ATTRIB_DRIFT_THRESHOLDS, + ) + ) + if job.state == gca_job_state.JobState.JOB_STATE_PENDING: + with pytest.raises(core_exceptions.FailedPrecondition): + job.update(objective_configs=new_obj_config) # generate traffic to force MDM job to come online - for i in range(1100): + for i in range(2000): DEFAULT_INPUT["cnt_user_engagement"] += i self.endpoint.predict([DEFAULT_INPUT], use_raw_predict=True) - + # test job update - new_obj_config = model_monitoring.ObjectiveConfig( - skew_detection_config=skew_config - ) - - gca_obj_config = ( - job._gca_resource.model_deployment_monitoring_objective_configs[ - 0 - ].objective_config - ) - - while job.state != gca_job_state.JobState.JOB_STATE_RUNNING: + while True: time.sleep(1) if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: job.update(objective_configs=new_obj_config) - assert str(gca_obj_config.prediction_drift_detection_config) == "" - assert ( - gca_obj_config.training_prediction_skew_detection_config - == skew_config.as_proto() + break + + # verify job update + while True: + time.sleep(1) + if job.state == gca_job_state.JobState.JOB_STATE_RUNNING: + gca_obj_config = ( + job._gca_resource.model_deployment_monitoring_objective_configs[ + 0 + ].objective_config ) + assert gca_obj_config.prediction_drift_detection_config == new_obj_config.drift_detection_config.as_proto() break + # test pause job.pause() while job.state != gca_job_state.JobState.JOB_STATE_PAUSED: From 56a55da4de2ef6863a6751162299dcf0834406de Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 22 Sep 2022 00:12:59 +0000 Subject: [PATCH 09/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/system/aiplatform/test_model_monitoring.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index bf4556ea82..ff970af9ca 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -205,7 +205,7 @@ def test_mdm_pause_and_update_config(self): # test unsuccessful job update when it's pending DRIFT_THRESHOLDS["cnt_user_engagement"] += 0.01 new_obj_config = model_monitoring.ObjectiveConfig( - drift_detection_config = model_monitoring.DriftDetectionConfig( + drift_detection_config=model_monitoring.DriftDetectionConfig( drift_thresholds=DRIFT_THRESHOLDS, attribute_drift_thresholds=ATTRIB_DRIFT_THRESHOLDS, ) @@ -218,7 +218,7 @@ def test_mdm_pause_and_update_config(self): for i in range(2000): DEFAULT_INPUT["cnt_user_engagement"] += i self.endpoint.predict([DEFAULT_INPUT], use_raw_predict=True) - + # test job update while True: time.sleep(1) @@ -235,7 +235,10 @@ def test_mdm_pause_and_update_config(self): 0 ].objective_config ) - assert gca_obj_config.prediction_drift_detection_config == new_obj_config.drift_detection_config.as_proto() + assert ( + gca_obj_config.prediction_drift_detection_config + == new_obj_config.drift_detection_config.as_proto() + ) break # test pause From 849358950f34accd79456691024d19291219d7e9 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Thu, 22 Sep 2022 17:15:48 -0700 Subject: [PATCH 10/22] addressed more PR comments --- google/cloud/aiplatform/jobs.py | 6 +++--- tests/system/aiplatform/test_model_monitoring.py | 15 --------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 5defd73547..ebd557d4b4 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2484,11 +2484,11 @@ def update( update_mask.append("model_deployment_monitoring_objective_configs") current_job.model_deployment_monitoring_objective_configs = ( ModelDeploymentMonitoringJob._parse_configs( - objective_configs, - aiplatform.Endpoint( + objective_configs = objective_configs, + endpoint = aiplatform.Endpoint( current_job.endpoint, credentials=self.credentials ), - deployed_model_ids, + deployed_model_ids = deployed_model_ids, ) ) self.api_client.update_model_deployment_monitoring_job( diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index ff970af9ca..5b283624dd 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -127,7 +127,6 @@ def test_mdm_two_models_one_valid_config(self): Enable model monitoring on two existing models deployed to the same endpoint. """ # test model monitoring configurations - job = None job = aiplatform.ModelDeploymentMonitoringJob.create( display_name=self._make_display_name(key=JOB_NAME), logging_sampling_strategy=sampling_strategy, @@ -138,8 +137,6 @@ def test_mdm_two_models_one_valid_config(self): project=e2e_base._PROJECT, location=e2e_base._LOCATION, endpoint=self.endpoint, - predict_instance_schema_uri="", - analysis_instance_schema_uri="", ) gapic_job = job._gca_resource @@ -186,7 +183,6 @@ def test_mdm_two_models_one_valid_config(self): def test_mdm_pause_and_update_config(self): """Test objective config updates for existing MDM job""" - job = None job = aiplatform.ModelDeploymentMonitoringJob.create( display_name=self._make_display_name(key=JOB_NAME), logging_sampling_strategy=sampling_strategy, @@ -199,8 +195,6 @@ def test_mdm_pause_and_update_config(self): project=e2e_base._PROJECT, location=e2e_base._LOCATION, endpoint=self.endpoint, - predict_instance_schema_uri="", - analysis_instance_schema_uri="", ) # test unsuccessful job update when it's pending DRIFT_THRESHOLDS["cnt_user_engagement"] += 0.01 @@ -261,7 +255,6 @@ def test_mdm_two_models_two_valid_configs(self): deployed_model1: objective_config, deployed_model2: objective_config2, } - job = None job = aiplatform.ModelDeploymentMonitoringJob.create( display_name=self._make_display_name(key=JOB_NAME), logging_sampling_strategy=sampling_strategy, @@ -272,8 +265,6 @@ def test_mdm_two_models_two_valid_configs(self): project=e2e_base._PROJECT, location=e2e_base._LOCATION, endpoint=self.endpoint, - predict_instance_schema_uri="", - analysis_instance_schema_uri="", ) gapic_job = job._gca_resource @@ -325,8 +316,6 @@ def test_mdm_invalid_config_incorrect_model_id(self): project=e2e_base._PROJECT, location=e2e_base._LOCATION, endpoint=self.endpoint, - predict_instance_schema_uri="", - analysis_instance_schema_uri="", deployed_model_ids=[""], ) assert "Invalid model ID" in str(e.value) @@ -344,8 +333,6 @@ def test_mdm_invalid_config_xai(self): project=e2e_base._PROJECT, location=e2e_base._LOCATION, endpoint=self.endpoint, - predict_instance_schema_uri="", - analysis_instance_schema_uri="", ) assert ( "`explanation_config` should only be enabled if the model has `explanation_spec populated" @@ -373,8 +360,6 @@ def test_mdm_two_models_invalid_configs_xai(self): project=e2e_base._PROJECT, location=e2e_base._LOCATION, endpoint=self.endpoint, - predict_instance_schema_uri="", - analysis_instance_schema_uri="", ) assert ( "`explanation_config` should only be enabled if the model has `explanation_spec populated" From 81f70dd95b10d33716eb995a1db4670e47c51df6 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 23 Sep 2022 00:18:42 +0000 Subject: [PATCH 11/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/aiplatform/jobs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index ebd557d4b4..af3815e47f 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2484,11 +2484,11 @@ def update( update_mask.append("model_deployment_monitoring_objective_configs") current_job.model_deployment_monitoring_objective_configs = ( ModelDeploymentMonitoringJob._parse_configs( - objective_configs = objective_configs, - endpoint = aiplatform.Endpoint( + objective_configs=objective_configs, + endpoint=aiplatform.Endpoint( current_job.endpoint, credentials=self.credentials ), - deployed_model_ids = deployed_model_ids, + deployed_model_ids=deployed_model_ids, ) ) self.api_client.update_model_deployment_monitoring_job( From df0bd213eb3ea6a374839eff5133268dbb5da80e Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 23 Sep 2022 00:18:57 +0000 Subject: [PATCH 12/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- google/cloud/aiplatform/jobs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index ebd557d4b4..af3815e47f 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -2484,11 +2484,11 @@ def update( update_mask.append("model_deployment_monitoring_objective_configs") current_job.model_deployment_monitoring_objective_configs = ( ModelDeploymentMonitoringJob._parse_configs( - objective_configs = objective_configs, - endpoint = aiplatform.Endpoint( + objective_configs=objective_configs, + endpoint=aiplatform.Endpoint( current_job.endpoint, credentials=self.credentials ), - deployed_model_ids = deployed_model_ids, + deployed_model_ids=deployed_model_ids, ) ) self.api_client.update_model_deployment_monitoring_job( From 61af46c9caf296ed2e188ac5bfd3909042414c9f Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Fri, 23 Sep 2022 15:32:31 -0700 Subject: [PATCH 13/22] removed unused code --- tests/unit/aiplatform/test_jobs.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 43dda023c9..6116e0d737 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -979,21 +979,6 @@ def test_batch_predict_job_with_versioned_model( ) -@pytest.fixture -def create_mdm_job_mock(): - with mock.patch.object( - _TEST_API_CLIENT, "create_model_deployment_monitoring_job" - ) as create_mdm_job_mock: - create_mdm_job_mock.return_value = ( - gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( - name=_TEST_MDM_JOB_NAME, - display_name=_TEST_DISPLAY_NAME, - state=_TEST_JOB_STATE_RUNNING, - ) - ) - yield create_mdm_job_mock - - @pytest.fixture def get_mdm_job_mock(): with mock.patch.object( @@ -1054,7 +1039,7 @@ def teardown_method(self): initializer.global_pool.shutdown(wait=True) @pytest.mark.usefixtures( - "create_mdm_job_mock", "get_mdm_job_mock", "update_mdm_job_mock" + "get_mdm_job_mock", "update_mdm_job_mock" ) def test_update_mdm_job(self): job = jobs.ModelDeploymentMonitoringJob( From beee888654133e7dcebfbb58a8f953aa4ae9d4f0 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 23 Sep 2022 22:34:40 +0000 Subject: [PATCH 14/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/unit/aiplatform/test_jobs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 6116e0d737..97e3ff4879 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -1038,9 +1038,7 @@ def setup_method(self): def teardown_method(self): initializer.global_pool.shutdown(wait=True) - @pytest.mark.usefixtures( - "get_mdm_job_mock", "update_mdm_job_mock" - ) + @pytest.mark.usefixtures("get_mdm_job_mock", "update_mdm_job_mock") def test_update_mdm_job(self): job = jobs.ModelDeploymentMonitoringJob( model_deployment_monitoring_job_name=_TEST_MDM_JOB_NAME From 26c517ff12310ae8d92da55696e96678d28862f0 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Mon, 26 Sep 2022 18:50:19 -0700 Subject: [PATCH 15/22] addressed more PR feedback --- .../aiplatform/test_model_monitoring.py | 2 - tests/unit/aiplatform/test_jobs.py | 62 ++++++++++--------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/tests/system/aiplatform/test_model_monitoring.py b/tests/system/aiplatform/test_model_monitoring.py index 5b283624dd..cf347337ff 100644 --- a/tests/system/aiplatform/test_model_monitoring.py +++ b/tests/system/aiplatform/test_model_monitoring.py @@ -239,8 +239,6 @@ def test_mdm_pause_and_update_config(self): job.pause() while job.state != gca_job_state.JobState.JOB_STATE_PAUSED: time.sleep(1) - if job.state == gca_job_state.JobState.JOB_STATE_PAUSED: - break job.delete() # confirm deletion diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 97e3ff4879..46dca15f86 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -39,6 +39,7 @@ machine_resources as gca_machine_resources_compat, manual_batch_tuning_parameters as gca_manual_batch_tuning_parameters_compat, model_deployment_monitoring_job as gca_model_deployment_monitoring_job_compat, + model_monitoring as gca_model_monitoring_compat, endpoint as gca_endpoint_compat, ) @@ -46,6 +47,9 @@ job_service_client, ) from google.protobuf import field_mask_pb2 # type: ignore +from google.api_core import operation + +from test_endpoints import get_endpoint_with_models_mock _TEST_API_CLIENT = job_service_client.JobServiceClient @@ -172,6 +176,8 @@ _TEST_JOB_DELETE_METHOD_NAME = "delete_custom_job" _TEST_JOB_RESOURCE_NAME = f"{_TEST_PARENT}/customJobs/{_TEST_ID}" +_TEST_MDM_JOB_DRIFT_DETECTION_CONFIG = {"TEST_KEY": 0.01} + # TODO(b/171333554): Move reusable test fixtures to conftest.py file @@ -996,35 +1002,28 @@ def get_mdm_job_mock(): @pytest.fixture -def mock_deployed_model(): - with mock.patch.object(gca_endpoint_compat, "DeployedModel") as mock_deployed_model: - mock_deployed_model.id = _TEST_ID - mock_deployed_model.explanation_spec.parameters = "TEST_VALUE" - yield mock_deployed_model - - -@pytest.fixture -def endpoint_list_models_mock(mock_deployed_model): - with mock.patch.object( - aiplatform.Endpoint, "list_models" - ) as endpoint_list_models_mock: - endpoint_list_models_mock.return_value = [mock_deployed_model] - yield endpoint_list_models_mock - - -@pytest.fixture -def update_mdm_job_mock(get_mdm_job_mock, endpoint_list_models_mock): +def update_mdm_job_mock(get_mdm_job_mock, get_endpoint_with_models_mock): with mock.patch.object( - job_service_client.JobServiceClient, "update_model_deployment_monitoring_job" + _TEST_API_CLIENT,"update_model_deployment_monitoring_job" ) as update_mdm_job_mock: - mock_client_call = mock.Mock( - spec=job_service_client.JobServiceClient.update_model_deployment_monitoring_job + expected_objective_config = gca_model_monitoring_compat.ModelMonitoringObjectiveConfig( + prediction_drift_detection_config = gca_model_monitoring_compat.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig( + drift_thresholds = {"TEST_KEY":gca_model_monitoring_compat.ThresholdConfig(value = 0.01)} + ) ) - update_mdm_job_mock.return_value = mock_client_call( - model_deployment_monitoring_job=get_mdm_job_mock, - update_mask=field_mask_pb2.FieldMask( - paths=["model_deployment_monitoring_objective_configs"] - ), + all_configs = [] + for model in get_endpoint_with_models_mock.return_value.deployed_models: + all_configs.append(gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig( + deployed_model_id=model.id, + objective_config=expected_objective_config + )) + + update_mdm_job_mock.return_vaue.result_type = gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( + name=_TEST_MDM_JOB_NAME, + display_name=_TEST_DISPLAY_NAME, + state=_TEST_JOB_STATE_RUNNING, + endpoint=_TEST_ENDPOINT, + model_deployment_monitoring_objective_configs = all_configs ) yield update_mdm_job_mock @@ -1039,14 +1038,19 @@ def teardown_method(self): initializer.global_pool.shutdown(wait=True) @pytest.mark.usefixtures("get_mdm_job_mock", "update_mdm_job_mock") - def test_update_mdm_job(self): + def test_update_mdm_job(self, get_mdm_job_mock, update_mdm_job_mock): job = jobs.ModelDeploymentMonitoringJob( model_deployment_monitoring_job_name=_TEST_MDM_JOB_NAME ) + drift_detection_config = aiplatform.model_monitoring.DriftDetectionConfig(drift_thresholds = _TEST_MDM_JOB_DRIFT_DETECTION_CONFIG) new_config = aiplatform.model_monitoring.ObjectiveConfig( - explanation_config=aiplatform.model_monitoring.ExplanationConfig() + drift_detection_config = drift_detection_config ) job.update(objective_configs=new_config) assert job._gca_resource.model_deployment_monitoring_objective_configs[ 0 - ].objective_config.explanation_config.enable_feature_attributes + ].objective_config.prediction_drift_detection_config == drift_detection_config.as_proto() + update_mdm_job_mock.assert_called_once_with( + model_deployment_monitoring_job = get_mdm_job_mock.return_value, + update_mask = field_mask_pb2.FieldMask(paths=["model_deployment_monitoring_objective_configs"]) + ) From c991ffa558440f2b6905c941406e6ae6b41a93ac Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 27 Sep 2022 01:53:21 +0000 Subject: [PATCH 16/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/unit/aiplatform/test_jobs.py | 55 ++++++++++++++++++------------ 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 46dca15f86..4cf861160d 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -1004,26 +1004,32 @@ def get_mdm_job_mock(): @pytest.fixture def update_mdm_job_mock(get_mdm_job_mock, get_endpoint_with_models_mock): with mock.patch.object( - _TEST_API_CLIENT,"update_model_deployment_monitoring_job" + _TEST_API_CLIENT, "update_model_deployment_monitoring_job" ) as update_mdm_job_mock: expected_objective_config = gca_model_monitoring_compat.ModelMonitoringObjectiveConfig( - prediction_drift_detection_config = gca_model_monitoring_compat.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig( - drift_thresholds = {"TEST_KEY":gca_model_monitoring_compat.ThresholdConfig(value = 0.01)} + prediction_drift_detection_config=gca_model_monitoring_compat.ModelMonitoringObjectiveConfig.PredictionDriftDetectionConfig( + drift_thresholds={ + "TEST_KEY": gca_model_monitoring_compat.ThresholdConfig(value=0.01) + } ) ) all_configs = [] for model in get_endpoint_with_models_mock.return_value.deployed_models: - all_configs.append(gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig( - deployed_model_id=model.id, - objective_config=expected_objective_config - )) - - update_mdm_job_mock.return_vaue.result_type = gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( - name=_TEST_MDM_JOB_NAME, - display_name=_TEST_DISPLAY_NAME, - state=_TEST_JOB_STATE_RUNNING, - endpoint=_TEST_ENDPOINT, - model_deployment_monitoring_objective_configs = all_configs + all_configs.append( + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringObjectiveConfig( + deployed_model_id=model.id, + objective_config=expected_objective_config, + ) + ) + + update_mdm_job_mock.return_vaue.result_type = ( + gca_model_deployment_monitoring_job_compat.ModelDeploymentMonitoringJob( + name=_TEST_MDM_JOB_NAME, + display_name=_TEST_DISPLAY_NAME, + state=_TEST_JOB_STATE_RUNNING, + endpoint=_TEST_ENDPOINT, + model_deployment_monitoring_objective_configs=all_configs, + ) ) yield update_mdm_job_mock @@ -1042,15 +1048,22 @@ def test_update_mdm_job(self, get_mdm_job_mock, update_mdm_job_mock): job = jobs.ModelDeploymentMonitoringJob( model_deployment_monitoring_job_name=_TEST_MDM_JOB_NAME ) - drift_detection_config = aiplatform.model_monitoring.DriftDetectionConfig(drift_thresholds = _TEST_MDM_JOB_DRIFT_DETECTION_CONFIG) + drift_detection_config = aiplatform.model_monitoring.DriftDetectionConfig( + drift_thresholds=_TEST_MDM_JOB_DRIFT_DETECTION_CONFIG + ) new_config = aiplatform.model_monitoring.ObjectiveConfig( - drift_detection_config = drift_detection_config + drift_detection_config=drift_detection_config ) job.update(objective_configs=new_config) - assert job._gca_resource.model_deployment_monitoring_objective_configs[ - 0 - ].objective_config.prediction_drift_detection_config == drift_detection_config.as_proto() + assert ( + job._gca_resource.model_deployment_monitoring_objective_configs[ + 0 + ].objective_config.prediction_drift_detection_config + == drift_detection_config.as_proto() + ) update_mdm_job_mock.assert_called_once_with( - model_deployment_monitoring_job = get_mdm_job_mock.return_value, - update_mask = field_mask_pb2.FieldMask(paths=["model_deployment_monitoring_objective_configs"]) + model_deployment_monitoring_job=get_mdm_job_mock.return_value, + update_mask=field_mask_pb2.FieldMask( + paths=["model_deployment_monitoring_objective_configs"] + ), ) From b3c920c40b3efa17f47a1797b21c1f82035fe367 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 27 Sep 2022 10:16:10 -0700 Subject: [PATCH 17/22] fixing linter issues --- tests/unit/aiplatform/test_jobs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 4cf861160d..00de1f998d 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -40,14 +40,12 @@ manual_batch_tuning_parameters as gca_manual_batch_tuning_parameters_compat, model_deployment_monitoring_job as gca_model_deployment_monitoring_job_compat, model_monitoring as gca_model_monitoring_compat, - endpoint as gca_endpoint_compat, ) from google.cloud.aiplatform.compat.services import ( job_service_client, ) from google.protobuf import field_mask_pb2 # type: ignore -from google.api_core import operation from test_endpoints import get_endpoint_with_models_mock From 780556c090d924713067797cbe1dc3bb718065b1 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 27 Sep 2022 17:23:52 -0700 Subject: [PATCH 18/22] addressed more PR comments --- tests/unit/aiplatform/test_jobs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 00de1f998d..362e767436 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -1000,7 +1000,8 @@ def get_mdm_job_mock(): @pytest.fixture -def update_mdm_job_mock(get_mdm_job_mock, get_endpoint_with_models_mock): +@pytest.mark.usefixtures("get_mdm_job_mock") +def update_mdm_job_mock(get_endpoint_with_models_mock): with mock.patch.object( _TEST_API_CLIENT, "update_model_deployment_monitoring_job" ) as update_mdm_job_mock: @@ -1041,7 +1042,6 @@ def setup_method(self): def teardown_method(self): initializer.global_pool.shutdown(wait=True) - @pytest.mark.usefixtures("get_mdm_job_mock", "update_mdm_job_mock") def test_update_mdm_job(self, get_mdm_job_mock, update_mdm_job_mock): job = jobs.ModelDeploymentMonitoringJob( model_deployment_monitoring_job_name=_TEST_MDM_JOB_NAME @@ -1059,6 +1059,9 @@ def test_update_mdm_job(self, get_mdm_job_mock, update_mdm_job_mock): ].objective_config.prediction_drift_detection_config == drift_detection_config.as_proto() ) + get_mdm_job_mock.assert_called_with( + name=_TEST_MDM_JOB_NAME, + ) update_mdm_job_mock.assert_called_once_with( model_deployment_monitoring_job=get_mdm_job_mock.return_value, update_mask=field_mask_pb2.FieldMask( From b4afa3d9c3cc55c6c6d0b99963db1f0b09a41d8a Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Tue, 27 Sep 2022 21:38:05 -0700 Subject: [PATCH 19/22] fixing pylint errors --- tests/unit/aiplatform/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 362e767436..0dff76f9eb 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -47,7 +47,7 @@ ) from google.protobuf import field_mask_pb2 # type: ignore -from test_endpoints import get_endpoint_with_models_mock +from test_endpoints import get_endpoint_with_models_mock # type: ignore _TEST_API_CLIENT = job_service_client.JobServiceClient From 7f708f72b9f9c2c79cca160e445453649af780fa Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Wed, 28 Sep 2022 04:41:02 +0000 Subject: [PATCH 20/22] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- tests/unit/aiplatform/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 0dff76f9eb..0eccfbef7d 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -47,7 +47,7 @@ ) from google.protobuf import field_mask_pb2 # type: ignore -from test_endpoints import get_endpoint_with_models_mock # type: ignore +from test_endpoints import get_endpoint_with_models_mock # type: ignore _TEST_API_CLIENT = job_service_client.JobServiceClient From 61f978c386d7257190dfa18723853dcf574f92a1 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Wed, 28 Sep 2022 00:02:19 -0700 Subject: [PATCH 21/22] silencing unused import warning --- tests/unit/aiplatform/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 0eccfbef7d..797910ef21 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -47,7 +47,7 @@ ) from google.protobuf import field_mask_pb2 # type: ignore -from test_endpoints import get_endpoint_with_models_mock # type: ignore +from test_endpoints import get_endpoint_with_models_mock # noqa: F401 _TEST_API_CLIENT = job_service_client.JobServiceClient From f0b846bf01bef085e3e082f9dc2c230fd4aed723 Mon Sep 17 00:00:00 2001 From: Rosie Zou Date: Wed, 28 Sep 2022 22:41:03 +0000 Subject: [PATCH 22/22] fixed unused import error --- tests/unit/aiplatform/test_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/aiplatform/test_jobs.py b/tests/unit/aiplatform/test_jobs.py index 797910ef21..9d788df262 100644 --- a/tests/unit/aiplatform/test_jobs.py +++ b/tests/unit/aiplatform/test_jobs.py @@ -1001,7 +1001,7 @@ def get_mdm_job_mock(): @pytest.fixture @pytest.mark.usefixtures("get_mdm_job_mock") -def update_mdm_job_mock(get_endpoint_with_models_mock): +def update_mdm_job_mock(get_endpoint_with_models_mock): # noqa: F811 with mock.patch.object( _TEST_API_CLIENT, "update_model_deployment_monitoring_job" ) as update_mdm_job_mock: