diff --git a/src/k8s-extension/HISTORY.rst b/src/k8s-extension/HISTORY.rst index b9a5a22c41d..99b8f0a1a80 100644 --- a/src/k8s-extension/HISTORY.rst +++ b/src/k8s-extension/HISTORY.rst @@ -3,6 +3,11 @@ Release History =============== +1.2.4 +++++++++++++++++++ +* microsoft.azureml.kubernetes: Do not invoke `create_or_update` for already existed resources. +* microsoft.azuremonitor.containers: ContainerInsights Extension Managed Identity Auth Onboarding updates. + 1.2.3 ++++++++++++++++++ * Fix warning message returned on PATCH diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py index db20b3bec3a..4f3613daa09 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMLKubernetes.py @@ -26,6 +26,8 @@ from azure.mgmt.resource.locks.models import ManagementLockObject from knack.log import get_logger from msrestazure.azure_exceptions import CloudError +from msrest.exceptions import HttpOperationError +import azure.core.exceptions from .._client_factory import cf_resources from .DefaultExtension import DefaultExtension, user_confirmation_factory @@ -553,11 +555,11 @@ def __create_required_resource( self.SERVICE_BUS_COMPUTE_STATE_TOPIC: self.SERVICE_BUS_COMPUTE_STATE_SUB, self.SERVICE_BUS_JOB_STATE_TOPIC: self.SERVICE_BUS_JOB_STATE_SUB } - service_bus_connection_string, service_buse_resource_id = _get_service_bus_connection_string( + service_bus_connection_string, service_bus_resource_id = _get_service_bus_connection_string( cmd, subscription_id, resource_group_name, cluster_name, cluster_location, topic_sub_mapping) logger.info('==== END SERVICE BUS CREATION ====') configuration_protected_settings[self.SERVICE_BUS_CONNECTION_STRING] = service_bus_connection_string - configuration_settings[self.SERVICE_BUS_RESOURCE_ID_KEY] = service_buse_resource_id + configuration_settings[self.SERVICE_BUS_RESOURCE_ID_KEY] = service_bus_resource_id configuration_settings[f'{self.SERVICE_BUS_TOPIC_SUB_MAPPING_KEY}.{self.SERVICE_BUS_COMPUTE_STATE_TOPIC}'] = self.SERVICE_BUS_COMPUTE_STATE_SUB configuration_settings[f'{self.SERVICE_BUS_TOPIC_SUB_MAPPING_KEY}.{self.SERVICE_BUS_JOB_STATE_TOPIC}'] = self.SERVICE_BUS_JOB_STATE_SUB @@ -603,7 +605,18 @@ def _get_relay_connection_str( cluster_id, suffix_len=6, max_len=50) hybrid_connection_name = cluster_name hc_resource_id = '' - if not get_key_only: + + # only create relay if not found + try: + # get connection string + hybrid_connection_object = relay_client.hybrid_connections.get( + resource_group_name, relay_namespace_name, hybrid_connection_name) + hc_resource_id = hybrid_connection_object.id + key: azure.mgmt.relay.models.AccessKeys = relay_client.hybrid_connections.list_keys( + resource_group_name, relay_namespace_name, hybrid_connection_name, auth_rule_name) + except HttpOperationError as e: + if e.response.status_code != 404 or get_key_only: + raise e # create namespace relay_namespace_params = azure.mgmt.relay.models.RelayNamespace( location=cluster_location, tags=resource_tag) @@ -626,9 +639,9 @@ def _get_relay_connection_str( relay_client.hybrid_connections.create_or_update_authorization_rule( resource_group_name, relay_namespace_name, hybrid_connection_name, auth_rule_name, rights=auth_rule_rights) - # get connection string - key: azure.mgmt.relay.models.AccessKeys = relay_client.hybrid_connections.list_keys( - resource_group_name, relay_namespace_name, hybrid_connection_name, auth_rule_name) + # get connection string + key: azure.mgmt.relay.models.AccessKeys = relay_client.hybrid_connections.list_keys( + resource_group_name, relay_namespace_name, hybrid_connection_name, auth_rule_name) return f'{key.primary_connection_string}', hc_resource_id, hybrid_connection_name @@ -640,8 +653,20 @@ def _get_service_bus_connection_string(cmd, subscription_id, resource_group_name subscription_id, resource_group_name) service_bus_namespace_name = _get_valid_name( cluster_id, suffix_len=6, max_len=50) - - if not get_key_only: + try: + service_bus_object = service_bus_client.namespaces.get(resource_group_name, service_bus_namespace_name) + service_bus_resource_id = service_bus_object.id + + # get connection string + auth_rules = service_bus_client.namespaces.list_authorization_rules( + resource_group_name, service_bus_namespace_name) + for rule in auth_rules: + key: azure.mgmt.servicebus.models.AccessKeys = service_bus_client.namespaces.list_keys( + resource_group_name, service_bus_namespace_name, rule.name) + return key.primary_connection_string, service_bus_resource_id + except azure.core.exceptions.HttpResponseError as e: + if e.response.status_code != 404 or get_key_only: + raise e # create namespace service_bus_sku = azure.mgmt.servicebus.models.SBSku( name=azure.mgmt.servicebus.models.SkuName.standard.name) @@ -652,8 +677,9 @@ def _get_service_bus_connection_string(cmd, subscription_id, resource_group_name async_poller = service_bus_client.namespaces.begin_create_or_update( resource_group_name, service_bus_namespace_name, service_bus_namespace) while True: - async_poller.result(15) + service_bus_object = async_poller.result(15) if async_poller.done(): + service_bus_resource_id = service_bus_object.id break for topic_name, service_bus_subscription_name in topic_sub_mapping.items(): @@ -668,16 +694,13 @@ def _get_service_bus_connection_string(cmd, subscription_id, resource_group_name service_bus_client.subscriptions.create_or_update( resource_group_name, service_bus_namespace_name, topic_name, service_bus_subscription_name, sub) - service_bus_object = service_bus_client.namespaces.get(resource_group_name, service_bus_namespace_name) - service_bus_resource_id = service_bus_object.id - - # get connection string - auth_rules = service_bus_client.namespaces.list_authorization_rules( - resource_group_name, service_bus_namespace_name) - for rule in auth_rules: - key: azure.mgmt.servicebus.models.AccessKeys = service_bus_client.namespaces.list_keys( - resource_group_name, service_bus_namespace_name, rule.name) - return key.primary_connection_string, service_bus_resource_id + # get connection string + auth_rules = service_bus_client.namespaces.list_authorization_rules( + resource_group_name, service_bus_namespace_name) + for rule in auth_rules: + key: azure.mgmt.servicebus.models.AccessKeys = service_bus_client.namespaces.list_keys( + resource_group_name, service_bus_namespace_name, rule.name) + return key.primary_connection_string, service_bus_resource_id def _get_log_analytics_ws_connection_string( @@ -689,7 +712,15 @@ def _get_log_analytics_ws_connection_string( cluster_id = '{}-{}-{}'.format(cluster_name, subscription_id, resource_group_name) log_analytics_ws_name = _get_valid_name(cluster_id, suffix_len=6, max_len=63) customer_id = '' - if not get_key_only: + try: + # get workspace shared keys + log_analytics_ws_object = log_analytics_ws_client.workspaces.get(resource_group_name, log_analytics_ws_name) + customer_id = log_analytics_ws_object.customer_id + shared_key = log_analytics_ws_client.shared_keys.get_shared_keys( + resource_group_name, log_analytics_ws_name).primary_shared_key + except azure.core.exceptions.HttpResponseError as e: + if e.response.status_code != 404 or get_key_only: + raise e log_analytics_ws = azure.mgmt.loganalytics.models.Workspace(location=cluster_location, tags=resource_tag) async_poller = log_analytics_ws_client.workspaces.begin_create_or_update( resource_group_name, log_analytics_ws_name, log_analytics_ws) @@ -698,10 +729,8 @@ def _get_log_analytics_ws_connection_string( if async_poller.done(): customer_id = log_analytics_ws_object.customer_id break - - # get workspace shared keys - shared_key = log_analytics_ws_client.shared_keys.get_shared_keys( - resource_group_name, log_analytics_ws_name).primary_shared_key + shared_key = log_analytics_ws_client.shared_keys.get_shared_keys( + resource_group_name, log_analytics_ws_name).primary_shared_key return customer_id, shared_key diff --git a/src/k8s-extension/azext_k8s_extension/partner_extensions/ContainerInsights.py b/src/k8s-extension/azext_k8s_extension/partner_extensions/ContainerInsights.py index b2eecf32f89..f2eeb3ff5be 100644 --- a/src/k8s-extension/azext_k8s_extension/partner_extensions/ContainerInsights.py +++ b/src/k8s-extension/azext_k8s_extension/partner_extensions/ContainerInsights.py @@ -92,7 +92,7 @@ def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_t if (isinstance(useAADAuthSetting, str) and str(useAADAuthSetting).lower() == "true") or (isinstance(useAADAuthSetting, bool) and useAADAuthSetting): useAADAuth = True if useAADAuth: - association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2019-11-01-preview" + association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2021-04-01" for _ in range(3): try: send_raw_request(cmd.cli_ctx, "GET", association_url,) @@ -106,7 +106,7 @@ def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_t pass # its OK to ignore the exception since MSI auth in preview if isDCRAExists: - association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2019-11-01-preview" + association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2021-04-01" for _ in range(3): try: send_raw_request(cmd.cli_ctx, "DELETE", association_url,) @@ -612,7 +612,7 @@ def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_ if (cluster_region not in region_ids): raise ClientRequestError(f"Data Collection Rule Associations are not supported for cluster region {cluster_region}") - dcr_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{dcr_resource_id}?api-version=2019-11-01-preview" + dcr_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{dcr_resource_id}?api-version=2021-04-01" # get existing tags on the container insights extension DCR if the customer added any existing_tags = get_existing_container_insights_extension_dcr_tags(cmd, dcr_url) @@ -627,18 +627,7 @@ def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_ { "name": "ContainerInsightsExtension", "streams": [ - "Microsoft-Perf", - "Microsoft-ContainerInventory", - "Microsoft-ContainerLog", - "Microsoft-ContainerLogV2", - "Microsoft-ContainerNodeInventory", - "Microsoft-KubeEvents", - "Microsoft-KubeMonAgentEvents", - "Microsoft-KubeNodeInventory", - "Microsoft-KubePodInventory", - "Microsoft-KubePVInventory", - "Microsoft-KubeServices", - "Microsoft-InsightsMetrics", + "Microsoft-ContainerInsights-Group-Default" ], "extensionName": "ContainerInsights", } @@ -647,18 +636,8 @@ def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_ "dataFlows": [ { "streams": [ - "Microsoft-Perf", - "Microsoft-ContainerInventory", - "Microsoft-ContainerLog", - "Microsoft-ContainerLogV2", - "Microsoft-ContainerNodeInventory", - "Microsoft-KubeEvents", - "Microsoft-KubeMonAgentEvents", - "Microsoft-KubeNodeInventory", - "Microsoft-KubePodInventory", - "Microsoft-KubePVInventory", - "Microsoft-KubeServices", - "Microsoft-InsightsMetrics", + "Microsoft-ContainerInsights-Group-Default" + ], "destinations": ["la-workspace"], } @@ -694,7 +673,7 @@ def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_ }, } ) - association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2019-11-01-preview" + association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2021-04-01" for _ in range(3): try: send_raw_request(cmd.cli_ctx, "PUT", association_url, body=association_body,) diff --git a/src/k8s-extension/azext_k8s_extension/tests/latest/test_custom.py b/src/k8s-extension/azext_k8s_extension/tests/latest/test_custom.py index 0ea7cefb9ba..7a8716ecc73 100644 --- a/src/k8s-extension/azext_k8s_extension/tests/latest/test_custom.py +++ b/src/k8s-extension/azext_k8s_extension/tests/latest/test_custom.py @@ -14,17 +14,23 @@ class TestIsDogfoodCluster(unittest.TestCase): def test_dogfood_cluster(self): cmd = MockCommand() - cmd.cli_ctx.cloud.endpoints.resource_manager = "https://api-dogfood.resources.windows-int.net" + cmd.cli_ctx.cloud.endpoints.resource_manager = ( + "https://api-dogfood.resources.windows-int.net" + ) assert is_dogfood_cluster(cmd) def test_dogfood_cluster_with_slash(self): cmd = MockCommand() - cmd.cli_ctx.cloud.endpoints.resource_manager = "https://api-dogfood.resources.windows-int.net/" + cmd.cli_ctx.cloud.endpoints.resource_manager = ( + "https://api-dogfood.resources.windows-int.net/" + ) assert is_dogfood_cluster(cmd) def test_longer_hostname(self): cmd = MockCommand() - cmd.cli_ctx.cloud.endpoints.resource_manager = "https://api-dogfood.resources.windows-int.otherwebsite.net/" + cmd.cli_ctx.cloud.endpoints.resource_manager = ( + "https://api-dogfood.resources.windows-int.otherwebsite.net/" + ) assert not is_dogfood_cluster(cmd) def malformed_url(self): @@ -35,4 +41,4 @@ def malformed_url(self): def test_prod_cluster(self): cmd = MockCommand() cmd.cli_ctx.cloud.endpoints.resource_manager = "https://management.azure.com" - assert not is_dogfood_cluster(cmd) \ No newline at end of file + assert not is_dogfood_cluster(cmd) diff --git a/src/k8s-extension/setup.py b/src/k8s-extension/setup.py index 2635d1027f9..a2e25840afb 100644 --- a/src/k8s-extension/setup.py +++ b/src/k8s-extension/setup.py @@ -33,7 +33,7 @@ # TODO: Add any additional SDK dependencies here DEPENDENCIES = [] -VERSION = "1.2.3" +VERSION = "1.2.4" with open("README.rst", "r", encoding="utf-8") as f: README = f.read()