[Connectedk8s] Kubernetes distro and infra parameter and heuristics s…

…upport (#2620)

src/connectedk8s/HISTORY.rst

@@ -7,6 +7,7 @@ Release History
 ++++++
 * Added checks for proxy and added disable-proxy
 * Updated config dataplane endpoint to support other clouds
+* `az connectedk8s connect`: Added support for kubernetes distro/infra parameters and heuristics
 
 0.2.7
 ++++++

src/connectedk8s/azext_connectedk8s/_client_factory.py

@@ -10,8 +10,8 @@
 
 
 def cf_connectedk8s(cli_ctx, *_):
-    from azext_connectedk8s.vendored_sdks import KubernetesConnectRPClient
-    return get_mgmt_service_client(cli_ctx, KubernetesConnectRPClient)
+    from azext_connectedk8s.vendored_sdks import ConnectedKubernetesClient
+    return get_mgmt_service_client(cli_ctx, ConnectedKubernetesClient)
 
 
 def cf_connected_cluster(cli_ctx, _):

src/connectedk8s/azext_connectedk8s/_constants.py

@@ -6,6 +6,9 @@
 
 # pylint: disable=line-too-long
 
+Distribution_Enum_Values = ["auto", "generic", "openshift", "rancher_rke", "kind", "k3s", "minikube", "gke", "eks", "aks", "aks_hci", "capz", "aks_engine", "tkg"]
+Infrastructure_Enum_Values = ["auto", "generic", "azure", "aws", "gcp", "azure_stack_hci", "azure_stack_hub", "azure_stack_edge", "vsphere"]
+
 Azure_PublicCloudName = 'AZUREPUBLICCLOUD'
 Azure_USGovCloudName = 'AZUREUSGOVERNMENTCLOUD'
 Azure_DogfoodCloudName = 'AZUREDOGFOOD'
@@ -50,5 +53,6 @@
 Kubeconfig_Failed_To_Load_Fault_Type = "failed-to-load-kubeconfig-file"
 Proxy_Cert_Path_Does_Not_Exist_Fault_Type = 'proxy-cert-path-does-not-exist-error'
 Proxy_Cert_Path_Does_Not_Exist_Error = 'Proxy cert path {} does not exist. Please check the path provided'
+Get_Kubernetes_Infra_Fault_Type = 'kubernetes-get-infrastructure-error'
 No_Param_Error = 'No parmeters were specified with update command. Please run az connectedk8s update --help to check parameters available for update'
 EnableProxy_Conflict_Error = 'Conflict detected: --disable-proxy can not be set with --https-proxy, --http-proxy, --proxy-skip-range and --proxy-cert at the same time. Please run az connectedk8s update --help for more information about the parameters'

src/connectedk8s/azext_connectedk8s/_params.py

@@ -5,9 +5,10 @@
 # pylint: disable=line-too-long
 
 from argcomplete.completers import FilesCompleter
-from azure.cli.core.commands.parameters import get_location_type
+from azure.cli.core.commands.parameters import get_location_type, get_enum_type
 from azure.cli.core.commands.parameters import (file_type)
 from azure.cli.core.commands.validators import get_default_location_from_resource_group
+from azext_connectedk8s._constants import Distribution_Enum_Values, Infrastructure_Enum_Values
 
 
 def load_arguments(self, _):
@@ -24,6 +25,8 @@ def load_arguments(self, _):
         c.argument('http_proxy', options_list=['--proxy-http'], help='Http proxy URL to be used.')
         c.argument('no_proxy', options_list=['--proxy-skip-range'], help='List of URLs/CIDRs for which proxy should not to be used.')
         c.argument('proxy_cert', options_list=['--proxy-cert'], type=file_type, completer=FilesCompleter(), help='Path to the certificate file for proxy')
+        c.argument('distribution', options_list=['--distribution'], help='The Kubernetes distribution which will be running on this connected cluster.', arg_type=get_enum_type(Distribution_Enum_Values))
+        c.argument('infrastructure', options_list=['--infrastructure'], help='The infrastructure on which the Kubernetes cluster represented by this connected cluster will be running on.', arg_type=get_enum_type(Infrastructure_Enum_Values))
 
     with self.argument_context('connectedk8s update') as c:
         c.argument('cluster_name', options_list=['--name', '-n'], id_part='name', help='The name of the connected cluster.')

src/connectedk8s/azext_connectedk8s/_utils.py

@@ -197,3 +197,10 @@ def kubernetes_exception_handler(ex, fault_type, summary, error_message='Error o
         if raise_error:
             telemetry.set_exception(exception=ex, fault_type=fault_type, summary=summary)
             raise CLIError(error_message + "\nError: " + str(ex))
+
+
+def validate_infrastructure_type(infra):
+    for s in consts.Infrastructure_Enum_Values[1:]:  # First value is "auto"
+        if s.lower() == infra.lower():
+            return s
+    return "generic"

src/connectedk8s/azext_connectedk8s/custom.py

@@ -39,7 +39,7 @@
 
 
 def create_connectedk8s(cmd, client, resource_group_name, cluster_name, https_proxy="", http_proxy="", no_proxy="", proxy_cert="", location=None,
-                        kube_config=None, kube_context=None, no_wait=False, tags=None):
+                        kube_config=None, kube_context=None, no_wait=False, tags=None, distribution='auto', infrastructure='auto'):
     logger.warning("Ensure that you have the latest helm version installed before proceeding.")
     logger.warning("This operation might take a while...\n")
 
@@ -108,13 +108,21 @@ def create_connectedk8s(cmd, client, resource_group_name, cluster_name, https_pr
     # if the user had not logged in.
     check_kube_connection(configuration)
 
-    # Get kubernetes cluster info for telemetry
+    # Get kubernetes cluster info
     kubernetes_version = get_server_version(configuration)
-    kubernetes_distro = get_kubernetes_distro(configuration)
+    if distribution == 'auto':
+        kubernetes_distro = get_kubernetes_distro(configuration)  # (cluster heuristics)
+    else:
+        kubernetes_distro = distribution
+    if infrastructure == 'auto':
+        kubernetes_infra = get_kubernetes_infra(configuration)  # (cluster heuristics)
+    else:
+        kubernetes_infra = infrastructure
 
     kubernetes_properties = {
         'Context.Default.AzureCLI.KubernetesVersion': kubernetes_version,
-        'Context.Default.AzureCLI.KubernetesDistro': kubernetes_distro
+        'Context.Default.AzureCLI.KubernetesDistro': kubernetes_distro,
+        'Context.Default.AzureCLI.KubernetesInfra': kubernetes_infra
     }
     telemetry.add_extension_event('connectedk8s', kubernetes_properties)
 
@@ -161,7 +169,7 @@ def create_connectedk8s(cmd, client, resource_group_name, cluster_name, https_pr
                                             configmap_cluster_name).agent_public_key_certificate
                 except Exception as e:  # pylint: disable=broad-except
                     utils.arm_exception_handler(e, consts.Get_ConnectedCluster_Fault_Type, 'Failed to check if connected cluster resource already exists.')
-                cc = generate_request_payload(configuration, location, public_key, tags)
+                cc = generate_request_payload(configuration, location, public_key, tags, kubernetes_distro, kubernetes_infra)
                 create_cc_resource(client, resource_group_name, cluster_name, cc, no_wait)
             else:
                 telemetry.set_user_fault()
@@ -229,13 +237,13 @@ def create_connectedk8s(cmd, client, resource_group_name, cluster_name, https_pr
         raise CLIError("Failed to export private key." + str(e))
 
     # Generate request payload
-    cc = generate_request_payload(configuration, location, public_key, tags)
+    cc = generate_request_payload(configuration, location, public_key, tags, kubernetes_distro, kubernetes_infra)
 
     # Create connected cluster resource
     put_cc_response = create_cc_resource(client, resource_group_name, cluster_name, cc, no_wait)
 
     # Install azure-arc agents
-    helm_install_release(chart_path, subscription_id, kubernetes_distro, resource_group_name, cluster_name,
+    helm_install_release(chart_path, subscription_id, kubernetes_distro, kubernetes_infra, resource_group_name, cluster_name,
                          location, onboarding_tenant_id, http_proxy, https_proxy, no_proxy, proxy_cert, private_key_pem, kube_config,
                          kube_context, no_wait, values_file_provided, values_file, azure_cloud)
 
@@ -415,22 +423,67 @@ def get_server_version(configuration):
                                            raise_error=False)
 
 
-def get_kubernetes_distro(configuration):
+def get_kubernetes_distro(configuration):  # Heuristic
     api_instance = kube_client.CoreV1Api(kube_client.ApiClient(configuration))
     try:
         api_response = api_instance.list_node()
         if api_response.items:
             labels = api_response.items[0].metadata.labels
-            if labels.get("node.openshift.io/os_id") == "rhcos" or labels.get("node.openshift.io/os_id") == "rhel":
+            provider_id = str(api_response.items[0].spec.provider_id)
+            annotations = list(api_response.items)[0].metadata.annotations
+            if labels.get("node.openshift.io/os_id"):
                 return "openshift"
-        return "default"
+            if labels.get("kubernetes.azure.com/node-image-version"):
+                return "aks"
+            if labels.get("cloud.google.com/gke-nodepool") or labels.get("cloud.google.com/gke-os-distribution"):
+                return "gke"
+            if labels.get("eks.amazonaws.com/nodegroup"):
+                return "eks"
+            if labels.get("minikube.k8s.io/version"):
+                return "minikube"
+            if provider_id.startswith("kind://"):
+                return "kind"
+            if provider_id.startswith("k3s://"):
+                return "k3s"
+            if annotations.get("rke.cattle.io/external-ip") or annotations.get("rke.cattle.io/internal-ip"):
+                return "rancher_rke"
+            if provider_id.startswith("moc://"):   # Todo: ask from aks hci team for more reliable identifier in node labels,etc
+                return "generic"                   # return "aks_hci"
+        return "generic"
     except Exception as e:  # pylint: disable=broad-except
         logger.warning("Error occured while trying to fetch kubernetes distribution.")
         utils.kubernetes_exception_handler(e, consts.Get_Kubernetes_Distro_Fault_Type, 'Unable to fetch kubernetes distribution',
                                            raise_error=False)
+        return "generic"
 
 
-def generate_request_payload(configuration, location, public_key, tags):
+def get_kubernetes_infra(configuration):  # Heuristic
+    api_instance = kube_client.CoreV1Api(kube_client.ApiClient(configuration))
+    try:
+        api_response = api_instance.list_node()
+        if api_response.items:
+            provider_id = str(api_response.items[0].spec.provider_id)
+            infra = provider_id.split(':')[0]
+            if infra == "k3s" or infra == "kind":
+                return "generic"
+            if infra == "azure":
+                return "azure"
+            if infra == "gce":
+                return "gcp"
+            if infra == "aws":
+                return "aws"
+            if infra == "moc":                  # Todo: ask from aks hci team for more reliable identifier in node labels,etc
+                return "generic"                # return "azure_stack_hci"
+            return utils.validate_infrastructure_type(infra)
+        return "generic"
+    except Exception as e:  # pylint: disable=broad-except
+        logger.warning("Error occured while trying to fetch kubernetes infrastructure.")
+        utils.kubernetes_exception_handler(e, consts.Get_Kubernetes_Infra_Fault_Type, 'Unable to fetch kubernetes infrastructure',
+                                           raise_error=False)
+        return "generic"
+
+
+def generate_request_payload(configuration, location, public_key, tags, kubernetes_distro, kubernetes_infra):
     # Create connected cluster resource object
     aad_profile = ConnectedClusterAADProfile(
         tenant_id="",
@@ -447,7 +500,9 @@ def generate_request_payload(configuration, location, public_key, tags):
         identity=identity,
         agent_public_key_certificate=public_key,
         aad_profile=aad_profile,
-        tags=tags
+        tags=tags,
+        distribution=kubernetes_distro,
+        infrastructure=kubernetes_infra
     )
     return cc
 
@@ -598,12 +653,13 @@ def get_release_namespace(kube_config, kube_context):
     return None
 
 
-def helm_install_release(chart_path, subscription_id, kubernetes_distro, resource_group_name, cluster_name,
+def helm_install_release(chart_path, subscription_id, kubernetes_distro, kubernetes_infra, resource_group_name, cluster_name,
                          location, onboarding_tenant_id, http_proxy, https_proxy, no_proxy, proxy_cert, private_key_pem,
                          kube_config, kube_context, no_wait, values_file_provided, values_file, cloud_name):
     cmd_helm_install = ["helm", "upgrade", "--install", "azure-arc", chart_path,
                         "--set", "global.subscriptionId={}".format(subscription_id),
                         "--set", "global.kubernetesDistro={}".format(kubernetes_distro),
+                        "--set", "global.kubernetesInfra={}".format(kubernetes_infra),
                         "--set", "global.resourceGroupName={}".format(resource_group_name),
                         "--set", "global.resourceName={}".format(cluster_name),
                         "--set", "global.location={}".format(location),
@@ -779,13 +835,6 @@ def update_agents(cmd, client, resource_group_name, cluster_name, https_proxy=""
 
     # Get kubernetes cluster info for telemetry
     kubernetes_version = get_server_version(configuration)
-    kubernetes_distro = get_kubernetes_distro(configuration)
-
-    kubernetes_properties = {
-        'Context.Default.AzureCLI.KubernetesVersion': kubernetes_version,
-        'Context.Default.AzureCLI.KubernetesDistro': kubernetes_distro
-    }
-    telemetry.add_extension_event('connectedk8s', kubernetes_properties)
 
     # Checking helm installation
     check_helm_install(kube_config, kube_context)
@@ -811,6 +860,23 @@ def update_agents(cmd, client, resource_group_name, cluster_name, https_proxy=""
     # Fetch Connected Cluster for agent version
     connected_cluster = get_connectedk8s(cmd, client, resource_group_name, cluster_name)
 
+    if hasattr(connected_cluster, 'distribution') and (connected_cluster.distribution is not None):
+        kubernetes_distro = connected_cluster.distribution
+    else:
+        kubernetes_distro = get_kubernetes_distro(configuration)
+
+    if hasattr(connected_cluster, 'infrastructure') and (connected_cluster.infrastructure is not None):
+        kubernetes_infra = connected_cluster.infrastructure
+    else:
+        kubernetes_infra = get_kubernetes_infra(configuration)
+
+    kubernetes_properties = {
+        'Context.Default.AzureCLI.KubernetesVersion': kubernetes_version,
+        'Context.Default.AzureCLI.KubernetesDistro': kubernetes_distro,
+        'Context.Default.AzureCLI.KubernetesInfra': kubernetes_infra
+    }
+    telemetry.add_extension_event('connectedk8s', kubernetes_properties)
+
     # Adding helm repo
     if os.getenv('HELMREPONAME') and os.getenv('HELMREPOURL'):
         utils.add_helm_repo(kube_config, kube_context)

src/connectedk8s/azext_connectedk8s/vendored_sdks/__init__.py

@@ -9,10 +9,10 @@
 # regenerated.
 # --------------------------------------------------------------------------
 
-from .kubernetes_connect_rp_client import KubernetesConnectRPClient
+from .connected_kubernetes_client import ConnectedKubernetesClient
 from .version import VERSION
 
-__all__ = ['KubernetesConnectRPClient']
+__all__ = ['ConnectedKubernetesClient']
 
 __version__ = VERSION
 

src/connectedk8s/azext_connectedk8s/vendored_sdks/kubernetes_connect_rp_client.py → src/connectedk8s/azext_connectedk8s/vendored_sdks/connected_kubernetes_client.py

@@ -18,8 +18,8 @@
 from . import models
 
 
-class KubernetesConnectRPClientConfiguration(AzureConfiguration):
-    """Configuration for KubernetesConnectRPClient
+class ConnectedKubernetesClientConfiguration(AzureConfiguration):
+    """Configuration for ConnectedKubernetesClient
     Note that all parameters used to create this instance are saved as instance
     attributes.
 
@@ -41,7 +41,7 @@ def __init__(
         if not base_url:
             base_url = 'https://management.azure.com'
 
-        super(KubernetesConnectRPClientConfiguration, self).__init__(base_url)
+        super(ConnectedKubernetesClientConfiguration, self).__init__(base_url)
 
         self.add_user_agent('azure-mgmt-hybridkubernetes/{}'.format(VERSION))
         self.add_user_agent('Azure-SDK-For-Python')
@@ -50,16 +50,16 @@ def __init__(
         self.subscription_id = subscription_id
 
 
-class KubernetesConnectRPClient(SDKClient):
+class ConnectedKubernetesClient(SDKClient):
     """Azure Connected Cluster Resource Provider API for adopting any Kubernetes Cluster
 
     :ivar config: Configuration for client.
-    :vartype config: KubernetesConnectRPClientConfiguration
+    :vartype config: ConnectedKubernetesClientConfiguration
 
     :ivar connected_cluster: ConnectedCluster operations
-    :vartype connected_cluster: azure.mgmt.hybridkubernetes.operations.ConnectedClusterOperations
+    :vartype connected_cluster: azure.mgmt.hybridkubernetes.v2020_01_01_preview.operations.ConnectedClusterOperations
     :ivar operations: Operations operations
-    :vartype operations: azure.mgmt.hybridkubernetes.operations.Operations
+    :vartype operations: azure.mgmt.hybridkubernetes.v2020_01_01_preview.operations.Operations
 
     :param credentials: Credentials needed for the client to connect to Azure.
     :type credentials: :mod:`A msrestazure Credentials
@@ -72,8 +72,8 @@ class KubernetesConnectRPClient(SDKClient):
     def __init__(
             self, credentials, subscription_id, base_url=None):
 
-        self.config = KubernetesConnectRPClientConfiguration(credentials, subscription_id, base_url)
-        super(KubernetesConnectRPClient, self).__init__(self.config.credentials, self.config)
+        self.config = ConnectedKubernetesClientConfiguration(credentials, subscription_id, base_url)
+        super(ConnectedKubernetesClient, self).__init__(self.config.credentials, self.config)
 
         client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)}
         self.api_version = '2020-01-01-preview'