diff --git a/.circleci/config.yml b/.circleci/config.yml index a046cded7a..d538cc3b3c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -137,6 +137,34 @@ jobs: path: /go/src/github.com/Azure/acs-engine/_logs - store_artifacts: path: /go/src/github.com/Azure/acs-engine/_output + k8s-linux-default-e2e: + working_directory: /go/src/github.com/Azure/acs-engine + docker: + - image: quay.io/deis/go-dev:v1.9.1 + environment: + GOPATH: /go + steps: + - checkout + - run: | + echo 'export TIMEOUT=10m' >> $BASH_ENV + echo 'export CLUSTER_DEFINITION=examples/kubernetes.json' >> $BASH_ENV + echo 'export CREATE_VNET=false' >> $BASH_ENV + echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV + echo 'export RETAIN_SSH=false' >> $BASH_ENV + echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV + echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV + - run: + name: compile + command: make build-binary + - run: + name: ginkgo k8s e2e tests + command: make test-kubernetes + no_output_timeout: "30m" + - store_artifacts: + path: /go/src/github.com/Azure/acs-engine/_logs + - store_artifacts: + path: /go/src/github.com/Azure/acs-engine/_output k8s-windows-1.7-release-e2e: working_directory: /go/src/github.com/Azure/acs-engine docker: @@ -344,6 +372,12 @@ workflows: filters: branches: ignore: master + - k8s-linux-default-e2e: + requires: + - test + filters: + branches: + ignore: master - pr-e2e-hold: type: approval requires: @@ -476,4 +510,4 @@ workflows: - test filters: branches: - only: master + only: master \ No newline at end of file diff --git a/examples/kubernetes.json b/examples/kubernetes.json index f210b874b5..aad12b0b8a 100644 --- a/examples/kubernetes.json +++ b/examples/kubernetes.json @@ -12,7 +12,7 @@ "agentPoolProfiles": [ { "name": "agentpool1", - "count": 3, + "count": 2, "vmSize": "Standard_D2_v2", "availabilityProfile": "AvailabilitySet" } diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index 456d5e3a9c..90cea4497a 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -301,10 +301,8 @@ function ensureDocker() { function ensureKubelet() { systemctlEnableAndCheck kubelet - # only start if a reboot is not required - if ! $REBOOTREQUIRED; then - retrycmd_if_failure 100 1 10 systemctl daemon-reload && systemctl restart kubelet - fi + retrycmd_if_failure 10d 1 10 systemctl daemon-reload && systemctl restart kubelet + retrycmd_if_failure 10 1 3 systemctl status kubelet --no-pager -l > /var/log/azure/kubelet-status.log } function extractHyperkube(){ @@ -353,36 +351,6 @@ function ensureK8s() { exit 3 fi ensurePodSecurityPolicy - for i in {1..1800}; do - nodes=$(${KUBECTL} get nodes 2>/dev/null | grep 'Ready' | wc -l) - if [ $nodes -eq $TOTAL_NODES ] - then - echo "all nodes are participating, took $i seconds" - nodesActive=0 - break - fi - sleep 1 - done - if [ $nodesActive -ne 0 ] - then - echo "still waiting for active nodes after $i seconds" - exit 3 - fi - for i in {1..600}; do - notReady=$(${KUBECTL} get nodes 2>/dev/null | grep 'NotReady' | wc -l) - if [ $notReady -eq 0 ] - then - echo "all nodes are Ready, took $i seconds" - nodesReady=0 - break - fi - sleep 1 - done - if [ $nodesReady -ne 0 ] - then - echo "still waiting for Ready nodes after $i seconds" - exit 3 - fi } function ensureEtcd() { diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index cd9466d5a3..812eb0af68 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -4,7 +4,6 @@ "maxVMsPerPool": 100, "apiServerCertificate": "[parameters('apiServerCertificate')]", {{ if not IsHostedMaster }} - "totalNodes": "[parameters('totalNodes')]", "apiServerPrivateKey": "[parameters('apiServerPrivateKey')]", "etcdServerCertificate": "[parameters('etcdServerCertificate')]", "etcdServerPrivateKey": "[parameters('etcdServerPrivateKey')]", @@ -215,7 +214,7 @@ "provisionSource": "{{GetKubernetesB64ProvisionSource}}", "mountetcdScript": "{{GetKubernetesB64Mountetcd}}", {{if not IsHostedMaster}} - "provisionScriptParametersMaster": "[concat('MASTER_NODE=true TOTAL_NODES=',variables('totalNodes'),' APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ADMINUSER=',variables('username'))]", + "provisionScriptParametersMaster": "[concat('MASTER_NODE=true APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ADMINUSER=',variables('username'))]", "provisionScriptParametersCommon": "[concat('TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',variables('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',variables('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',variables('clientPrivateKey'),' TARGET_ENVIRONMENT=',variables('targetEnvironment'),' NETWORK_POLICY=',variables('networkPolicy'),' FQDNSuffix=',variables('fqdnEndpointSuffix'),' VNET_CNI_PLUGINS_URL=',variables('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',variables('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',variables('cloudProviderBackoff'),' CLOUDPROVIDER_BACKOFF_RETRIES=',variables('cloudProviderBackoffRetries'),' CLOUDPROVIDER_BACKOFF_EXPONENT=',variables('cloudProviderBackoffExponent'),' CLOUDPROVIDER_BACKOFF_DURATION=',variables('cloudProviderBackoffDuration'),' CLOUDPROVIDER_BACKOFF_JITTER=',variables('cloudProviderBackoffJitter'),' CLOUDPROVIDER_RATELIMIT=',variables('cloudProviderRatelimit'),' CLOUDPROVIDER_RATELIMIT_QPS=',variables('cloudProviderRatelimitQPS'),' CLOUDPROVIDER_RATELIMIT_BUCKET=',variables('cloudProviderRatelimitBucket'),' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',variables('containerRuntime'),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]", {{else}} "provisionScriptParametersCommon": "[concat('TENANT_ID=',variables('tenantID'),' HYPERKUBE_URL=',variables('kubernetesHyperkubeSpec'),' APISERVER_PUBLIC_KEY=',variables('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',variables('clientPrivateKey'),' TARGET_ENVIRONMENT=',variables('targetEnvironment'),' NETWORK_POLICY=',variables('networkPolicy'),' FQDNSuffix=',variables('fqdnEndpointSuffix'),' VNET_CNI_PLUGINS_URL=',variables('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',variables('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',variables('cloudProviderBackoff'),' CLOUDPROVIDER_BACKOFF_RETRIES=',variables('cloudProviderBackoffRetries'),' CLOUDPROVIDER_BACKOFF_EXPONENT=',variables('cloudProviderBackoffExponent'),' CLOUDPROVIDER_BACKOFF_DURATION=',variables('cloudProviderBackoffDuration'),' CLOUDPROVIDER_BACKOFF_JITTER=',variables('cloudProviderBackoffJitter'),' CLOUDPROVIDER_RATELIMIT=',variables('cloudProviderRatelimit'),' CLOUDPROVIDER_RATELIMIT_QPS=',variables('cloudProviderRatelimitQPS'),' CLOUDPROVIDER_RATELIMIT_BUCKET=',variables('cloudProviderRatelimitBucket'),' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',variables('containerRuntime'))]", diff --git a/parts/k8s/kubernetesparams.t b/parts/k8s/kubernetesparams.t index 349a66ac89..1b220e968b 100644 --- a/parts/k8s/kubernetesparams.t +++ b/parts/k8s/kubernetesparams.t @@ -22,12 +22,6 @@ "type": "string" }, {{else}} - "totalNodes": { - "metadata": { - "description": "Number of nodes (masters + agents) in the cluster" - }, - "type": "int" - }, "etcdServerCertificate": { "metadata": { "description": "The base 64 server certificate used on the master" diff --git a/pkg/acsengine/engine.go b/pkg/acsengine/engine.go index 4866e07ef9..4612fdc9f9 100644 --- a/pkg/acsengine/engine.go +++ b/pkg/acsengine/engine.go @@ -678,9 +678,6 @@ func getParameters(cs *api.ContainerService, isClassicMode bool, generatorCode s addValue(parametersMap, "jumpboxPublicKey", cs.Properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.PublicKey) addValue(parametersMap, "jumpboxStorageProfile", cs.Properties.OrchestratorProfile.KubernetesConfig.PrivateCluster.JumpboxProfile.StorageProfile) } - if cs.Properties.HostedMasterProfile == nil { - addValue(parametersMap, "totalNodes", cs.Properties.TotalNodes()) - } if properties.OrchestratorProfile.KubernetesConfig == nil || !properties.OrchestratorProfile.KubernetesConfig.UseManagedIdentity { diff --git a/test/e2e/runner/cli_provisioner.go b/test/e2e/runner/cli_provisioner.go index aa0c0934a8..d6394c66a5 100644 --- a/test/e2e/runner/cli_provisioner.go +++ b/test/e2e/runner/cli_provisioner.go @@ -233,7 +233,7 @@ func (cli *CLIProvisioner) FetchProvisioningMetrics(path string, cfg *config.Con agentFiles := []string{"/var/log/azure/cluster-provision.log", "/var/log/cloud-init.log", "/var/log/cloud-init-output.log", "/var/log/syslog", "/var/log/azure/custom-script/handler.log", "/opt/m", "/opt/azure/containers/kubelet.sh", "/opt/azure/containers/provision.sh", - "/opt/azure/provision-ps.log", "/var/log/azure/dnsdump.pcap"} + "/opt/azure/provision-ps.log", "/var/log/azure/kubelet-status.log"} masterFiles := agentFiles masterFiles = append(masterFiles, "/opt/azure/containers/mountetcd.sh", "/opt/azure/containers/setup-etcd.sh", "/opt/azure/containers/setup-etcd.log") hostname := fmt.Sprintf("%s.%s.cloudapp.azure.com", cli.Config.Name, cli.Config.Location)