Skip to content
This repository has been archived by the owner on Jan 11, 2023. It is now read-only.

E2E: rationalize node check + kube-system check, no kms #4273

Merged
merged 2 commits into from
Nov 21, 2018
+29 −10
Merged

E2E: rationalize node check + kube-system check, no kms #4273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
977f737
e2e tweaks plus disable kms
jackfrancis Nov 21, 2018
20f33e8
prop doesn’t exist
jackfrancis Nov 21, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ jobs:
echo 'export ORCHESTRATOR_RELEASE=1.10' >> $BASH_ENV
echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
echo 'export CREATE_VNET=true' >> $BASH_ENV
echo 'export ENABLE_KMS_ENCRYPTION=true' >> $BASH_ENV
echo 'export ENABLE_KMS_ENCRYPTION=false' >> $BASH_ENV
echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV
echo 'export CLEANUP_IF_FAIL=${CLEANUP_IF_FAIL_LINUX}' >> $BASH_ENV
echo 'export RETAIN_SSH=false' >> $BASH_ENV
Expand All @@ -98,7 +98,7 @@ jobs:
echo 'export ORCHESTRATOR_RELEASE=1.11' >> $BASH_ENV
echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
echo 'export CREATE_VNET=true' >> $BASH_ENV
echo 'export ENABLE_KMS_ENCRYPTION=true' >> $BASH_ENV
echo 'export ENABLE_KMS_ENCRYPTION=false' >> $BASH_ENV
echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV
echo 'export CLEANUP_IF_FAIL=${CLEANUP_IF_FAIL_LINUX}' >> $BASH_ENV
echo 'export RETAIN_SSH=false' >> $BASH_ENV
Expand All @@ -124,7 +124,7 @@ jobs:
echo 'export ORCHESTRATOR_RELEASE=1.12' >> $BASH_ENV
echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
echo 'export CREATE_VNET=true' >> $BASH_ENV
echo 'export ENABLE_KMS_ENCRYPTION=true' >> $BASH_ENV
echo 'export ENABLE_KMS_ENCRYPTION=false' >> $BASH_ENV
echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV
echo 'export CLEANUP_IF_FAIL=${CLEANUP_IF_FAIL_LINUX}' >> $BASH_ENV
echo 'export RETAIN_SSH=false' >> $BASH_ENV
Expand All @@ -150,7 +150,7 @@ jobs:
echo 'export ORCHESTRATOR_RELEASE=1.13' >> $BASH_ENV
echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
echo 'export CREATE_VNET=true' >> $BASH_ENV
echo 'export ENABLE_KMS_ENCRYPTION=true' >> $BASH_ENV
echo 'export ENABLE_KMS_ENCRYPTION=false' >> $BASH_ENV
echo 'export CLEANUP_ON_EXIT=${CLEANUP_ON_EXIT}' >> $BASH_ENV
echo 'export CLEANUP_IF_FAIL=${CLEANUP_IF_FAIL_LINUX}' >> $BASH_ENV
echo 'export RETAIN_SSH=false' >> $BASH_ENV
Expand Down
3 changes: 3 additions & 0 deletions test/e2e/engine/template.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,9 @@ func Build(cfg *config.Config, masterSubnetID string, agentSubnetID string, isVM
}

if config.EnableKMSEncryption && config.ClientObjectID != "" {
if cs.ContainerService.Properties.OrchestratorProfile.KubernetesConfig == nil {
cs.ContainerService.Properties.OrchestratorProfile.KubernetesConfig = &vlabs.KubernetesConfig{}
}
cs.ContainerService.Properties.OrchestratorProfile.KubernetesConfig.EnableEncryptionWithExternalKms = &config.EnableKMSEncryption
cs.ContainerService.Properties.ServicePrincipalProfile.ObjectID = config.ClientObjectID
}
Expand Down
13 changes: 9 additions & 4 deletions test/e2e/kubernetes/kubernetes_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,15 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
}
})

It("should have have the appropriate node count", func() {
nodeList, err := node.Get()
Expect(err).NotTo(HaveOccurred())
Expect(len(nodeList.Nodes)).To(Equal(eng.NodeCount()))
It("should report all nodes in a Ready state", func() {
ready := node.WaitOnReady(eng.NodeCount(), 10*time.Second, cfg.Timeout)
cmd := exec.Command("kubectl", "get", "nodes", "-o", "wide")
out, _ := cmd.CombinedOutput()
log.Printf("%s\n", out)
if !ready {
log.Printf("Error: Not all nodes in a healthy state\n")
}
Expect(ready).To(Equal(true))
})

It("should have DNS pod running", func() {
Expand Down
12 changes: 10 additions & 2 deletions test/e2e/kubernetes/node/node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,22 @@ type List struct {
// AreAllReady returns a bool depending on cluster state
func AreAllReady(nodeCount int) bool {
list, _ := Get()
var ready int
if list != nil && len(list.Nodes) == nodeCount {
for _, node := range list.Nodes {
nodeReady := false
for _, condition := range node.Status.Conditions {
if condition.Type == "KubeletReady" && condition.Status == "false" {
return false
if condition.Type == "Ready" && condition.Status == "True" {
ready++
nodeReady = true
}
}
if !nodeReady {
return false
}
}
}
if ready == nodeCount {
return true
}
return false
Expand Down
3 changes: 3 additions & 0 deletions test/e2e/runner/cli_provisioner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,9 @@ func (cli *CLIProvisioner) waitForNodes() error {
if !cli.IsPrivate() {
log.Println("Waiting on nodes to go into ready state...")
ready := node.WaitOnReady(cli.Engine.NodeCount(), 10*time.Second, cli.Config.Timeout)
cmd := exec.Command("kubectl", "get", "nodes", "-o", "wide")
out, _ := cmd.CombinedOutput()
log.Printf("%s\n", out)
if !ready {
return errors.New("Error: Not all nodes in a healthy state")
}
Expand Down