Skip to content

Commit

Permalink
test/e2e: More refactoring
Browse files Browse the repository at this point in the history 
Rather than poll all of the daemons, add a helper that waits
for a pool to complete a config.

One of our tests walks over the MCDs, change it to just assert
on all of the nodes.

The SSH test can also just wait for a pool and then `rsh` to
each node.
  • Loading branch information
@cgwalters
cgwalters committed May 17, 2019
1 parent 1c05bc8 commit 9970f24
Showing 1 changed file with 82 additions and 90 deletions.
172 changes: 82 additions & 90 deletions test/e2e/mcd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"time"

igntypes "github.com/coreos/ignition/config/v2_2/types"
"github.com/stretchr/testify/assert"
mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common"
"github.com/openshift/machine-config-operator/pkg/daemon/constants"
Expand Down Expand Up @@ -93,23 +94,49 @@ func createMCToAddFile(name, filename, data, fs string) *mcv1.MachineConfig {
// included the given mcName in its config, and returns the new
// rendered config name.
func waitForRenderedConfig(t *testing.T, cs *framework.ClientSet, pool, mcName string) (string, error) {
var newMCName string
var renderedConfig string
startTime := time.Now()
if err := wait.PollImmediate(2*time.Second, 5*time.Minute, func() (bool, error) {
mcp, err := cs.MachineConfigPools().Get(pool, metav1.GetOptions{})
if err != nil {
return false, err
}
for _, mc := range mcp.Status.Configuration.Source {
if mc.Name == mcName {
newMCName = mcp.Status.Configuration.Name
renderedConfig = mcp.Status.Configuration.Name
return true, nil
}
}
return false, nil
}); err != nil {
return "", errors.Wrapf(err, "machine config %s hasn't been picked by pool %s", mcName, pool)
}
return newMCName, nil
t.Logf("Pool %s has rendered config %s with %s (waited %v)", pool, mcName, renderedConfig, time.Since(startTime))
return renderedConfig, nil
}

// waitForPoolComplete polls a pool until it has completed an update to target
func waitForPoolComplete(t *testing.T, cs *framework.ClientSet, pool, target string) error {
// We need a spec and status separation https://github.com/openshift/machine-config-operator/pull/765#issuecomment-493136113
time.Sleep(time.Second * 13)
startTime := time.Now()
if err := wait.Poll(2*time.Second, 10*time.Minute, func() (bool, error) {
mcp, err := cs.MachineConfigPools().Get(pool, metav1.GetOptions{})
if err != nil {
return false, err
}
if mcp.Status.Configuration.Name != target {
return false, nil
}
if mcv1.IsMachineConfigPoolConditionTrue(mcp.Status.Conditions, mcv1.MachineConfigPoolUpdated) {
return true, nil
}
return false, nil
}); err != nil {
return errors.Wrapf(err, "pool %s didn't report updated to %s", pool, target)
}
t.Logf("Pool %s has completed %s (waited %v)", pool, target, time.Since(startTime))
return nil
}

func TestMCDeployed(t *testing.T) {
Expand All @@ -118,6 +145,7 @@ func TestMCDeployed(t *testing.T) {

// TODO: bring this back to 10
for i := 0; i < 5; i++ {
startTime := time.Now()
mcadd := createMCToAddFile("add-a-file", fmt.Sprintf("/etc/mytestconf%d", i), "test", "root")

// create the dummy MC now
Expand All @@ -126,34 +154,23 @@ func TestMCDeployed(t *testing.T) {
t.Errorf("failed to create machine config %v", err)
}

newMCName, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
t.Logf("Created %s", mcadd.Name)
renderedConfig, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
if err != nil {
t.Errorf("%v", err)
}

visited := make(map[string]bool)
if err := wait.Poll(2*time.Second, 30*time.Minute, func() (bool, error) {
nodes, err := getNodesByRole(cs, "worker")
if err != nil {
return false, nil
}
for _, node := range nodes {
if visited[node.Name] {
continue
}
if node.Annotations[constants.CurrentMachineConfigAnnotationKey] == newMCName &&
node.Annotations[constants.MachineConfigDaemonStateAnnotationKey] == constants.MachineConfigDaemonStateDone {
visited[node.Name] = true
if len(visited) == len(nodes) {
return true, nil
}
continue
}
}
return false, nil
}); err != nil {
t.Errorf("machine config didn't result in file being on any worker: %v", err)
if err := waitForPoolComplete(t, cs, "worker", renderedConfig); err != nil {
t.Fatal(err)
}
nodes, err := getNodesByRole(cs, "worker")
if err != nil {
t.Fatal(err)
}
for _, node := range nodes {
assert.Equal(t, renderedConfig, node.Annotations[constants.CurrentMachineConfigAnnotationKey])
assert.Equal(t, constants.MachineConfigDaemonStateDone, node.Annotations[constants.MachineConfigDaemonStateAnnotationKey])
}
t.Logf("All nodes updated with %s (%s elapsed)", mcadd.Name, time.Since(startTime))
}
}

Expand Down Expand Up @@ -199,65 +216,49 @@ func TestUpdateSSH(t *testing.T) {
if err != nil {
t.Errorf("failed to create machine config %v", err)
}
t.Logf("Created %s", mcadd.Name)

// grab the latest worker- MC
newMCName, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
renderedConfig, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
if err != nil {
t.Errorf("%v", err)
t.Fatal(err)
}
if err := waitForPoolComplete(t, cs, "worker", renderedConfig); err != nil {
t.Fatal(err)
}
nodes, err := getNodesByRole(cs, "worker")
if err != nil {
t.Fatal(err)
}
for _, node := range nodes {
assert.Equal(t, node.Annotations[constants.CurrentMachineConfigAnnotationKey], renderedConfig)
assert.Equal(t, node.Annotations[constants.MachineConfigDaemonStateAnnotationKey], constants.MachineConfigDaemonStateDone)
// find the MCD pod that has spec.nodeNAME = node.Name and get its name:
listOptions := metav1.ListOptions{
FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": node.Name}).String(),
}
listOptions.LabelSelector = labels.SelectorFromSet(labels.Set{"k8s-app": "machine-config-daemon"}).String()

visited := make(map[string]bool)
if err := wait.Poll(2*time.Second, 10*time.Minute, func() (bool, error) {
nodes, err := getNodesByRole(cs, "worker")
mcdList, err := cs.Pods("openshift-machine-config-operator").List(listOptions)
if err != nil {
return false, err
t.Fatal(err)
}
for _, node := range nodes {
if visited[node.Name] {
continue
}
// check that the new MC is in the annotations and that the MCD state is Done.
if node.Annotations[constants.CurrentMachineConfigAnnotationKey] == newMCName &&
node.Annotations[constants.MachineConfigDaemonStateAnnotationKey] == constants.MachineConfigDaemonStateDone {
// find the MCD pod that has spec.nodeNAME = node.Name and get its name:
listOptions := metav1.ListOptions{
FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": node.Name}).String(),
}
listOptions.LabelSelector = labels.SelectorFromSet(labels.Set{"k8s-app": "machine-config-daemon"}).String()

mcdList, err := cs.Pods("openshift-machine-config-operator").List(listOptions)
if err != nil {
return false, nil
}
if len(mcdList.Items) != 1 {
t.Logf("did not find any mcd pods")
return false, nil
}
mcdName := mcdList.Items[0].ObjectMeta.Name

// now rsh into that daemon and grep the authorized key file to check if 1234_test was written
// must do both commands in same shell, combine commands into one exec.Command()
found, err := exec.Command("oc", "rsh", "-n", "openshift-machine-config-operator", mcdName,
"grep", "1234_test", "/rootfs/home/core/.ssh/authorized_keys").CombinedOutput()
if err != nil {
t.Logf("unable to read authorized_keys on daemon: %s got: %s got err: %v", mcdName, found, err)
return false, nil
}
if !strings.Contains(string(found), "1234_test") {
t.Logf("updated ssh keys not found in authorized_keys, got %s", found)
return false, nil
}
if len(mcdList.Items) != 1 {
t.Fatalf("Failed to find MCD for node %s", node.Name)
}
mcdName := mcdList.Items[0].ObjectMeta.Name

visited[node.Name] = true
if len(visited) == len(nodes) {
return true, nil
}
continue
}
// now rsh into that daemon and grep the authorized key file to check if 1234_test was written
// must do both commands in same shell, combine commands into one exec.Command()
found, err := exec.Command("oc", "rsh", "-n", "openshift-machine-config-operator", mcdName,
"grep", "1234_test", "/rootfs/home/core/.ssh/authorized_keys").CombinedOutput()
if err != nil {
t.Fatalf("unable to read authorized_keys on daemon: %s got: %s got err: %v", mcdName, found, err)
}
return false, nil
}); err != nil {
t.Errorf("machine config didn't result in ssh keys being on any worker: %v", err)
if !strings.Contains(string(found), "1234_test") {
t.Fatalf("updated ssh keys not found in authorized_keys, got %s", found)
}
t.Logf("Node %s has SSH key", node.Name)
}
}

Expand Down Expand Up @@ -340,7 +341,7 @@ func TestReconcileAfterBadMC(t *testing.T) {
t.Errorf("failed to create machine config %v", err)
}

newMCName, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
renderedConfig, err := waitForRenderedConfig(t, cs, "worker", mcadd.Name)
if err != nil {
t.Errorf("%v", err)
}
Expand All @@ -352,7 +353,7 @@ func TestReconcileAfterBadMC(t *testing.T) {
return false, err
}
for _, node := range nodes {
if node.Annotations[constants.DesiredMachineConfigAnnotationKey] == newMCName &&
if node.Annotations[constants.DesiredMachineConfigAnnotationKey] == renderedConfig &&
node.Annotations[constants.MachineConfigDaemonStateAnnotationKey] != constants.MachineConfigDaemonStateDone {
// just check that we have the annotation here, w/o strings checking anything that can flip fast causing flakes
if node.Annotations[constants.MachineConfigDaemonReasonAnnotationKey] != "" {
Expand Down Expand Up @@ -385,17 +386,8 @@ func TestReconcileAfterBadMC(t *testing.T) {
}

// wait for the mcp to go back to previous config
if err := wait.PollImmediate(2*time.Second, 5*time.Minute, func() (bool, error) {
mcp, err := cs.MachineConfigPools().Get("worker", metav1.GetOptions{})
if err != nil {
return false, err
}
if mcp.Status.Configuration.Name == workerOldMc {
return true, nil
}
return false, nil
}); err != nil {
t.Errorf("old machine config hasn't been picked by the pool: %v", err)
if err := waitForPoolComplete(t, cs, "worker", workerOldMc); err != nil {
t.Fatal(err)
}

visited := make(map[string]bool)
Expand Down

0 comments on commit 9970f24

Please sign in to comment.