fix: fix issue where file is read before written in e2e tests

test/const/test_consts.go

@@ -81,9 +81,10 @@ const (
 	// Description that will printed during tests
 	FailedConditionDescription = "Error status code"
 
-	Poll            = 2 * time.Second
-	PollLongTimeout = 5 * time.Minute
-	PollTimeout     = 10 * time.Minute
+	Poll                 = 2 * time.Second
+	PollLongTimeout      = 5 * time.Minute
+	PollTimeout          = 10 * time.Minute
+	PollForStringTimeout = 1 * time.Minute
 
 	testTolerationKey   = "test-toleration-key"
 	testTolerationValue = "test-toleration-value"

test/e2e/testsuites/dynamically_provisioned_delete_pod_tester.go

@@ -17,8 +17,6 @@ limitations under the License.
 package testsuites
 
 import (
-	"time"
-
 	"sigs.k8s.io/azuredisk-csi-driver/test/e2e/driver"
 	"sigs.k8s.io/azuredisk-csi-driver/test/resources"
 
@@ -55,9 +53,8 @@ func (t *DynamicallyProvisionedDeletePodTest) Run(client clientset.Interface, na
 	tDeployment.WaitForPodReady()
 
 	if t.PodCheck != nil {
-		ginkgo.By("sleep 5s and then check pod exec")
-		time.Sleep(5 * time.Second)
-		tDeployment.Exec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
+		ginkgo.By("check pod exec")
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
 	}
 
 	ginkgo.By("deleting the pod for deployment")
@@ -67,9 +64,8 @@ func (t *DynamicallyProvisionedDeletePodTest) Run(client clientset.Interface, na
 	tDeployment.WaitForPodReady()
 
 	if t.PodCheck != nil {
-		ginkgo.By("sleep 5s and then check pod exec after pod restart again")
-		time.Sleep(5 * time.Second)
+		ginkgo.By("check pod exec after pod restart again")
 		// pod will be restarted so expect to see 2 instances of string
-		tDeployment.Exec(t.PodCheck.Cmd, t.PodCheck.ExpectedString+t.PodCheck.ExpectedString)
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString+t.PodCheck.ExpectedString)
 	}
 }

test/e2e/testsuites/dynamically_provisioned_pod_failover_tester.go

@@ -17,8 +17,6 @@ limitations under the License.
 package testsuites
 
 import (
-	"time"
-
 	"github.com/onsi/ginkgo"
 	v1 "k8s.io/api/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
@@ -58,9 +56,8 @@ func (t *PodFailover) Run(client clientset.Interface, namespace *v1.Namespace, s
 	tDeployment.WaitForPodReady()
 
 	if t.PodCheck != nil {
-		ginkgo.By("sleep 3s and then check pod exec")
-		time.Sleep(3 * time.Second)
-		tDeployment.Exec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
+		ginkgo.By("check pod exec")
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
 	}
 
 	ginkgo.By("cordoning node 0")
@@ -80,9 +77,8 @@ func (t *PodFailover) Run(client clientset.Interface, namespace *v1.Namespace, s
 	tDeployment.WaitForPodReady()
 
 	if t.PodCheck != nil {
-		ginkgo.By("sleep 3s and then check pod exec")
-		time.Sleep(3 * time.Second)
+		ginkgo.By("check pod exec")
 		// pod will be restarted so expect to see 2 instances of string
-		tDeployment.Exec(t.PodCheck.Cmd, t.PodCheck.ExpectedString+t.PodCheck.ExpectedString)
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString+t.PodCheck.ExpectedString)
 	}
 }

test/e2e/testsuites/dynamically_provisioned_pod_failover_with_replicas_tester.go

@@ -71,9 +71,8 @@ func (t *PodFailoverWithReplicas) Run(client clientset.Interface, namespace *v1.
 	tDeployment.WaitForPodReady()
 
 	if t.PodCheck != nil {
-		ginkgo.By("sleep 3s and then check pod exec")
-		time.Sleep(3 * time.Second)
-		tDeployment.Exec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
+		ginkgo.By("check pod exec")
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
 	}
 
 	//Check that AzVolumeAttachment resources were created correctly
@@ -144,10 +143,9 @@ func (t *PodFailoverWithReplicas) Run(client clientset.Interface, namespace *v1.
 	tDeployment.WaitForPodReady()
 
 	if t.PodCheck != nil {
-		ginkgo.By("sleep 3s and then check pod exec")
-		time.Sleep(3 * time.Second)
+		ginkgo.By("check pod exec")
 		// pod will be restarted so expect to see 2 instances of string
-		tDeployment.Exec(t.PodCheck.Cmd, t.PodCheck.ExpectedString+t.PodCheck.ExpectedString)
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString+t.PodCheck.ExpectedString)
 	}
 
 	ginkgo.By("Verifying that pod got created on the correct node.")

test/e2e/testsuites/dynamically_provisioned_shared_disk_tester.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
@@ -66,9 +65,8 @@ func (t *DynamicallyProvisionedSharedDiskTester) Run(client clientset.Interface,
 	tDeployment.WaitForPodReady()
 
 	if t.PodCheck != nil {
-		ginkgo.By("sleep 5s and then check pod exec")
-		time.Sleep(5 * time.Second)
-		tDeployment.Exec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
+		ginkgo.By("check pod exec")
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
 	}
 
 	ginkgo.By("verifying shared attachment")

test/e2e/testsuites/dynamically_provisioned_statefulset_e2e_tester.go

@@ -51,9 +51,8 @@ func (t *DynamicallyProvisionedStatefulSetTest) Run(client clientset.Interface,
 	framework.ExpectNoError(err)
 
 	if t.PodCheck != nil {
-		ginkgo.By("sleep 5s and then check pod exec")
-		time.Sleep(5 * time.Second)
-		tStatefulSet.Exec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
+		ginkgo.By("check pod exec")
+		tStatefulSet.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
 	}
 
 	ginkgo.By("deleting the pod for statefulset")
@@ -64,9 +63,8 @@ func (t *DynamicallyProvisionedStatefulSetTest) Run(client clientset.Interface,
 	framework.ExpectNoError(err)
 
 	if t.PodCheck != nil {
-		ginkgo.By("sleep 5s and then check pod exec after pod restart again")
-		time.Sleep(5 * time.Second)
+		ginkgo.By("check pod exec after pod restart again")
 		// pod will be restarted so expect to see 2 instances of string
-		tStatefulSet.Exec(t.PodCheck.Cmd, t.PodCheck.ExpectedString+t.PodCheck.ExpectedString)
+		tStatefulSet.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString+t.PodCheck.ExpectedString)
 	}
 }

test/e2e/testsuites/pre_provisioned_shared_disk_tester.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/onsi/ginkgo"
 	"github.com/onsi/gomega"
@@ -66,9 +65,8 @@ func (t *PreProvisionedSharedDiskTester) Run(client clientset.Interface, namespa
 	tDeployment.WaitForPodReady()
 
 	if t.PodCheck != nil {
-		ginkgo.By("sleep 5s and then check pod exec")
-		time.Sleep(5 * time.Second)
-		tDeployment.Exec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
+		ginkgo.By("check pod exec")
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
 	}
 
 	ginkgo.By("verifying shared attachment")

test/resources/deployment.go

@@ -173,11 +173,16 @@ func (t *TestDeployment) WaitForPodReady() {
 	}
 }
 
-func (t *TestDeployment) Exec(command []string, expectedString string) {
-	for _, pod := range t.Pods {
-		_, err := framework.LookForStringInPodExec(t.Namespace.Name, pod.Name, command, expectedString, testconsts.ExecTimeout)
-		framework.ExpectNoError(err)
+func (t *TestDeployment) podNames() []string {
+	names := make([]string, 0, len(t.Pods))
+	for _, podDetails := range t.Pods {
+		names = append(names, podDetails.Name)
 	}
+	return names
+}
+
+func (t *TestDeployment) PollForStringInPodsExec(command []string, expectedString string) {
+	pollForStringInPodsExec(t.Namespace.Name, t.podNames(), command, expectedString)
 }
 
 func (t *TestDeployment) DeletePodAndWait() {

test/resources/statefulset.go

@@ -182,11 +182,8 @@ func (t *TestStatefulset) WaitForPodReadyOrFail() error {
 	return err
 }
 
-func (t *TestStatefulset) Exec(command []string, expectedString string) {
-	for _, podName := range t.PodNames {
-		_, err := framework.LookForStringInPodExec(t.Namespace.Name, podName, command, expectedString, testconsts.ExecTimeout)
-		framework.ExpectNoError(err)
-	}
+func (t *TestStatefulset) PollForStringInPodsExec(command []string, expectedString string) {
+	pollForStringInPodsExec(t.Namespace.Name, t.PodNames, command, expectedString)
 }
 
 func (t *TestStatefulset) DeletePodAndWait() {

test/resources/utils.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/onsi/ginkgo"
@@ -30,6 +31,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/labels"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
@@ -97,6 +99,36 @@ func VerifySuccessfulAzVolumeAttachments(pod PodDetails, azDiskClient *azDiskCli
 	return
 }
 
+func pollForStringWorker(namespace string, pod string, command []string, expectedString string, ch chan<- error) {
+	args := append([]string{"exec", pod, "--"}, command...)
+	err := wait.PollImmediate(testconsts.Poll, testconsts.PollForStringTimeout, func() (bool, error) {
+		stdout, err := framework.RunKubectl(namespace, args...)
+		if err != nil {
+			framework.Logf("Error waiting for output %q in pod %q: %v.", expectedString, pod, err)
+			return false, nil
+		}
+		if !strings.Contains(stdout, expectedString) {
+			framework.Logf("The stdout did not contain output %q in pod %q, found: %q.", expectedString, pod, stdout)
+			return false, nil
+		}
+		return true, nil
+	})
+	ch <- err
+}
+
+// Execute the command for all pods in the namespace, looking for expectedString in stdout
+func pollForStringInPodsExec(namespace string, pods []string, command []string, expectedString string) {
+	ch := make(chan error, len(pods))
+	for _, pod := range pods {
+		go pollForStringWorker(namespace, pod, command, expectedString, ch)
+	}
+	errs := make([]error, 0, len(pods))
+	for range pods {
+		errs = append(errs, <-ch)
+	}
+	framework.ExpectNoError(utilerrors.NewAggregate(errs), "Failed to find %q in at least one pod's output.", expectedString)
+}
+
 func generatePVC(namespace, storageClassName, name, claimSize string, volumeMode v1.PersistentVolumeMode, accessMode v1.PersistentVolumeAccessMode, dataSource *v1.TypedLocalObjectReference) *v1.PersistentVolumeClaim {
 	if accessMode != v1.ReadWriteOnce && accessMode != v1.ReadOnlyMany && accessMode != v1.ReadWriteMany {
 		accessMode = v1.ReadWriteOnce