Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

connectivity: Retry on inconclusive results #1543

Merged
merged 1 commit into from
Apr 28, 2023
Merged

connectivity: Retry on inconclusive results #1543

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 30 additions & 5 deletions connectivity/check/action.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package check

import (
"bytes"
"context"
"encoding/base64"
"errors"
Expand All @@ -28,6 +29,10 @@ import (
"github.com/cilium/cilium-cli/defaults"
)

const (
testCommandRetries = 3
)

// Action represents an individual action (e.g. a curl call) in a Scenario
// between a source and a destination peer.
type Action struct {
Expand Down Expand Up @@ -236,13 +241,33 @@ func (a *Action) ExecInPod(ctx context.Context, cmd []string) {
pod := a.src

a.Debug("Executing command", cmd)

output, err := pod.K8sClient.ExecInPod(ctx,
pod.Pod.Namespace, pod.Pod.Name, pod.Pod.Labels["name"], cmd)

cmdName := cmd[0]
cmdStr := strings.Join(cmd, " ")
a.cmdOutput = output.String()

var output bytes.Buffer
var err error
// We retry the command in case of inconclusive results. The result is
// deemed inconclusive when the command succeeded, but we don't have any
// output. We've seen this happen when there are connectivity blips on the
// k8s side.
// This check currently only works because all our test commands expect an
// output.
for i := 1; i <= testCommandRetries; i++ {
output, err = pod.K8sClient.ExecInPod(ctx,
pod.Pod.Namespace, pod.Pod.Name, pod.Pod.Labels["name"], cmd)
a.cmdOutput = output.String()
// Check for inconclusive results.
if err == nil && strings.TrimSpace(a.cmdOutput) == "" {
a.Debugf("retrying command %s due to inconclusive results", cmdStr)
continue
}
break
}
// Check for inconclusive results.
if err == nil && strings.TrimSpace(a.cmdOutput) == "" {
a.Failf("inconclusive results: command %q was successful but without output", cmdStr)
}

showOutput := false
expectedExitCode := a.expectedExitCode()
if err != nil {
Expand Down