Retry docker manifest push on network failure

We can encounter network issues on pushing the manifest as described in: #2810 We now retry in case of that network error up to 5 times by using an exponential back off timer. Signed-off-by: Sascha Grunert <[email protected]>
kubernetes · Dec 13, 2022 · 43e131f · 43e131f
1 parent 6c047c2
commit 43e131f
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 4 deletions.
diff --git a/go.mod b/go.mod
@@ -36,6 +36,7 @@ require (
 	golang.org/x/text v0.5.0
 	google.golang.org/api v0.103.0
 	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/apimachinery v0.26.0
 	sigs.k8s.io/bom v0.4.1
 	sigs.k8s.io/mdtoc v1.1.0
 	sigs.k8s.io/promo-tools/v3 v3.4.10
@@ -337,7 +338,6 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	helm.sh/helm/v3 v3.10.0 // indirect
 	k8s.io/api v0.25.0 // indirect
-	k8s.io/apimachinery v0.26.0 // indirect
 	k8s.io/cli-runtime v0.25.0 // indirect
 	k8s.io/client-go v0.25.0 // indirect
 	k8s.io/klog/v2 v2.80.1 // indirect

diff --git a/pkg/release/images.go b/pkg/release/images.go
@@ -22,9 +22,11 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"time"
 
 	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"sigs.k8s.io/release-sdk/sign"
 	"sigs.k8s.io/release-utils/command"
@@ -182,9 +184,24 @@ func (i *Images) Publish(registry, version, buildPath string) error {
 		}
 
 		logrus.Infof("Pushing manifest image %s", imageVersion)
-		if err := i.Execute(
-			"docker", "manifest", "push", imageVersion, "--purge",
-		); err != nil {
+		if err := wait.ExponentialBackoff(wait.Backoff{
+			Duration: time.Second,
+			Factor:   1.5,
+			Steps:    5,
+		}, func() (bool, error) {
+			if err := i.Execute("docker", "manifest", "push", imageVersion, "--purge"); err == nil {
+				return true, nil
+			} else if strings.Contains(err.Error(), "request canceled while waiting for connection") {
+				// The error is unfortunately not exported:
+				// https://github.com/golang/go/blob/dc04f3b/src/net/http/client.go#L720
+				// https://github.com/golang/go/blob/dc04f3b/src/net/http/transport.go#L2518
+				// ref: https://github.com/kubernetes/release/issues/2810
+				logrus.Info("Retrying manifest push")
+				return false, nil
+			}
+
+			return false, err
+		}); err != nil {
 			return fmt.Errorf("push manifest: %w", err)
 		}