#1279 added retry with exponential backoff (#151)

pkg/lib/events.go

@@ -11,6 +11,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/keptn/go-utils/pkg/api/models"
 	"log"
+	"math/rand"
 	"net/url"
 	"time"
 )
@@ -19,6 +20,8 @@ import (
 	"encoding/json"
 )
 
+const MAX_SEND_RETRIES = 3
+
 // InternalProjectCreateEventType is a CloudEvent type for creating a new project
 const InternalProjectCreateEventType = "sh.keptn.internal.event.project.create"
 
@@ -534,8 +537,28 @@ func (k *Keptn) SendCloudEvent(event cloudevents.Event) error {
 		return errors.New("Failed to create HTTP client:" + err.Error())
 	}
 
-	if _, _, err := c.Send(context.Background(), event); err != nil {
-		return errors.New("Failed to send cloudevent:, " + err.Error())
+	for i := 0; i <= MAX_SEND_RETRIES; i++ {
+		_, _, err = c.Send(context.Background(), event)
+		if err == nil {
+			return nil
+		}
+		<-time.After(getExpBackoffTime(i + 1))
+	}
+	return errors.New("Failed to send cloudevent:, " + err.Error())
+}
+
+func getExpBackoffTime(retryNr int) time.Duration {
+	f := 1.5 * float64(retryNr)
+	if retryNr <= 1 {
+		f = 1.5
 	}
-	return nil
+	currentInterval := float64(500*time.Millisecond) * f
+	randomizationFactor := 0.5
+	random := rand.Float64()
+
+	var delta = randomizationFactor * currentInterval
+	minInterval := float64(currentInterval) - delta
+	maxInterval := float64(currentInterval) + delta
+
+	return time.Duration(minInterval + (random * (maxInterval - minInterval + 1)))
 }

pkg/lib/events_test.go

@@ -43,10 +43,15 @@ func getKeptnFields(ts *httptest.Server) fields {
 }
 
 func TestKeptn_SendCloudEvent(t *testing.T) {
-
+	failOnFirstTry := true
 	ts := httptest.NewServer(
 		http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			w.Header().Add("Content-Type", "application/json")
+			if failOnFirstTry {
+				failOnFirstTry = false
+				w.WriteHeader(500)
+				w.Write([]byte(`{}`))
+			}
 			w.WriteHeader(200)
 			w.Write([]byte(`{}`))
 		}),
@@ -207,3 +212,57 @@ func verifyReceivedEventType(receivedCorrectType chan bool, t *testing.T) {
 		t.Errorf("SendTestsFinishedEvent(): timed out waiting for event")
 	}
 }
+
+func Test_getExpBackoffTime(t *testing.T) {
+	type args struct {
+		retryNr int
+	}
+	type durationRange struct {
+		min time.Duration
+		max time.Duration
+	}
+	tests := []struct {
+		name string
+		args args
+		want durationRange
+	}{
+		{
+			name: "Get exponential backoff time (1)",
+			args: args{
+				retryNr: 1,
+			},
+			want: durationRange{
+				min: 375.0 * time.Millisecond,
+				max: 1125.0 * time.Millisecond,
+			},
+		},
+		{
+			name: "Get exponential backoff time (2)",
+			args: args{
+				retryNr: 2,
+			},
+			want: durationRange{
+				min: 750.0 * time.Millisecond,
+				max: 2250.0 * time.Millisecond,
+			},
+		},
+		{
+			name: "Get exponential backoff time (3)",
+			args: args{
+				retryNr: 3,
+			},
+			want: durationRange{
+				min: 1125.0 * time.Millisecond,
+				max: 3375.0 * time.Millisecond,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := getExpBackoffTime(tt.args.retryNr)
+			if got < tt.want.min || got > tt.want.max {
+				t.Errorf("getExpBackoffTime() = %v, want [%v,%v]", got, tt.want.min, tt.want.max)
+			}
+		})
+	}
+}