diff --git a/pkg/lib/events.go b/pkg/lib/events.go index c09e53b4..ddec556e 100644 --- a/pkg/lib/events.go +++ b/pkg/lib/events.go @@ -11,6 +11,7 @@ import ( "github.com/google/uuid" "github.com/keptn/go-utils/pkg/api/models" "log" + "math/rand" "net/url" "time" ) @@ -19,6 +20,8 @@ import ( "encoding/json" ) +const MAX_SEND_RETRIES = 3 + // InternalProjectCreateEventType is a CloudEvent type for creating a new project const InternalProjectCreateEventType = "sh.keptn.internal.event.project.create" @@ -534,8 +537,28 @@ func (k *Keptn) SendCloudEvent(event cloudevents.Event) error { return errors.New("Failed to create HTTP client:" + err.Error()) } - if _, _, err := c.Send(context.Background(), event); err != nil { - return errors.New("Failed to send cloudevent:, " + err.Error()) + for i := 0; i <= MAX_SEND_RETRIES; i++ { + _, _, err = c.Send(context.Background(), event) + if err == nil { + return nil + } + <-time.After(getExpBackoffTime(i + 1)) + } + return errors.New("Failed to send cloudevent:, " + err.Error()) +} + +func getExpBackoffTime(retryNr int) time.Duration { + f := 1.5 * float64(retryNr) + if retryNr <= 1 { + f = 1.5 } - return nil + currentInterval := float64(500*time.Millisecond) * f + randomizationFactor := 0.5 + random := rand.Float64() + + var delta = randomizationFactor * currentInterval + minInterval := float64(currentInterval) - delta + maxInterval := float64(currentInterval) + delta + + return time.Duration(minInterval + (random * (maxInterval - minInterval + 1))) } diff --git a/pkg/lib/events_test.go b/pkg/lib/events_test.go index 642921ed..356c5eed 100644 --- a/pkg/lib/events_test.go +++ b/pkg/lib/events_test.go @@ -43,10 +43,15 @@ func getKeptnFields(ts *httptest.Server) fields { } func TestKeptn_SendCloudEvent(t *testing.T) { - + failOnFirstTry := true ts := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", "application/json") + if failOnFirstTry { + failOnFirstTry = false + w.WriteHeader(500) + w.Write([]byte(`{}`)) + } w.WriteHeader(200) w.Write([]byte(`{}`)) }), @@ -207,3 +212,57 @@ func verifyReceivedEventType(receivedCorrectType chan bool, t *testing.T) { t.Errorf("SendTestsFinishedEvent(): timed out waiting for event") } } + +func Test_getExpBackoffTime(t *testing.T) { + type args struct { + retryNr int + } + type durationRange struct { + min time.Duration + max time.Duration + } + tests := []struct { + name string + args args + want durationRange + }{ + { + name: "Get exponential backoff time (1)", + args: args{ + retryNr: 1, + }, + want: durationRange{ + min: 375.0 * time.Millisecond, + max: 1125.0 * time.Millisecond, + }, + }, + { + name: "Get exponential backoff time (2)", + args: args{ + retryNr: 2, + }, + want: durationRange{ + min: 750.0 * time.Millisecond, + max: 2250.0 * time.Millisecond, + }, + }, + { + name: "Get exponential backoff time (3)", + args: args{ + retryNr: 3, + }, + want: durationRange{ + min: 1125.0 * time.Millisecond, + max: 3375.0 * time.Millisecond, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getExpBackoffTime(tt.args.retryNr) + if got < tt.want.min || got > tt.want.max { + t.Errorf("getExpBackoffTime() = %v, want [%v,%v]", got, tt.want.min, tt.want.max) + } + }) + } +}