Use variable for err message and try fleet addresses until success

sourcemap/es_store.go

@@ -42,7 +42,7 @@ const (
 )
 
 var (
-	errMsgESFailure         = "failure querying ES"
+	errMsgESFailure         = errMsgFailure + " ES"
 	errSourcemapWrongFormat = errors.New("Sourcemapping ES Result not in expected format")
 )
 

sourcemap/fleet_store.go

@@ -25,6 +25,7 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -38,11 +39,14 @@ import (
 
 const defaultFleetPort = 8220
 
+var errMsgFleetFailure = errMsgFailure + " fleet"
+
 type fleetStore struct {
 	apikey        string
 	c             *http.Client
 	sourceMapURLs map[key]string
 	fleetBaseURLs []string
+	rng           *rand.Rand
 }
 
 type key struct {
@@ -96,6 +100,7 @@ func newFleetStore(
 		fleetBaseURLs: fleetBaseURLs,
 		sourceMapURLs: sourceMapURLs,
 		c:             c,
+		rng:           rand.New(rand.NewSource(time.Now().UnixNano())),
 	}, nil
 }
 
@@ -108,12 +113,25 @@ func (f fleetStore) fetch(ctx context.Context, name, version, path string) (stri
 		)
 	}
 
-	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
-	idx := rng.Int() % len(f.fleetBaseURLs)
-	baseURL := f.fleetBaseURLs[idx]
+	idxs := f.rng.Perm(len(f.fleetBaseURLs))
+	for idx := range idxs {
+		baseURL := f.fleetBaseURLs[idx]
+		fleetURL := baseURL + sourceMapURL
+		resp, err := sendRequest(f, ctx, fleetURL)
+		if err != nil {
+			if strings.Contains(err.Error(), errMsgFailure) {
+				continue
+			} else {
+				return "", err
+			}
+		}
+		return resp, nil
+	}
 
-	fleetURL := baseURL + sourceMapURL
+	return "", fmt.Errorf(errMsgFleetFailure)
+}
 
+func sendRequest(f fleetStore, ctx context.Context, fleetURL string) (string, error) {
 	req, err := http.NewRequest(http.MethodGet, fleetURL, nil)
 	if err != nil {
 		return "", err
@@ -130,9 +148,9 @@ func (f fleetStore) fetch(ctx context.Context, name, version, path string) (stri
 	if resp.StatusCode != http.StatusOK {
 		body, err := ioutil.ReadAll(resp.Body)
 		if err != nil {
-			return "", fmt.Errorf("failure querying fleet: statuscode=%d response=(failed to read body)", resp.StatusCode)
+			return "", fmt.Errorf(errMsgFleetFailure, ": statuscode=%d response=(failed to read body)", resp.StatusCode)
 		}
-		return "", fmt.Errorf("failure querying fleet: statuscode=%d response=%s", resp.StatusCode, body)
+		return "", fmt.Errorf(errMsgFleetFailure, ": statuscode=%d response=%s", resp.StatusCode, body)
 	}
 
 	// Looking at the index in elasticsearch, currently

sourcemap/store.go

@@ -36,7 +36,8 @@ const (
 )
 
 var (
-	errInit = errors.New("Cache cannot be initialized. Expiration and CleanupInterval need to be >= 0")
+	errMsgFailure = "failure querying"
+	errInit       = errors.New("Cache cannot be initialized. Expiration and CleanupInterval need to be >= 0")
 )
 
 // Store holds information necessary to fetch a sourcemap, either from an
@@ -114,10 +115,10 @@ func (s *Store) Fetch(ctx context.Context, name, version, path string) (*sourcem
 		s.mu.Unlock()
 	}()
 
-	// fetch from Elasticsearch and ensure caching for all non-temporary results
+	// fetch from the store and ensure caching for all non-temporary results
 	sourcemapStr, err := s.backend.fetch(ctx, name, version, path)
 	if err != nil {
-		if !strings.Contains(err.Error(), "failure querying") {
+		if !strings.Contains(err.Error(), errMsgFailure) {
 			s.add(key, nil)
 		}
 		return nil, err