Abort request on missing Graylog server version (#482)

* Abort request on missing Graylog server version

If we fail to read the server version from Graylog,
stop using a fallback to 4.0.0 and retry the request
instead.

In scenarios with multiple Graylog nodes behind a load balancer,
this can lead to the sidecar temporarly stopping all collectors.

Fixes #481

* dont try to json parse arbitrary http error responses

api/graylog.go

@@ -42,22 +42,20 @@ var (
 )
 
 func GetServerVersion(httpClient *http.Client, ctx *context.Ctx) (*GraylogVersion, error) {
-	// In case of an error just assume 4.0.0
-	fallbackVersion, _ := NewGraylogVersion("4.0.0")
-
 	c := rest.NewClient(httpClient, ctx)
 	c.BaseURL = ctx.ServerUrl
 	r, err := c.NewRequest("GET", "/", nil, nil)
 	if err != nil {
 		log.Errorf("Cannot retrieve server version %v", err)
-		return fallbackVersion, err
+		return nil, err
 	}
 	versionResponse := graylog.ServerVersionResponse{}
 	resp, err := c.Do(r, &versionResponse)
 	if err != nil || resp == nil {
 		log.Errorf("Error fetching server version %v", err)
-		return fallbackVersion, err
+		return nil, err
 	}
+	log.Debugf("Graylog server version %v", versionResponse.Version)
 	return NewGraylogVersion(versionResponse.Version)
 }
 

api/rest/rest.go

@@ -149,12 +149,9 @@ func CheckResponse(r *http.Response) error {
 	}
 
 	errorResponse := &ErrorResponse{Response: r}
-	data, err := ioutil.ReadAll(r.Body)
+	data, err := io.ReadAll(io.LimitReader(r.Body, 2048))
 	if err == nil && len(data) > 0 {
-		err := json.Unmarshal(data, errorResponse)
-		if err != nil {
-			return err
-		}
+		errorResponse.Message = string(data)
 	}
 
 	return errorResponse

changelog/unreleased/issue-481.toml

@@ -0,0 +1,11 @@
+# PLEASE REMOVE COMMENTS AND OPTIONAL FIELDS! THANKS!
+
+# Entry type according to https://keepachangelog.com/en/1.0.0/
+# One of: a(dded), c(hanged), d(eprecated), r(emoved), f(ixed), s(ecurity)
+type = "fixed"
+message = "Avoid stopping collectors if a Graylog cluster behind load balancers is partly shut down."
+
+issues = ["481"]
+pulls = ["482"]
+
+contributors = [""]

services/periodicals.go

@@ -49,7 +49,10 @@ func StartPeriodicals(context *context.Ctx) {
 			}
 			firstRun = false
 
-			serverVersion, _ := api.GetServerVersion(httpClient, context)
+			serverVersion, err := api.GetServerVersion(httpClient, context)
+			if err != nil {
+				continue
+			}
 
 			// registration regResponse contains configuration assignments
 			regResponse, err := updateCollectorRegistration(httpClient, lastRegResponse.Checksum, context, serverVersion)