From 41bec868a8030884dfdb42ead6a2d13b9991038c Mon Sep 17 00:00:00 2001
From: Mahmood Ali <mahmood@hashicorp.com>
Date: Wed, 22 Apr 2020 16:19:59 -0400
Subject: [PATCH] http: adjust log level for request failure

Failed requests due to API client errors are to be marked as DEBUG.

The Error log level should be reserved to signal problems with the
cluster and are actionable for nomad system operators.  Logs due to
misbehaving API clients don't represent a system level problem and seem
spurius to nomad maintainers at best.  These log messages can also be
attack vectors for deniel of service attacks by filling servers disk
space with spurious log messages.
---
 command/agent/http.go      | 17 +++++++++++++++--
 command/agent/http_test.go | 12 ++++++++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/command/agent/http.go b/command/agent/http.go
index 0fffaee8bb1..3993bdc6307 100644
--- a/command/agent/http.go
+++ b/command/agent/http.go
@@ -467,7 +467,11 @@ func (s *HTTPServer) wrap(handler func(resp http.ResponseWriter, req *http.Reque
 
 			resp.WriteHeader(code)
 			resp.Write([]byte(errMsg))
-			s.logger.Error("request failed", "method", req.Method, "path", reqURL, "error", err, "code", code)
+			if isAPIClientError(code) {
+				s.logger.Debug("request failed", "method", req.Method, "path", reqURL, "error", err, "code", code)
+			} else {
+				s.logger.Error("request failed", "method", req.Method, "path", reqURL, "error", err, "code", code)
+			}
 			return
 		}
 
@@ -521,7 +525,11 @@ func (s *HTTPServer) wrapNonJSON(handler func(resp http.ResponseWriter, req *htt
 			code, errMsg := errCodeFromHandler(err)
 			resp.WriteHeader(code)
 			resp.Write([]byte(errMsg))
-			s.logger.Error("request failed", "method", req.Method, "path", reqURL, "error", err, "code", code)
+			if isAPIClientError(code) {
+				s.logger.Debug("request failed", "method", req.Method, "path", reqURL, "error", err, "code", code)
+			} else {
+				s.logger.Error("request failed", "method", req.Method, "path", reqURL, "error", err, "code", code)
+			}
 			return
 		}
 
@@ -533,6 +541,11 @@ func (s *HTTPServer) wrapNonJSON(handler func(resp http.ResponseWriter, req *htt
 	return f
 }
 
+// isAPIClientError returns true if the passed http code represents a client error
+func isAPIClientError(code int) bool {
+	return 400 <= code && code <= 499
+}
+
 // decodeBody is used to decode a JSON request body
 func decodeBody(req *http.Request, out interface{}) error {
 	dec := json.NewDecoder(req.Body)
diff --git a/command/agent/http_test.go b/command/agent/http_test.go
index 5659db111bd..1b05706599b 100644
--- a/command/agent/http_test.go
+++ b/command/agent/http_test.go
@@ -1082,6 +1082,18 @@ func TestHTTPServer_Limits_OK(t *testing.T) {
 	}
 }
 
+func Test_IsAPIClientError(t *testing.T) {
+	trueCases := []int{400, 403, 404, 499}
+	for _, c := range trueCases {
+		require.Truef(t, isAPIClientError(c), "code: %v", c)
+	}
+
+	falseCases := []int{100, 300, 500, 501, 505}
+	for _, c := range falseCases {
+		require.Falsef(t, isAPIClientError(c), "code: %v", c)
+	}
+}
+
 func httpTest(t testing.TB, cb func(c *Config), f func(srv *TestAgent)) {
 	s := makeHTTPServer(t, cb)
 	defer s.Shutdown()