From a62e8144dc3b65466d47eec59c92523da32298c5 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Thu, 19 Oct 2023 14:47:34 +0200 Subject: [PATCH] fix: disable HTTP2 connections by default This change mitigates CVE-2023-44487 by disabling HTTP2 by default and forcing HTTP/1.1 until the Go standard library and golang.org/x/net are fully fixed. Right now, it is possible for authenticated and unauthenticated users to hold open HTTP2 connections and consume huge amounts of memory. It is possible to revert back the change by using the `--web.enable-http2` argument. Before this change: ``` curl -kv https://localhost:8443/metrics * Trying 127.0.0.1:8443... * Connected to localhost (127.0.0.1) port 8443 (#0) * ALPN: offers h2,http/1.1 [...] * ALPN: server accepted h2 [...] * using HTTP/2 * h2h3 [:method: GET] * h2h3 [:path: /metrics] * h2h3 [:scheme: https] * h2h3 [:authority: localhost:8443] * h2h3 [user-agent: curl/8.0.1] * h2h3 [accept: */*] * Using Stream ID: 1 (easy handle 0x5594d4614b10) [...] > GET /metrics HTTP/2 [...] ``` After this change: ``` curl -kv https://localhost:8443/metrics * Trying 127.0.0.1:8443... * Connected to localhost (127.0.0.1) port 8443 (#0) * ALPN: offers h2,http/1.1 [...] * ALPN: server accepted http/1.1 [...] * using HTTP/1.1 > GET /metrics HTTP/1.1 > Host: localhost:8443 > User-Agent: curl/8.0.1 > Accept: */* [...] < HTTP/1.1 200 OK [...] ``` See also: * https://github.com/kubernetes/kubernetes/pull/121120 * https://github.com/kubernetes/kubernetes/issues/121197 * https://github.com/golang/go/issues/63417#issuecomment-1758858612 Signed-off-by: Simon Pasquier --- Documentation/operator.md | 2 ++ cmd/admission-webhook/main.go | 37 ++++++++++++++++++++++++----------- cmd/operator/main.go | 15 +++++++++++++- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Documentation/operator.md b/Documentation/operator.md index dcef81a7003..ae7e92970cc 100644 --- a/Documentation/operator.md +++ b/Documentation/operator.md @@ -89,6 +89,8 @@ Usage of ./operator: Cert file to be used for operator web server endpoints. (default "/etc/tls/private/tls.crt") -web.client-ca-file string Client CA certificate file to be used for operator web server endpoints. (default "/etc/tls/private/tls-ca.crt") + -web.enable-http2 + Enable HTTP2 connections. -web.enable-tls Activate prometheus operator web server TLS. This is useful for example when using the rule validation webhook. -web.key-file string diff --git a/cmd/admission-webhook/main.go b/cmd/admission-webhook/main.go index 3a5ff2b7cc0..fbdacf1fc1a 100644 --- a/cmd/admission-webhook/main.go +++ b/cmd/admission-webhook/main.go @@ -53,12 +53,22 @@ var ( cfg = config{} flagset = flag.CommandLine + enableHTTP2 bool serverTLS bool rawTLSCipherSuites string ) func main() { flagset.StringVar(&cfg.ListenAddress, "web.listen-address", ":8443", "Address on which the admission webhook service listens") + // Mitigate CVE-2023-44487 by disabling HTTP2 by default until the Go + // standard library and golang.org/x/net are fully fixed. + // Right now, it is possible for authenticated and unauthenticated users to + // hold open HTTP2 connections and consume huge amounts of memory. + // See: + // * https://github.com/kubernetes/kubernetes/pull/121120 + // * https://github.com/kubernetes/kubernetes/issues/121197 + // * https://github.com/golang/go/issues/63417#issuecomment-1758858612 + flagset.BoolVar(&enableHTTP2, "web.enable-http2", false, "Enable HTTP2 connections.") flagset.BoolVar(&serverTLS, "web.enable-tls", true, "Enable TLS web server") flagset.StringVar(&cfg.ServerTLSConfig.CertFile, "web.cert-file", defaultCrtFile, "Certificate file to be used for the web server.") @@ -127,7 +137,7 @@ func (s *srv) run(listener net.Listener) error { log := log.With(s.logger, "address", listener.Addr().String()) if s.s.TLSConfig != nil { - level.Info(log).Log("msg", "Starting TLS enabled server") + level.Info(log).Log("msg", "Starting TLS enabled server", "http2", enableHTTP2) if err := s.s.ServeTLS(listener, "", ""); err != http.ErrServerClosed { return err } @@ -163,18 +173,23 @@ func newSrv(logger log.Logger, tlsConf *tls.Config) *srv { w.WriteHeader(http.StatusOK) }) + httpServer := http.Server{ + Handler: mux, + TLSConfig: tlsConf, + ReadHeaderTimeout: 30 * time.Second, + ReadTimeout: 30 * time.Second, + // use flags on standard logger to align with base logger and get consistent parsed fields form adapter: + // use shortfile flag to get proper 'caller' field (avoid being wrongly parsed/extracted from message) + // and no datetime related flag to keep 'ts' field from base logger (with controlled format) + ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile), + } + if !enableHTTP2 { + httpServer.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) + } + return &srv{ logger: logger, - s: &http.Server{ - Handler: mux, - TLSConfig: tlsConf, - ReadHeaderTimeout: 30 * time.Second, - ReadTimeout: 30 * time.Second, - // use flags on standard logger to align with base logger and get consistent parsed fields form adapter: - // use shortfile flag to get proper 'caller' field (avoid being wrongly parsed/extracted from message) - // and no datetime related flag to keep 'ts' field from base logger (with controlled format) - ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile), - }, + s: &httpServer, } } diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 6265e8de869..fd5046c1fee 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -144,7 +144,7 @@ func serve(srv *http.Server, listener net.Listener, logger log.Logger) func() er func serveTLS(srv *http.Server, listener net.Listener, logger log.Logger) func() error { return func() error { - level.Info(logger).Log("msg", "Starting secure server on "+listener.Addr().String()) + level.Info(logger).Log("msg", "Starting secure server on "+listener.Addr().String(), "http2", enableHTTP2) if err := srv.ServeTLS(listener, "", ""); err != http.ErrServerClosed { return err } @@ -161,6 +161,7 @@ var ( cfg = operator.DefaultConfig(defaultReloaderCPU, defaultReloaderMemory) rawTLSCipherSuites string + enableHTTP2 bool serverTLS bool flagset = flag.CommandLine @@ -168,6 +169,15 @@ var ( func init() { flagset.StringVar(&cfg.ListenAddress, "web.listen-address", ":8080", "Address on which to expose metrics and web interface.") + // Mitigate CVE-2023-44487 by disabling HTTP2 by default until the Go + // standard library and golang.org/x/net are fully fixed. + // Right now, it is possible for authenticated and unauthenticated users to + // hold open HTTP2 connections and consume huge amounts of memory. + // See: + // * https://github.com/kubernetes/kubernetes/pull/121120 + // * https://github.com/kubernetes/kubernetes/issues/121197 + // * https://github.com/golang/go/issues/63417#issuecomment-1758858612 + flagset.BoolVar(&enableHTTP2, "web.enable-http2", false, "Enable HTTP2 connections.") flagset.BoolVar(&serverTLS, "web.enable-tls", false, "Activate prometheus operator web server TLS. "+ " This is useful for example when using the rule validation webhook.") flagset.StringVar(&cfg.ServerTLSConfig.CertFile, "web.cert-file", defaultOperatorTLSDir+"/tls.crt", "Cert file to be used for operator web server endpoints.") @@ -525,6 +535,9 @@ func run() int { // and no datetime related flag to keep 'ts' field from base logger (with controlled format) ErrorLog: stdlog.New(log.NewStdlibAdapter(logger), "", stdlog.Lshortfile), } + if !enableHTTP2 { + srv.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler)) + } if srv.TLSConfig == nil { wg.Go(serve(srv, l, logger)) } else {