From 54266b219497706f6a08494a884b72f096547772 Mon Sep 17 00:00:00 2001 From: Alessandro Verzicco Date: Wed, 13 Oct 2021 16:17:23 +0200 Subject: [PATCH] Add support for grpc health check Signed-off-by: Alessandro Verzicco --- blackbox.yml | 10 +++ config/config.go | 23 +++++ go.mod | 1 + go.sum | 3 + main.go | 1 + prober/grpc.go | 201 ++++++++++++++++++++++++++++++++++++++++++++ prober/grpc_test.go | 160 +++++++++++++++++++++++++++++++++++ 7 files changed, 399 insertions(+) create mode 100644 prober/grpc.go create mode 100644 prober/grpc_test.go diff --git a/blackbox.yml b/blackbox.yml index 65c97cc0c..eb276f8c6 100644 --- a/blackbox.yml +++ b/blackbox.yml @@ -15,6 +15,16 @@ modules: tls: true tls_config: insecure_skip_verify: false + grpc: + prober: grpc + grpc: + tls: true + preferred_ip_protocol: "ip4" + grpc_plain: + prober: grpc + grpc: + tls: false + preferred_ip_protocol: "ip4" ssh_banner: prober: tcp tcp: diff --git a/config/config.go b/config/config.go index 07e84c05a..d2a93d1f2 100644 --- a/config/config.go +++ b/config/config.go @@ -62,6 +62,11 @@ var ( HTTPClientConfig: config.DefaultHTTPClientConfig, } + // DefaultGRPCProbe set default value for HTTPProbe + DefaultGRPCProbe = GRPCProbe{ + IPProtocolFallback: true, + } + // DefaultTCPProbe set default value for TCPProbe DefaultTCPProbe = TCPProbe{ IPProtocolFallback: true, @@ -187,6 +192,7 @@ type Module struct { TCP TCPProbe `yaml:"tcp,omitempty"` ICMP ICMPProbe `yaml:"icmp,omitempty"` DNS DNSProbe `yaml:"dns,omitempty"` + GRPC GRPCProbe `yaml:"grpc,omitempty"` } type HTTPProbe struct { @@ -209,6 +215,13 @@ type HTTPProbe struct { Compression string `yaml:"compression,omitempty"` } +type GRPCProbe struct { + TLS bool `yaml:"tls,omitempty"` + TLSConfig config.TLSConfig `yaml:"tls_config,omitempty"` + IPProtocolFallback bool `yaml:"ip_protocol_fallback,omitempty"` + PreferredIPProtocol string `yaml:"preferred_ip_protocol,omitempty"` +} + type HeaderMatch struct { Header string `yaml:"header,omitempty"` Regexp Regexp `yaml:"regexp,omitempty"` @@ -307,6 +320,16 @@ func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { return nil } +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (s *GRPCProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { + *s = DefaultGRPCProbe + type plain GRPCProbe + if err := unmarshal((*plain)(s)); err != nil { + return err + } + return nil +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. func (s *DNSProbe) UnmarshalYAML(unmarshal func(interface{}) error) error { *s = DefaultDNSProbe diff --git a/go.mod b/go.mod index ec2c3d9d4..31f1bafb8 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/prometheus/common v0.23.0 github.com/prometheus/exporter-toolkit v0.5.1 golang.org/x/net v0.0.0-20210505214959-0714010a04ed + google.golang.org/grpc v1.26.0 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/go.sum b/go.sum index e0116f897..154982002 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,7 @@ github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFG github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -400,6 +401,7 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -409,6 +411,7 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/main.go b/main.go index d0a746fad..7dc486708 100644 --- a/main.go +++ b/main.go @@ -67,6 +67,7 @@ var ( "tcp": prober.ProbeTCP, "icmp": prober.ProbeICMP, "dns": prober.ProbeDNS, + "grpc": prober.ProbeGRPC, } moduleUnknownCounter = prometheus.NewCounter(prometheus.CounterOpts{ diff --git a/prober/grpc.go b/prober/grpc.go new file mode 100644 index 000000000..079d0443c --- /dev/null +++ b/prober/grpc.go @@ -0,0 +1,201 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prober + +import ( + "context" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/blackbox_exporter/config" + "github.com/prometheus/client_golang/prometheus" + pconfig "github.com/prometheus/common/config" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + "net" + "net/url" + "strings" +) + +// Health defines a health-check connection. +type Health interface { + Check(c context.Context) (bool, codes.Code, *peer.Peer, error) +} + +type healthClient struct { + client grpc_health_v1.HealthClient + conn *grpc.ClientConn +} + +func NewGrpcHealthClient(conn *grpc.ClientConn) Health { + client := new(healthClient) + client.client = grpc_health_v1.NewHealthClient(conn) + client.conn = conn + return client +} + +func (c *healthClient) Close() error { + return c.conn.Close() +} + +func (c *healthClient) Check(ctx context.Context) (bool, codes.Code, *peer.Peer, error) { + var res *grpc_health_v1.HealthCheckResponse + var err error + req := new(grpc_health_v1.HealthCheckRequest) + + serverPeer := new(peer.Peer) + res, err = c.client.Check(ctx, req, grpc.Peer(serverPeer)) + if err == nil { + if res.GetStatus() == grpc_health_v1.HealthCheckResponse_SERVING { + return true, codes.OK, serverPeer, nil + } + return false, codes.OK, serverPeer, nil + } + + returnStatus, _ := status.FromError(err) + + return false, returnStatus.Code(), nil, err +} + +func ProbeGRPC(ctx context.Context, target string, module config.Module, registry *prometheus.Registry, logger log.Logger) (success bool) { + + var ( + durationGaugeVec = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "probe_grpc_duration_seconds", + Help: "Duration of gRPC request by phase", + }, []string{"phase"}) + + isSSLGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_grpc_ssl", + Help: "Indicates if SSL was used for the connection", + }) + + statusCodeGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_grpc_status_code", + Help: "Response gRPC status code", + }) + + probeSSLEarliestCertExpiryGauge = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_ssl_earliest_cert_expiry", + Help: "Returns earliest SSL cert expiry in unixtime", + }) + + probeTLSVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "probe_tls_version_info", + Help: "Contains the TLS version used", + }, + []string{"version"}, + ) + ) + + for _, lv := range []string{"resolve"} { + durationGaugeVec.WithLabelValues(lv) + } + + registry.MustRegister(durationGaugeVec) + registry.MustRegister(isSSLGauge) + registry.MustRegister(statusCodeGauge) + registry.MustRegister(probeSSLEarliestCertExpiryGauge) + registry.MustRegister(probeTLSVersion) + + if !strings.HasPrefix(target, "http://") && !strings.HasPrefix(target, "https://") { + target = "http://" + target + } + + targetURL, err := url.Parse(target) + if err != nil { + level.Error(logger).Log("msg", "Could not parse target URL", "err", err) + return false + } + + targetHost, targetPort, err := net.SplitHostPort(targetURL.Host) + // If split fails, assuming it's a hostname without port part. + if err != nil { + targetHost = targetURL.Host + } + + tlsConfig, err := pconfig.NewTLSConfig(&module.GRPC.TLSConfig) + if err != nil { + level.Error(logger).Log("msg", "Error creating TLS configuration", "err", err) + return false + } + + ip, lookupTime, err := chooseProtocol(ctx, module.GRPC.PreferredIPProtocol, module.GRPC.IPProtocolFallback, targetHost, registry, logger) + if err != nil { + level.Error(logger).Log("msg", "Error resolving address", "err", err) + return false + } + durationGaugeVec.WithLabelValues("resolve").Add(lookupTime) + + if len(tlsConfig.ServerName) == 0 { + // If there is no `server_name` in tls_config, use + // the hostname of the target. + tlsConfig.ServerName = targetHost + } + + if targetPort == "" { + targetURL.Host = "[" + ip.String() + "]" + } else { + targetURL.Host = net.JoinHostPort(ip.String(), targetPort) + } + + var opts []grpc.DialOption + target = targetHost + ":" + targetPort + if !module.GRPC.TLS { + level.Debug(logger).Log("msg", "Dialing GRPC without TLS") + opts = append(opts, grpc.WithInsecure()) + if len(targetPort) == 0 { + target = targetHost + ":80" + } + } else { + creds := credentials.NewTLS(tlsConfig) + opts = append(opts, grpc.WithTransportCredentials(creds)) + if len(targetPort) == 0 { + target = targetHost + ":443" + } + } + + conn, err := grpc.Dial(target, opts...) + + if err != nil { + level.Error(logger).Log("did not connect: %v", err) + } + + client := NewGrpcHealthClient(conn) + defer conn.Close() + ok, statusCode, serverPeer, err := client.Check(context.Background()) + + tlsInfo, tlsOk := serverPeer.AuthInfo.(credentials.TLSInfo) + if tlsOk { + isSSLGauge.Set(float64(1)) + probeSSLEarliestCertExpiryGauge.Set(float64(getEarliestCertExpiry(&tlsInfo.State).Unix())) + probeTLSVersion.WithLabelValues(getTLSVersion(&tlsInfo.State)).Set(1) + } else { + isSSLGauge.Set(float64(0)) + } + statusCodeGauge.Set(float64(statusCode)) + + if !ok || err != nil { + level.Error(logger).Log("msg", "can't connect grpc server:", "err", err) + success = false + } else { + level.Debug(logger).Log("connect the grpc server successfully") + success = true + } + + return +} diff --git a/prober/grpc_test.go b/prober/grpc_test.go new file mode 100644 index 000000000..8ebf91993 --- /dev/null +++ b/prober/grpc_test.go @@ -0,0 +1,160 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package prober + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "github.com/go-kit/kit/log" + "github.com/prometheus/blackbox_exporter/config" + "github.com/prometheus/client_golang/prometheus" + pconfig "github.com/prometheus/common/config" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health" + "google.golang.org/grpc/health/grpc_health_v1" + "io/ioutil" + "net" + "os" + "testing" + "time" +) + +func TestGRPCConnection(t *testing.T) { + + ln, err := net.Listen("tcp", "localhost:8080") + if err != nil { + t.Fatalf("Error listening on socket: %s", err) + } + defer ln.Close() + s := grpc.NewServer() + healthServer := health.NewServer() + healthServer.SetServingStatus("SERVING", grpc_health_v1.HealthCheckResponse_SERVING) + grpc_health_v1.RegisterHealthServer(s, healthServer) + + go func() { + if err := s.Serve(ln); err != nil { + t.Errorf("failed to serve: %v", err) + return + } + }() + + testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + registry := prometheus.NewRegistry() + + result := ProbeGRPC(testCTX, "localhost:8080", + config.Module{Timeout: time.Second, GRPC: config.GRPCProbe{ + IPProtocolFallback: false, + }, + }, registry, log.NewNopLogger()) + + if !result { + t.Fatalf("GRPC probe failed") + } + + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + + expectedResults := map[string]float64{ + "probe_grpc_ssl": 0, + "probe_grpc_status_code": 0, + } + + checkRegistryResults(expectedResults, mfs, t) + s.GracefulStop() +} + +func TestGRPCTLSConnection(t *testing.T) { + + certExpiry := time.Now().AddDate(0, 0, 1) + testCertTmpl := generateCertificateTemplate(certExpiry, false) + testCertTmpl.IsCA = true + _, testcertPem, testKey := generateSelfSignedCertificate(testCertTmpl) + + // CAFile must be passed via filesystem, use a tempfile. + tmpCaFile, err := ioutil.TempFile("", "cafile.pem") + if err != nil { + t.Fatalf("Error creating CA tempfile: %s", err) + } + if _, err = tmpCaFile.Write(testcertPem); err != nil { + t.Fatalf("Error writing CA tempfile: %s", err) + } + if err = tmpCaFile.Close(); err != nil { + t.Fatalf("Error closing CA tempfile: %s", err) + } + defer os.Remove(tmpCaFile.Name()) + + testKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(testKey)}) + testcert, err := tls.X509KeyPair(testcertPem, testKeyPem) + if err != nil { + panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err)) + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{testcert}, + } + + ln, err := net.Listen("tcp", "localhost:8080") + if err != nil { + t.Fatalf("Error listening on socket: %s", err) + } + defer ln.Close() + + s := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig))) + healthServer := health.NewServer() + healthServer.SetServingStatus("SERVING", grpc_health_v1.HealthCheckResponse_SERVING) + grpc_health_v1.RegisterHealthServer(s, healthServer) + + go func() { + if err := s.Serve(ln); err != nil { + t.Errorf("failed to serve: %v", err) + return + } + }() + + testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + registry := prometheus.NewRegistry() + + result := ProbeGRPC(testCTX, "localhost:8080", + config.Module{Timeout: time.Second, GRPC: config.GRPCProbe{ + TLS: true, + TLSConfig: pconfig.TLSConfig{InsecureSkipVerify: true}, + IPProtocolFallback: false, + }, + }, registry, log.NewNopLogger()) + + if !result { + t.Fatalf("GRPC probe failed") + } + + mfs, err := registry.Gather() + if err != nil { + t.Fatal(err) + } + + expectedResults := map[string]float64{ + "probe_grpc_ssl": 1, + "probe_grpc_status_code": 0, + } + + checkRegistryResults(expectedResults, mfs, t) + s.GracefulStop() +}