Add new probe_ssl_latest_verified_chain_expiry metric

prober/http.go

@@ -261,6 +261,11 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 			Help: "Returns earliest SSL cert expiry in unixtime",
 		})
 
+		probeSSLLastChainExpiryTimestampSeconds = prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "probe_ssl_last_chain_expiry_timestamp_seconds",
+			Help: "Returns last SSL chain expiry in timestamp seconds",
+		})
+
 		probeTLSVersion = prometheus.NewGaugeVec(
 			prometheus.GaugeOpts{
 				Name: "probe_tls_version_info",
@@ -546,9 +551,10 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 
 	if resp.TLS != nil {
 		isSSLGauge.Set(float64(1))
-		registry.MustRegister(probeSSLEarliestCertExpiryGauge, probeTLSVersion)
+		registry.MustRegister(probeSSLEarliestCertExpiryGauge, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds)
 		probeSSLEarliestCertExpiryGauge.Set(float64(getEarliestCertExpiry(resp.TLS).Unix()))
 		probeTLSVersion.WithLabelValues(getTLSVersion(resp.TLS)).Set(1)
+		probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(resp.TLS).Unix()))
 		if httpConfig.FailIfSSL {
 			level.Error(logger).Log("msg", "Final request was over SSL")
 			success = false

prober/http_test.go

@@ -16,6 +16,8 @@ package prober
 import (
 	"context"
 	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -597,7 +599,9 @@ func TestTLSConfigIsIgnoredForPlainHTTP(t *testing.T) {
 func TestHTTPUsesTargetAsTLSServerName(t *testing.T) {
 	// Create test certificates valid for 1 day.
 	certExpiry := time.Now().AddDate(0, 0, 1)
-	testcertPem, testKeyPem := generateTestCertificate(certExpiry, false)
+	testCertTmpl := generateCertificateTemplate(certExpiry, false)
+	testCertTmpl.IsCA = true
+	_, testcertPem, testKey := generateSelfSignedCertificate(testCertTmpl)
 
 	// CAFile must be passed via filesystem, use a tempfile.
 	tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
@@ -612,6 +616,7 @@ func TestHTTPUsesTargetAsTLSServerName(t *testing.T) {
 	}
 	defer os.Remove(tmpCaFile.Name())
 
+	testKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(testKey)})
 	testcert, err := tls.X509KeyPair(testcertPem, testKeyPem)
 	if err != nil {
 		panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))

prober/tcp.go

@@ -94,6 +94,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
 		Name: "probe_ssl_earliest_cert_expiry",
 		Help: "Returns earliest SSL cert expiry date",
 	})
+	probeSSLLastChainExpiryTimestampSeconds := prometheus.NewGauge(prometheus.GaugeOpts{
+		Name: "probe_ssl_last_chain_expiry_timestamp_seconds",
+		Help: "Returns last SSL chain expiry in unixtime",
+	})
 	probeTLSVersion := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "probe_tls_version_info",
@@ -125,9 +129,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
 	}
 	if module.TCP.TLS {
 		state := conn.(*tls.Conn).ConnectionState()
-		registry.MustRegister(probeSSLEarliestCertExpiry, probeTLSVersion)
+		registry.MustRegister(probeSSLEarliestCertExpiry, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds)
 		probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).Unix()))
 		probeTLSVersion.WithLabelValues(getTLSVersion(&state)).Set(1)
+		probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(&state).Unix()))
 	}
 	scanner := bufio.NewScanner(conn)
 	for i, qr := range module.TCP.QueryResponse {
@@ -194,9 +199,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
 
 			// Get certificate expiry.
 			state := tlsConn.ConnectionState()
-			registry.MustRegister(probeSSLEarliestCertExpiry)
+			registry.MustRegister(probeSSLEarliestCertExpiry, probeSSLLastChainExpiryTimestampSeconds)
 			probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).Unix()))
 			probeTLSVersion.WithLabelValues(getTLSVersion(&state)).Set(1)
+			probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(&state).Unix()))
 		}
 	}
 	return true

prober/tcp_test.go

@@ -14,8 +14,13 @@
 package prober
 
 import (
+	"bytes"
 	"context"
+	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
+	"crypto/x509"
+	"encoding/pem"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -84,14 +89,16 @@ func TestTCPConnectionWithTLS(t *testing.T) {
 
 	// Create test certificates valid for 1 day.
 	certExpiry := time.Now().AddDate(0, 0, 1)
-	testcert_pem, testkey_pem := generateTestCertificate(certExpiry, false)
+	rootCertTmpl := generateCertificateTemplate(certExpiry, false)
+	rootCertTmpl.IsCA = true
+	_, rootCertPem, rootKey := generateSelfSignedCertificate(rootCertTmpl)
 
 	// CAFile must be passed via filesystem, use a tempfile.
 	tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
 	if err != nil {
 		t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err))
 	}
-	if _, err := tmpCaFile.Write(testcert_pem); err != nil {
+	if _, err := tmpCaFile.Write(rootCertPem); err != nil {
 		t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err))
 	}
 	if err := tmpCaFile.Close(); err != nil {
@@ -109,7 +116,8 @@ func TestTCPConnectionWithTLS(t *testing.T) {
 		}
 		defer conn.Close()
 
-		testcert, err := tls.X509KeyPair(testcert_pem, testkey_pem)
+		rootKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey)})
+		testcert, err := tls.X509KeyPair(rootCertPem, rootKeyPem)
 		if err != nil {
 			panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
 		}
@@ -193,6 +201,127 @@ func TestTCPConnectionWithTLS(t *testing.T) {
 	checkRegistryResults(expectedResults, mfs, t)
 }
 
+func TestTCPConnectionWithTLSAndVerifiedCertificateChain(t *testing.T) {
+	if os.Getenv("TRAVIS") == "true" {
+		t.Skip("skipping; travisci is failing on ipv6 dns requests")
+	}
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("Error listening on socket: %s", err)
+	}
+	defer ln.Close()
+	_, listenPort, _ := net.SplitHostPort(ln.Addr().String())
+
+	testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	// From here prepare two certificate chains where one is expired
+
+	rootPrivatekey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		panic(fmt.Sprintf("Error creating rsa key: %s", err))
+	}
+
+	rootCertExpiry := time.Now().AddDate(0, 0, 2)
+	rootCertTmpl := generateCertificateTemplate(rootCertExpiry, false)
+	rootCertTmpl.IsCA = true
+	_, rootCertPem := generateSelfSignedCertificateWithPrivateKey(rootCertTmpl, rootPrivatekey)
+
+	oldRootCertExpiry := time.Now().AddDate(0, 0, -1)
+	expiredRootCertTmpl := generateCertificateTemplate(oldRootCertExpiry, false)
+	expiredRootCertTmpl.IsCA = true
+	expiredRootCert, expiredRootCertPem := generateSelfSignedCertificateWithPrivateKey(expiredRootCertTmpl, rootPrivatekey)
+
+	serverCertExpiry := time.Now().AddDate(0, 0, 1)
+	serverCertTmpl := generateCertificateTemplate(serverCertExpiry, false)
+	_, serverCertPem, serverKey := generateSignedCertificate(serverCertTmpl, expiredRootCert, rootPrivatekey)
+
+	// CAFile must be passed via filesystem, use a tempfile.
+	tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
+	if err != nil {
+		t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err))
+	}
+	if _, err := tmpCaFile.Write(bytes.Join([][]byte{rootCertPem, expiredRootCertPem}, []byte("\n"))); err != nil {
+		t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err))
+	}
+	if err := tmpCaFile.Close(); err != nil {
+		t.Fatalf(fmt.Sprintf("Error closing CA tempfile: %s", err))
+	}
+	defer os.Remove(tmpCaFile.Name())
+
+	ch := make(chan (struct{}))
+	logger := log.NewNopLogger()
+	// Handle server side of this test.
+	serverFunc := func() {
+		conn, err := ln.Accept()
+		if err != nil {
+			panic(fmt.Sprintf("Error accepting on socket: %s", err))
+		}
+		defer conn.Close()
+
+		serverKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)})
+
+		keypair, err := tls.X509KeyPair(serverCertPem, serverKeyPem)
+		if err != nil {
+			panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
+		}
+
+		// Immediately upgrade to TLS.
+		tlsConfig := &tls.Config{
+			ServerName:   "localhost",
+			Certificates: []tls.Certificate{keypair},
+			MinVersion:   tls.VersionTLS12,
+			MaxVersion:   tls.VersionTLS12,
+		}
+		tlsConn := tls.Server(conn, tlsConfig)
+		defer tlsConn.Close()
+		if err := tlsConn.Handshake(); err != nil {
+			level.Error(logger).Log("msg", "Error TLS Handshake (server) failed", "err", err)
+		} else {
+			// Send some bytes before terminating the connection.
+			fmt.Fprintf(tlsConn, "Hello World!\n")
+		}
+		ch <- struct{}{}
+	}
+
+	// Expect name-verified TLS connection.
+	module := config.Module{
+		TCP: config.TCPProbe{
+			IPProtocol:         "ip4",
+			IPProtocolFallback: true,
+			TLS:                true,
+			TLSConfig: pconfig.TLSConfig{
+				CAFile:             tmpCaFile.Name(),
+				InsecureSkipVerify: false,
+			},
+		},
+	}
+
+	registry := prometheus.NewRegistry()
+	go serverFunc()
+	// Test name-verification with name from target.
+	target := net.JoinHostPort("localhost", listenPort)
+	if !ProbeTCP(testCTX, target, module, registry, log.NewNopLogger()) {
+		t.Fatalf("TCP module failed, expected success.")
+	}
+	<-ch
+
+	// Check the resulting metrics.
+	mfs, err := registry.Gather()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Check values
+	expectedResults := map[string]float64{
+		"probe_ssl_earliest_cert_expiry":                float64(serverCertExpiry.Unix()),
+		"probe_ssl_last_chain_expiry_timestamp_seconds": float64(serverCertExpiry.Unix()),
+		"probe_tls_version_info":                        1,
+	}
+	checkRegistryResults(expectedResults, mfs, t)
+}
+
 func TestTCPConnectionQueryResponseStartTLS(t *testing.T) {
 	ln, err := net.Listen("tcp", "localhost:0")
 	if err != nil {
@@ -205,14 +334,16 @@ func TestTCPConnectionQueryResponseStartTLS(t *testing.T) {
 
 	// Create test certificates valid for 1 day.
 	certExpiry := time.Now().AddDate(0, 0, 1)
-	testcert_pem, testkey_pem := generateTestCertificate(certExpiry, true)
+	testCertTmpl := generateCertificateTemplate(certExpiry, true)
+	testCertTmpl.IsCA = true
+	_, testCertPem, testKey := generateSelfSignedCertificate(testCertTmpl)
 
 	// CAFile must be passed via filesystem, use a tempfile.
 	tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
 	if err != nil {
 		t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err))
 	}
-	if _, err := tmpCaFile.Write(testcert_pem); err != nil {
+	if _, err := tmpCaFile.Write(testCertPem); err != nil {
 		t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err))
 	}
 	if err := tmpCaFile.Close(); err != nil {
@@ -263,7 +394,8 @@ func TestTCPConnectionQueryResponseStartTLS(t *testing.T) {
 		}
 		fmt.Fprintf(conn, "220 2.0.0 Ready to start TLS\n")
 
-		testcert, err := tls.X509KeyPair(testcert_pem, testkey_pem)
+		testKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(testKey)})
+		testcert, err := tls.X509KeyPair(testCertPem, testKeyPem)
 		if err != nil {
 			panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
 		}

prober/tls.go

@@ -28,6 +28,23 @@ func getEarliestCertExpiry(state *tls.ConnectionState) time.Time {
 	return earliest
 }
 
+func getLastChainExpiry(state *tls.ConnectionState) time.Time {
+	lastChainExpiry := time.Time{}
+	for _, chain := range state.VerifiedChains {
+		earliestCertExpiry := time.Time{}
+		for _, cert := range chain {
+			if (earliestCertExpiry.IsZero() || cert.NotAfter.Before(earliestCertExpiry)) && !cert.NotAfter.IsZero() {
+				earliestCertExpiry = cert.NotAfter
+			}
+		}
+		if lastChainExpiry.IsZero() || lastChainExpiry.After(earliestCertExpiry) {
+			lastChainExpiry = earliestCertExpiry
+		}
+
+	}
+	return lastChainExpiry
+}
+
 func getTLSVersion(state *tls.ConnectionState) string {
 	switch state.Version {
 	case tls.VersionTLS10:

prober/utils_test.go

@@ -79,18 +79,8 @@ func checkRegistryLabels(expRes map[string]map[string]string, mfs []*dto.MetricF
 	}
 }
 
-// Create test certificate with specified expiry date
-// Certificate will be self-signed and use localhost/127.0.0.1
-// Generated certificate and key are returned in PEM encoding
-func generateTestCertificate(expiry time.Time, IPAddressSAN bool) ([]byte, []byte) {
-	privatekey, err := rsa.GenerateKey(rand.Reader, 2048)
-	if err != nil {
-		panic(fmt.Sprintf("Error creating rsa key: %s", err))
-	}
-	publickey := &privatekey.PublicKey
-
-	cert := x509.Certificate{
-		IsCA:                  true,
+func generateCertificateTemplate(expiry time.Time, IPAddressSAN bool) *x509.Certificate {
+	template := &x509.Certificate{
 		BasicConstraintsValid: true,
 		SubjectKeyId:          []byte{1},
 		SerialNumber:          big.NewInt(1),
@@ -102,18 +92,54 @@ func generateTestCertificate(expiry time.Time, IPAddressSAN bool) ([]byte, []byt
 		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
 		KeyUsage:    x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
 	}
-	cert.DNSNames = append(cert.DNSNames, "localhost")
+
+	template.DNSNames = append(template.DNSNames, "localhost")
 	if IPAddressSAN {
-		cert.IPAddresses = append(cert.IPAddresses, net.ParseIP("127.0.0.1"))
-		cert.IPAddresses = append(cert.IPAddresses, net.ParseIP("::1"))
+		template.IPAddresses = append(template.IPAddresses, net.ParseIP("127.0.0.1"))
+		template.IPAddresses = append(template.IPAddresses, net.ParseIP("::1"))
 	}
-	derCert, err := x509.CreateCertificate(rand.Reader, &cert, &cert, publickey, privatekey)
+
+	return template
+}
+
+func generateCertificate(template, parent *x509.Certificate, publickey *rsa.PublicKey, privatekey *rsa.PrivateKey) (*x509.Certificate, []byte) {
+	derCert, err := x509.CreateCertificate(rand.Reader, template, template, publickey, privatekey)
 	if err != nil {
 		panic(fmt.Sprintf("Error signing test-certificate: %s", err))
 	}
+	cert, err := x509.ParseCertificate(derCert)
+	if err != nil {
+		panic(fmt.Sprintf("Error parsing test-certificate: %s", err))
+	}
 	pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derCert})
-	pemKey := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privatekey)})
-	return pemCert, pemKey
+	return cert, pemCert
+
+}
+
+func generateSignedCertificate(template, parentCert *x509.Certificate, parentKey *rsa.PrivateKey) (*x509.Certificate, []byte, *rsa.PrivateKey) {
+	privatekey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		panic(fmt.Sprintf("Error creating rsa key: %s", err))
+	}
+	cert, pemCert := generateCertificate(template, parentCert, &privatekey.PublicKey, parentKey)
+	return cert, pemCert, privatekey
+}
+
+func generateSelfSignedCertificate(template *x509.Certificate) (*x509.Certificate, []byte, *rsa.PrivateKey) {
+	privatekey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		panic(fmt.Sprintf("Error creating rsa key: %s", err))
+	}
+	publickey := &privatekey.PublicKey
+
+	cert, pemCert := generateCertificate(template, template, publickey, privatekey)
+	return cert, pemCert, privatekey
+}
+
+func generateSelfSignedCertificateWithPrivateKey(template *x509.Certificate, privatekey *rsa.PrivateKey) (*x509.Certificate, []byte) {
+	publickey := &privatekey.PublicKey
+	cert, pemCert := generateCertificate(template, template, publickey, privatekey)
+	return cert, pemCert
 }
 
 func TestChooseProtocol(t *testing.T) {