Skip to content

Commit

Permalink
Add new probe_ssl_last_chain_expiry_timestamp_seconds metric (#636)
Browse files Browse the repository at this point in the history 
* Add new probe_ssl_latest_verified_chain_expiry metric

Resolves #340

Based on the discussion in the issue above, this metric will help determine
when the SSL/TLS certificate expiration error actually happens on clients
like a browser that attempts to verify certificates by building one or
more chains from peer certificates.

Signed-off-by: Takuya Kosugiyama <[email protected]>
  • Loading branch information
@itkq
itkq authored Jun 10, 2020
1 parent 261b0eb commit 725a683
Show file tree
Hide file tree
Showing 6 changed files with 220 additions and 28 deletions.
8 changes: 7 additions & 1 deletion prober/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,11 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
Help: "Returns earliest SSL cert expiry in unixtime",
})

probeSSLLastChainExpiryTimestampSeconds = prometheus.NewGauge(prometheus.GaugeOpts{
Name: "probe_ssl_last_chain_expiry_timestamp_seconds",
Help: "Returns last SSL chain expiry in timestamp seconds",
})

probeTLSVersion = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "probe_tls_version_info",
Expand Down Expand Up @@ -546,9 +551,10 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr

if resp.TLS != nil {
isSSLGauge.Set(float64(1))
registry.MustRegister(probeSSLEarliestCertExpiryGauge, probeTLSVersion)
registry.MustRegister(probeSSLEarliestCertExpiryGauge, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds)
probeSSLEarliestCertExpiryGauge.Set(float64(getEarliestCertExpiry(resp.TLS).Unix()))
probeTLSVersion.WithLabelValues(getTLSVersion(resp.TLS)).Set(1)
probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(resp.TLS).Unix()))
if httpConfig.FailIfSSL {
level.Error(logger).Log("msg", "Final request was over SSL")
success = false
Expand Down
7 changes: 6 additions & 1 deletion prober/http_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ package prober
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"fmt"
"io/ioutil"
"net/http"
Expand Down Expand Up @@ -597,7 +599,9 @@ func TestTLSConfigIsIgnoredForPlainHTTP(t *testing.T) {
func TestHTTPUsesTargetAsTLSServerName(t *testing.T) {
// Create test certificates valid for 1 day.
certExpiry := time.Now().AddDate(0, 0, 1)
testcertPem, testKeyPem := generateTestCertificate(certExpiry, false)
testCertTmpl := generateCertificateTemplate(certExpiry, false)
testCertTmpl.IsCA = true
_, testcertPem, testKey := generateSelfSignedCertificate(testCertTmpl)

// CAFile must be passed via filesystem, use a tempfile.
tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
Expand All @@ -612,6 +616,7 @@ func TestHTTPUsesTargetAsTLSServerName(t *testing.T) {
}
defer os.Remove(tmpCaFile.Name())

testKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(testKey)})
testcert, err := tls.X509KeyPair(testcertPem, testKeyPem)
if err != nil {
panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
Expand Down
10 changes: 8 additions & 2 deletions prober/tcp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
Name: "probe_ssl_earliest_cert_expiry",
Help: "Returns earliest SSL cert expiry date",
})
probeSSLLastChainExpiryTimestampSeconds := prometheus.NewGauge(prometheus.GaugeOpts{
Name: "probe_ssl_last_chain_expiry_timestamp_seconds",
Help: "Returns last SSL chain expiry in unixtime",
})
probeTLSVersion := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "probe_tls_version_info",
Expand Down Expand Up @@ -125,9 +129,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry
}
if module.TCP.TLS {
state := conn.(*tls.Conn).ConnectionState()
registry.MustRegister(probeSSLEarliestCertExpiry, probeTLSVersion)
registry.MustRegister(probeSSLEarliestCertExpiry, probeTLSVersion, probeSSLLastChainExpiryTimestampSeconds)
probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).Unix()))
probeTLSVersion.WithLabelValues(getTLSVersion(&state)).Set(1)
probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(&state).Unix()))
}
scanner := bufio.NewScanner(conn)
for i, qr := range module.TCP.QueryResponse {
Expand Down Expand Up @@ -194,9 +199,10 @@ func ProbeTCP(ctx context.Context, target string, module config.Module, registry

// Get certificate expiry.
state := tlsConn.ConnectionState()
registry.MustRegister(probeSSLEarliestCertExpiry)
registry.MustRegister(probeSSLEarliestCertExpiry, probeSSLLastChainExpiryTimestampSeconds)
probeSSLEarliestCertExpiry.Set(float64(getEarliestCertExpiry(&state).Unix()))
probeTLSVersion.WithLabelValues(getTLSVersion(&state)).Set(1)
probeSSLLastChainExpiryTimestampSeconds.Set(float64(getLastChainExpiry(&state).Unix()))
}
}
return true
Expand Down
144 changes: 138 additions & 6 deletions prober/tcp_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,13 @@
package prober

import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"fmt"
"io/ioutil"
"net"
Expand Down Expand Up @@ -84,14 +89,16 @@ func TestTCPConnectionWithTLS(t *testing.T) {

// Create test certificates valid for 1 day.
certExpiry := time.Now().AddDate(0, 0, 1)
testcert_pem, testkey_pem := generateTestCertificate(certExpiry, false)
rootCertTmpl := generateCertificateTemplate(certExpiry, false)
rootCertTmpl.IsCA = true
_, rootCertPem, rootKey := generateSelfSignedCertificate(rootCertTmpl)

// CAFile must be passed via filesystem, use a tempfile.
tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
if err != nil {
t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err))
}
if _, err := tmpCaFile.Write(testcert_pem); err != nil {
if _, err := tmpCaFile.Write(rootCertPem); err != nil {
t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err))
}
if err := tmpCaFile.Close(); err != nil {
Expand All @@ -109,7 +116,8 @@ func TestTCPConnectionWithTLS(t *testing.T) {
}
defer conn.Close()

testcert, err := tls.X509KeyPair(testcert_pem, testkey_pem)
rootKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(rootKey)})
testcert, err := tls.X509KeyPair(rootCertPem, rootKeyPem)
if err != nil {
panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
}
Expand Down Expand Up @@ -193,6 +201,127 @@ func TestTCPConnectionWithTLS(t *testing.T) {
checkRegistryResults(expectedResults, mfs, t)
}

func TestTCPConnectionWithTLSAndVerifiedCertificateChain(t *testing.T) {
if os.Getenv("TRAVIS") == "true" {
t.Skip("skipping; travisci is failing on ipv6 dns requests")
}

ln, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("Error listening on socket: %s", err)
}
defer ln.Close()
_, listenPort, _ := net.SplitHostPort(ln.Addr().String())

testCTX, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()

// From here prepare two certificate chains where one is expired

rootPrivatekey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(fmt.Sprintf("Error creating rsa key: %s", err))
}

rootCertExpiry := time.Now().AddDate(0, 0, 2)
rootCertTmpl := generateCertificateTemplate(rootCertExpiry, false)
rootCertTmpl.IsCA = true
_, rootCertPem := generateSelfSignedCertificateWithPrivateKey(rootCertTmpl, rootPrivatekey)

oldRootCertExpiry := time.Now().AddDate(0, 0, -1)
expiredRootCertTmpl := generateCertificateTemplate(oldRootCertExpiry, false)
expiredRootCertTmpl.IsCA = true
expiredRootCert, expiredRootCertPem := generateSelfSignedCertificateWithPrivateKey(expiredRootCertTmpl, rootPrivatekey)

serverCertExpiry := time.Now().AddDate(0, 0, 1)
serverCertTmpl := generateCertificateTemplate(serverCertExpiry, false)
_, serverCertPem, serverKey := generateSignedCertificate(serverCertTmpl, expiredRootCert, rootPrivatekey)

// CAFile must be passed via filesystem, use a tempfile.
tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
if err != nil {
t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err))
}
if _, err := tmpCaFile.Write(bytes.Join([][]byte{rootCertPem, expiredRootCertPem}, []byte("\n"))); err != nil {
t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err))
}
if err := tmpCaFile.Close(); err != nil {
t.Fatalf(fmt.Sprintf("Error closing CA tempfile: %s", err))
}
defer os.Remove(tmpCaFile.Name())

ch := make(chan (struct{}))
logger := log.NewNopLogger()
// Handle server side of this test.
serverFunc := func() {
conn, err := ln.Accept()
if err != nil {
panic(fmt.Sprintf("Error accepting on socket: %s", err))
}
defer conn.Close()

serverKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)})

keypair, err := tls.X509KeyPair(serverCertPem, serverKeyPem)
if err != nil {
panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
}

// Immediately upgrade to TLS.
tlsConfig := &tls.Config{
ServerName: "localhost",
Certificates: []tls.Certificate{keypair},
MinVersion: tls.VersionTLS12,
MaxVersion: tls.VersionTLS12,
}
tlsConn := tls.Server(conn, tlsConfig)
defer tlsConn.Close()
if err := tlsConn.Handshake(); err != nil {
level.Error(logger).Log("msg", "Error TLS Handshake (server) failed", "err", err)
} else {
// Send some bytes before terminating the connection.
fmt.Fprintf(tlsConn, "Hello World!\n")
}
ch <- struct{}{}
}

// Expect name-verified TLS connection.
module := config.Module{
TCP: config.TCPProbe{
IPProtocol: "ip4",
IPProtocolFallback: true,
TLS: true,
TLSConfig: pconfig.TLSConfig{
CAFile: tmpCaFile.Name(),
InsecureSkipVerify: false,
},
},
}

registry := prometheus.NewRegistry()
go serverFunc()
// Test name-verification with name from target.
target := net.JoinHostPort("localhost", listenPort)
if !ProbeTCP(testCTX, target, module, registry, log.NewNopLogger()) {
t.Fatalf("TCP module failed, expected success.")
}
<-ch

// Check the resulting metrics.
mfs, err := registry.Gather()
if err != nil {
t.Fatal(err)
}

// Check values
expectedResults := map[string]float64{
"probe_ssl_earliest_cert_expiry": float64(serverCertExpiry.Unix()),
"probe_ssl_last_chain_expiry_timestamp_seconds": float64(serverCertExpiry.Unix()),
"probe_tls_version_info": 1,
}
checkRegistryResults(expectedResults, mfs, t)
}

func TestTCPConnectionQueryResponseStartTLS(t *testing.T) {
ln, err := net.Listen("tcp", "localhost:0")
if err != nil {
Expand All @@ -205,14 +334,16 @@ func TestTCPConnectionQueryResponseStartTLS(t *testing.T) {

// Create test certificates valid for 1 day.
certExpiry := time.Now().AddDate(0, 0, 1)
testcert_pem, testkey_pem := generateTestCertificate(certExpiry, true)
testCertTmpl := generateCertificateTemplate(certExpiry, true)
testCertTmpl.IsCA = true
_, testCertPem, testKey := generateSelfSignedCertificate(testCertTmpl)

// CAFile must be passed via filesystem, use a tempfile.
tmpCaFile, err := ioutil.TempFile("", "cafile.pem")
if err != nil {
t.Fatalf(fmt.Sprintf("Error creating CA tempfile: %s", err))
}
if _, err := tmpCaFile.Write(testcert_pem); err != nil {
if _, err := tmpCaFile.Write(testCertPem); err != nil {
t.Fatalf(fmt.Sprintf("Error writing CA tempfile: %s", err))
}
if err := tmpCaFile.Close(); err != nil {
Expand Down Expand Up @@ -263,7 +394,8 @@ func TestTCPConnectionQueryResponseStartTLS(t *testing.T) {
}
fmt.Fprintf(conn, "220 2.0.0 Ready to start TLS\n")

testcert, err := tls.X509KeyPair(testcert_pem, testkey_pem)
testKeyPem := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(testKey)})
testcert, err := tls.X509KeyPair(testCertPem, testKeyPem)
if err != nil {
panic(fmt.Sprintf("Failed to decode TLS testing keypair: %s\n", err))
}
Expand Down
17 changes: 17 additions & 0 deletions prober/tls.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,23 @@ func getEarliestCertExpiry(state *tls.ConnectionState) time.Time {
return earliest
}

func getLastChainExpiry(state *tls.ConnectionState) time.Time {
lastChainExpiry := time.Time{}
for _, chain := range state.VerifiedChains {
earliestCertExpiry := time.Time{}
for _, cert := range chain {
if (earliestCertExpiry.IsZero() || cert.NotAfter.Before(earliestCertExpiry)) && !cert.NotAfter.IsZero() {
earliestCertExpiry = cert.NotAfter
}
}
if lastChainExpiry.IsZero() || lastChainExpiry.After(earliestCertExpiry) {
lastChainExpiry = earliestCertExpiry
}

}
return lastChainExpiry
}

func getTLSVersion(state *tls.ConnectionState) string {
switch state.Version {
case tls.VersionTLS10:
Expand Down
62 changes: 44 additions & 18 deletions prober/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,18 +79,8 @@ func checkRegistryLabels(expRes map[string]map[string]string, mfs []*dto.MetricF
}
}

// Create test certificate with specified expiry date
// Certificate will be self-signed and use localhost/127.0.0.1
// Generated certificate and key are returned in PEM encoding
func generateTestCertificate(expiry time.Time, IPAddressSAN bool) ([]byte, []byte) {
privatekey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(fmt.Sprintf("Error creating rsa key: %s", err))
}
publickey := &privatekey.PublicKey

cert := x509.Certificate{
IsCA: true,
func generateCertificateTemplate(expiry time.Time, IPAddressSAN bool) *x509.Certificate {
template := &x509.Certificate{
BasicConstraintsValid: true,
SubjectKeyId: []byte{1},
SerialNumber: big.NewInt(1),
Expand All @@ -102,18 +92,54 @@ func generateTestCertificate(expiry time.Time, IPAddressSAN bool) ([]byte, []byt
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
}
cert.DNSNames = append(cert.DNSNames, "localhost")

template.DNSNames = append(template.DNSNames, "localhost")
if IPAddressSAN {
cert.IPAddresses = append(cert.IPAddresses, net.ParseIP("127.0.0.1"))
cert.IPAddresses = append(cert.IPAddresses, net.ParseIP("::1"))
template.IPAddresses = append(template.IPAddresses, net.ParseIP("127.0.0.1"))
template.IPAddresses = append(template.IPAddresses, net.ParseIP("::1"))
}
derCert, err := x509.CreateCertificate(rand.Reader, &cert, &cert, publickey, privatekey)

return template
}

func generateCertificate(template, parent *x509.Certificate, publickey *rsa.PublicKey, privatekey *rsa.PrivateKey) (*x509.Certificate, []byte) {
derCert, err := x509.CreateCertificate(rand.Reader, template, template, publickey, privatekey)
if err != nil {
panic(fmt.Sprintf("Error signing test-certificate: %s", err))
}
cert, err := x509.ParseCertificate(derCert)
if err != nil {
panic(fmt.Sprintf("Error parsing test-certificate: %s", err))
}
pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derCert})
pemKey := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privatekey)})
return pemCert, pemKey
return cert, pemCert

}

func generateSignedCertificate(template, parentCert *x509.Certificate, parentKey *rsa.PrivateKey) (*x509.Certificate, []byte, *rsa.PrivateKey) {
privatekey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(fmt.Sprintf("Error creating rsa key: %s", err))
}
cert, pemCert := generateCertificate(template, parentCert, &privatekey.PublicKey, parentKey)
return cert, pemCert, privatekey
}

func generateSelfSignedCertificate(template *x509.Certificate) (*x509.Certificate, []byte, *rsa.PrivateKey) {
privatekey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(fmt.Sprintf("Error creating rsa key: %s", err))
}
publickey := &privatekey.PublicKey

cert, pemCert := generateCertificate(template, template, publickey, privatekey)
return cert, pemCert, privatekey
}

func generateSelfSignedCertificateWithPrivateKey(template *x509.Certificate, privatekey *rsa.PrivateKey) (*x509.Certificate, []byte) {
publickey := &privatekey.PublicKey
cert, pemCert := generateCertificate(template, template, publickey, privatekey)
return cert, pemCert
}

func TestChooseProtocol(t *testing.T) {
Expand Down

0 comments on commit 725a683

Please sign in to comment.