From f25240818da453d4fcddbd779c7a347fc324ca09 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Thu, 18 Apr 2024 12:16:04 +0000 Subject: [PATCH 01/47] Add support of TLS to xserver --- src/x/server/buffered_conn.go | 75 +++++++++++++++ src/x/server/buffered_conn_test.go | 101 ++++++++++++++++++++ src/x/server/config.go | 41 ++++++++ src/x/server/config_test.go | 18 ++++ src/x/server/options.go | 111 ++++++++++++++++++++++ src/x/server/server.go | 76 ++++++++++++++- src/x/server/server_test.go | 147 ++++++++++++++++++++++++++++- src/x/server/testdata/client.crt | 20 ++++ src/x/server/testdata/client.key | 27 ++++++ src/x/server/testdata/rootCA.crt | 22 +++++ src/x/server/testdata/rootCA.key | 28 ++++++ src/x/server/testdata/server.crt | 20 ++++ src/x/server/testdata/server.key | 27 ++++++ src/x/server/tls_mode.go | 50 ++++++++++ src/x/server/tls_mode_test.go | 43 +++++++++ 15 files changed, 800 insertions(+), 6 deletions(-) create mode 100644 src/x/server/buffered_conn.go create mode 100644 src/x/server/buffered_conn_test.go create mode 100644 src/x/server/testdata/client.crt create mode 100644 src/x/server/testdata/client.key create mode 100644 src/x/server/testdata/rootCA.crt create mode 100644 src/x/server/testdata/rootCA.key create mode 100644 src/x/server/testdata/server.crt create mode 100644 src/x/server/testdata/server.key create mode 100644 src/x/server/tls_mode.go create mode 100644 src/x/server/tls_mode_test.go diff --git a/src/x/server/buffered_conn.go b/src/x/server/buffered_conn.go new file mode 100644 index 0000000000..0aa84edc73 --- /dev/null +++ b/src/x/server/buffered_conn.go @@ -0,0 +1,75 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package server implements a network server. +package server + +import ( + "bufio" + "net" +) + +// TLSHandshakeFirstByte is the first byte of a tls connection handshake +const TLSHandshakeFirstByte = 0x16 + +func newBufferedConn(conn net.Conn) BufferedConn { + return &bufferedConn{ + r: bufio.NewReader(conn), + Conn: conn, + } +} + +// BufferedConn represents the buffered connection +type BufferedConn interface { + net.Conn + IsTLS() (bool, error) + Peek(int) ([]byte, error) + GetConn() net.Conn +} + +type bufferedConn struct { + net.Conn + r *bufio.Reader +} + +// IsTLS returns is the connection is TLS or not +func (b *bufferedConn) IsTLS() (bool, error) { + connBytes, err := b.Peek(1) + if err != nil { + return false, err + } + isTLS := len(connBytes) > 0 && connBytes[0] == TLSHandshakeFirstByte + return isTLS, nil +} + +// Peek returns the next n bytes without advancing the reader +func (b *bufferedConn) Peek(n int) ([]byte, error) { + return b.r.Peek(n) +} + +// Read reads n bytes +func (b *bufferedConn) Read(n []byte) (int, error) { + return b.r.Read(n) +} + +// GetConn returns net.Conn connection +func (b *bufferedConn) GetConn() net.Conn { + return b.Conn +} diff --git a/src/x/server/buffered_conn_test.go b/src/x/server/buffered_conn_test.go new file mode 100644 index 0000000000..137a9fd4a7 --- /dev/null +++ b/src/x/server/buffered_conn_test.go @@ -0,0 +1,101 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package server implements a network server. +package server + +import ( + "crypto/tls" + "net" + "testing" + + "github.com/stretchr/testify/require" +) + +func testTCPServer(connCh chan net.Conn, errCh chan error) (net.Listener, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, err + } + go func(net.Listener, chan net.Conn, chan error) { + conn, err := listener.Accept() + if err != nil { + errCh <- err + } else { + connCh <- conn + } + }(listener, connCh, errCh) + return listener, nil +} + +func TestPlainTCPConnection(t *testing.T) { + connCh := make(chan net.Conn) + errCh := make(chan error) + listener, err := testTCPServer(connCh, errCh) + require.NoError(t, err) + defer listener.Close() + + clientConn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + _, err = clientConn.Write([]byte("not a tls connection")) + require.NoError(t, err) + + var conn BufferedConn + select { + case newConn := <-connCh: + conn = newBufferedConn(newConn) + case newErr := <-errCh: + err = newErr + } + require.NoError(t, err) + defer conn.Close() + + isTLS, err := conn.IsTLS() + require.NoError(t, err) + require.False(t, isTLS) +} + +func TestTLSConnection(t *testing.T) { + connCh := make(chan net.Conn) + errCh := make(chan error) + listener, err := testTCPServer(connCh, errCh) + require.NoError(t, err) + defer listener.Close() + + tcpConn, err := net.Dial("tcp", listener.Addr().String()) + require.NoError(t, err) + tlsConn := tls.Client(tcpConn, &tls.Config{InsecureSkipVerify: true}) + defer tlsConn.Close() + go tlsConn.Handshake() + + var conn BufferedConn + select { + case newConn := <-connCh: + conn = newBufferedConn(newConn) + case newErr := <-errCh: + err = newErr + } + require.NoError(t, err) + defer conn.Close() + + isTLS, err := conn.IsTLS() + require.NoError(t, err) + require.True(t, isTLS) +} diff --git a/src/x/server/config.go b/src/x/server/config.go index d49c1575da..4d1837603e 100644 --- a/src/x/server/config.go +++ b/src/x/server/config.go @@ -40,6 +40,9 @@ type Configuration struct { // KeepAlive period. KeepAlivePeriod *time.Duration `yaml:"keepAlivePeriod"` + + // TLS configuration + TLS *TLSConfiguration `yaml:"tls"` } // NewOptions creates server options. @@ -53,6 +56,9 @@ func (c Configuration) NewOptions(iOpts instrument.Options) Options { if c.KeepAlivePeriod != nil { opts = opts.SetTCPConnectionKeepAlivePeriod(*c.KeepAlivePeriod) } + if c.TLS != nil { + opts = opts.SetTLSOptions(c.TLS.NewOptions()) + } return opts } @@ -60,3 +66,38 @@ func (c Configuration) NewOptions(iOpts instrument.Options) Options { func (c Configuration) NewServer(handler Handler, iOpts instrument.Options) Server { return NewServer(c.ListenAddress, handler, c.NewOptions(iOpts)) } + +// TLSConfiguration configs a tls server +type TLSConfiguration struct { + // Mode is the tls server mode + // disabled - allows plaintext connections only + // permissive - allows both plaintext and TLS connections + // enforced - allows TLS connections only + Mode string `yaml:"mode" validate:"nonzero,regexp=^(disabled|permissive|enforced)$"` + + // MutualTLSEnabled sets mTLS + MutualTLSEnabled bool `yaml:"mTLSEnabled"` + + // CertFile path to a server certificate file + CertFile string `yaml:"certFile"` + + // KeyFile path to a server key file + KeyFile string `yaml:"keyFile"` + + // ClientCAFile path to a CA file for verifying clients + ClientCAFile string `yaml:"clientCAFile"` +} + +// NewOptions creates TLS options +func (c TLSConfiguration) NewOptions() TLSOptions { + opts := NewTLSOptions(). + SetMutualTLSEnabled(c.MutualTLSEnabled). + SetCertFile(c.CertFile). + SetKeyFile(c.KeyFile). + SetClientCAFile(c.ClientCAFile) + var tlsMode TLSMode + if err := tlsMode.UnmarshalText([]byte(c.Mode)); err == nil { + opts = opts.SetMode(tlsMode) + } + return opts +} diff --git a/src/x/server/config_test.go b/src/x/server/config_test.go index 2f353ad3cd..8a525f081d 100644 --- a/src/x/server/config_test.go +++ b/src/x/server/config_test.go @@ -35,6 +35,12 @@ func TestServerConfiguration(t *testing.T) { listenAddress: addr keepAliveEnabled: true keepAlivePeriod: 5s +tls: + mode: enforced + mTLSEnabled: true + certFile: /tmp/cert + keyFile: /tmp/key + clientCAFile: /tmp/ca ` var cfg Configuration @@ -43,9 +49,21 @@ keepAlivePeriod: 5s require.True(t, *cfg.KeepAliveEnabled) require.Equal(t, 5*time.Second, *cfg.KeepAlivePeriod) + require.Equal(t, "enforced", cfg.TLS.Mode) + require.True(t, cfg.TLS.MutualTLSEnabled) + require.Equal(t, "/tmp/cert", cfg.TLS.CertFile) + require.Equal(t, "/tmp/key", cfg.TLS.KeyFile) + require.Equal(t, "/tmp/ca", cfg.TLS.ClientCAFile) + opts := cfg.NewOptions(instrument.NewOptions()) require.Equal(t, 5*time.Second, opts.TCPConnectionKeepAlivePeriod()) require.True(t, opts.TCPConnectionKeepAlive()) + require.Equal(t, TLSEnforced, opts.TLSOptions().Mode()) + require.True(t, opts.TLSOptions().MutualTLSEnabled()) + require.Equal(t, "/tmp/cert", opts.TLSOptions().CertFile()) + require.Equal(t, "/tmp/key", opts.TLSOptions().KeyFile()) + require.Equal(t, "/tmp/ca", opts.TLSOptions().ClientCAFile()) + require.NotNil(t, cfg.NewServer(nil, instrument.NewOptions())) } diff --git a/src/x/server/options.go b/src/x/server/options.go index 46e61370b9..fa05142745 100644 --- a/src/x/server/options.go +++ b/src/x/server/options.go @@ -37,6 +37,33 @@ const ( defaultTCPConnectionKeepAlivePeriod = 10 * time.Second ) +type TLSOptions interface { + // SetMode sets the tls mode + SetMode(value TLSMode) TLSOptions + // Mode returns the tls mode + Mode() TLSMode + + // SetMutualTLSEnabled sets the mutual tls enabled option + SetMutualTLSEnabled(value bool) TLSOptions + // MutualTLSEnabled returns the mutual tls enabled option + MutualTLSEnabled() bool + + // SetCertFile sets the certificate file path + SetCertFile(value string) TLSOptions + // CertFile returns the certificate file path + CertFile() string + + // SetKeyFile sets the private key file path + SetKeyFile(value string) TLSOptions + // KeyFile returns the private key file path + KeyFile() string + + // SetClientCAFile sets the CA file path + SetClientCAFile(value string) TLSOptions + // ClientCAFile returns the CA file path + ClientCAFile() string +} + // Options provide a set of server options type Options interface { // SetInstrumentOptions sets the instrument options @@ -71,6 +98,12 @@ type Options interface { // ListenerOptions sets the listener options for the server. ListenerOptions() xnet.ListenerOptions + + // SetTLSOptions sets the tls options for the server + SetTLSOptions(value TLSOptions) Options + + // TLSOptions returns the tls options for the server + TLSOptions() TLSOptions } type options struct { @@ -79,6 +112,7 @@ type options struct { tcpConnectionKeepAlive bool tcpConnectionKeepAlivePeriod time.Duration listenerOpts xnet.ListenerOptions + tlsOptions TLSOptions } // NewOptions creates a new set of server options @@ -89,6 +123,7 @@ func NewOptions() Options { tcpConnectionKeepAlive: defaultTCPConnectionKeepAlive, tcpConnectionKeepAlivePeriod: defaultTCPConnectionKeepAlivePeriod, listenerOpts: xnet.NewListenerOptions(), + tlsOptions: NewTLSOptions(), } } @@ -141,3 +176,79 @@ func (o *options) SetListenerOptions(value xnet.ListenerOptions) Options { func (o *options) ListenerOptions() xnet.ListenerOptions { return o.listenerOpts } + +func (o *options) SetTLSOptions(value TLSOptions) Options { + opts := *o + opts.tlsOptions = value + return &opts +} + +func (o *options) TLSOptions() TLSOptions { + return o.tlsOptions +} + +type tlsOptions struct { + mode TLSMode + mTLSEnabled bool + certFile string + keyFile string + clientCAFile string +} + +// NewTLSOptions creates a new set of tls options +func NewTLSOptions() TLSOptions { + return &tlsOptions{ + mode: TLSDisabled, + mTLSEnabled: false, + } +} + +func (o *tlsOptions) SetMode(value TLSMode) TLSOptions { + opts := *o + opts.mode = value + return &opts +} + +func (o *tlsOptions) Mode() TLSMode { + return o.mode +} + +func (o *tlsOptions) SetMutualTLSEnabled(value bool) TLSOptions { + opts := *o + opts.mTLSEnabled = value + return &opts +} + +func (o *tlsOptions) MutualTLSEnabled() bool { + return o.mTLSEnabled +} + +func (o *tlsOptions) SetCertFile(value string) TLSOptions { + opts := *o + opts.certFile = value + return &opts +} + +func (o *tlsOptions) CertFile() string { + return o.certFile +} + +func (o *tlsOptions) SetKeyFile(value string) TLSOptions { + opts := *o + opts.keyFile = value + return &opts +} + +func (o *tlsOptions) KeyFile() string { + return o.keyFile +} + +func (o *tlsOptions) SetClientCAFile(value string) TLSOptions { + opts := *o + opts.clientCAFile = value + return &opts +} + +func (o *tlsOptions) ClientCAFile() string { + return o.clientCAFile +} diff --git a/src/x/server/server.go b/src/x/server/server.go index 09da8c08d0..950979b357 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -22,7 +22,11 @@ package server import ( + "crypto/tls" + "crypto/x509" + "fmt" "net" + "os" "sync" "sync/atomic" "time" @@ -91,6 +95,7 @@ type server struct { metrics serverMetrics handler Handler listenerOpts xnet.ListenerOptions + tlsOpts TLSOptions addConnectionFn addConnectionFn removeConnectionFn removeConnectionFn @@ -112,6 +117,7 @@ func NewServer(address string, handler Handler, opts Options) Server { metrics: newServerMetrics(scope), handler: handler, listenerOpts: opts.ListenerOptions(), + tlsOpts: opts.TLSOptions(), } // Set up the connection functions. @@ -140,16 +146,82 @@ func (s *server) Serve(l net.Listener) error { return nil } +func (s *server) upgradeToTLS(conn BufferedConn) (BufferedConn, error) { + certPool := x509.NewCertPool() + if s.tlsOpts.ClientCAFile() != "" { + certs, err := os.ReadFile(s.tlsOpts.ClientCAFile()) + if err != nil { + return conn, fmt.Errorf("read bundle error: %w", err) + } + if ok := certPool.AppendCertsFromPEM(certs); !ok { + return conn, fmt.Errorf("cannot append cert to cert pool") + } + } + clientAuthType := tls.NoClientCert + if s.tlsOpts.MutualTLSEnabled() { + clientAuthType = tls.RequireAndVerifyClientCert + } + tlsConfig := &tls.Config{ + ClientCAs: certPool, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(s.tlsOpts.CertFile(), s.tlsOpts.KeyFile()) + if err != nil { + return nil, fmt.Errorf("load x509 key pair error: %w", err) + } + return &cert, nil + }, + ClientAuth: clientAuthType, + } + tlsConn := tls.Server(conn, tlsConfig) + return newBufferedConn(tlsConn), nil +} + +func (s *server) maybeUpgradeToTLS(conn BufferedConn) (BufferedConn, error) { + switch s.tlsOpts.Mode() { + case TLSPermissive: + isTLSConnection, err := conn.IsTLS() + if err != nil { + return conn, err + } + if isTLSConnection { + conn, err = s.upgradeToTLS(conn) + if err != nil { + return conn, err + } + } + case TLSEnforced: + var err error + var isTLSConnection bool + isTLSConnection, err = conn.IsTLS() + if err != nil { + return conn, err + } + if !isTLSConnection { + return conn, fmt.Errorf("Not a tls connection") + } + conn, err = s.upgradeToTLS(conn) + if err != nil { + return conn, err + } + } + return conn, nil +} + func (s *server) serve() { connCh, errCh := xnet.StartForeverAcceptLoop(s.listener, s.retryOpts) for conn := range connCh { - conn := conn - if tcpConn, ok := conn.(*net.TCPConn); ok { + conn := newBufferedConn(conn) + if tcpConn, ok := conn.GetConn().(*net.TCPConn); ok { tcpConn.SetKeepAlive(s.tcpConnectionKeepAlive) if s.tcpConnectionKeepAlivePeriod != 0 { tcpConn.SetKeepAlivePeriod(s.tcpConnectionKeepAlivePeriod) } } + conn, err := s.maybeUpgradeToTLS(conn) + if err != nil { + conn.Close() + continue + } if !s.addConnectionFn(conn) { conn.Close() } else { diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index 20749a638c..fb7e146d3d 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -21,6 +21,7 @@ package server import ( + "crypto/tls" "fmt" "net" "sort" @@ -39,14 +40,21 @@ const ( ) // nolint: unparam -func testServer(addr string) (*server, *mockHandler, *int32, *int32) { +func testServer(addr string, tlsMode TLSMode, mTLSEnabled bool) (*server, *mockHandler, *int32, *int32) { var ( numAdded int32 numRemoved int32 ) + tlsOpts := NewTLSOptions(). + SetMode(tlsMode). + SetMutualTLSEnabled(mTLSEnabled). + SetClientCAFile("./testdata/rootCA.crt"). + SetCertFile("./testdata/server.crt"). + SetKeyFile("./testdata/server.key") + opts := NewOptions().SetRetryOptions(retry.NewOptions().SetMaxRetries(2)) - opts = opts.SetInstrumentOptions(opts.InstrumentOptions().SetReportInterval(time.Second)) + opts = opts.SetInstrumentOptions(opts.InstrumentOptions().SetReportInterval(time.Second)).SetTLSOptions(tlsOpts) h := newMockHandler() s := NewServer(addr, h, opts).(*server) @@ -66,7 +74,7 @@ func testServer(addr string) (*server, *mockHandler, *int32, *int32) { } func TestServerListenAndClose(t *testing.T) { - s, h, numAdded, numRemoved := testServer(testListenAddress) + s, h, numAdded, numRemoved := testServer(testListenAddress, TLSDisabled, false) var ( numClients = 9 @@ -104,7 +112,7 @@ func TestServerListenAndClose(t *testing.T) { } func TestServe(t *testing.T) { - s, _, _, _ := testServer(testListenAddress) + s, _, _, _ := testServer(testListenAddress, TLSDisabled, false) l, err := net.Listen("tcp", testListenAddress) require.NoError(t, err) @@ -117,6 +125,137 @@ func TestServe(t *testing.T) { s.Close() } +func TestTLSPermissiveServerListenAndClose(t *testing.T) { + s, h, numAdded, numRemoved := testServer(testListenAddress, TLSPermissive, false) + + var ( + numClients = 9 + expectedRes []string + ) + + err := s.ListenAndServe() + require.NoError(t, err) + listenAddr := s.listener.Addr().String() + + for i := 0; i < numClients; i++ { + var conn net.Conn + var err error + if i%2 == 0 { + conn, err = net.Dial("tcp", listenAddr) + } else { + conn, err = tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true}) + + } + require.NoError(t, err) + + msg := fmt.Sprintf("msg%d", i) + expectedRes = append(expectedRes, msg) + + _, err = conn.Write([]byte(msg)) + require.NoError(t, err) + } + + for h.called() < numClients { + time.Sleep(100 * time.Millisecond) + } + + require.False(t, h.isClosed()) + + s.Close() + + require.True(t, h.isClosed()) + require.Equal(t, int32(numClients), atomic.LoadInt32(numAdded)) + require.Equal(t, int32(numClients), atomic.LoadInt32(numRemoved)) + require.Equal(t, numClients, h.called()) + require.Equal(t, expectedRes, h.res()) +} + +func TestTLSEnforcedServerListenAndClose(t *testing.T) { + s, h, numAdded, numRemoved := testServer(testListenAddress, TLSEnforced, false) + + var ( + numClients = 10 + expectedRes []string + ) + + err := s.ListenAndServe() + require.NoError(t, err) + listenAddr := s.listener.Addr().String() + + for i := 0; i < numClients; i++ { + var conn net.Conn + var err error + msg := fmt.Sprintf("msg%d", i) + + if i%2 == 0 { + conn, err = net.Dial("tcp", listenAddr) + } else { + conn, err = tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true}) + expectedRes = append(expectedRes, msg) + } + require.NoError(t, err) + + _, err = conn.Write([]byte(msg)) + require.NoError(t, err) + } + + for h.called() < numClients/2 { + time.Sleep(100 * time.Millisecond) + } + + require.False(t, h.isClosed()) + + s.Close() + + require.True(t, h.isClosed()) + require.Equal(t, int32(numClients)/2, atomic.LoadInt32(numAdded)) + require.Equal(t, int32(numClients)/2, atomic.LoadInt32(numRemoved)) + require.Equal(t, numClients/2, h.called()) + require.Equal(t, expectedRes, h.res()) +} + +func TestMutualTLSServerListenAndClose(t *testing.T) { + s, h, numAdded, numRemoved := testServer(testListenAddress, TLSEnforced, true) + + var ( + numClients = 9 + expectedRes []string + ) + + err := s.ListenAndServe() + require.NoError(t, err) + listenAddr := s.listener.Addr().String() + cert, err := tls.LoadX509KeyPair("./testdata/client.crt", "./testdata/client.key") + require.NoError(t, err) + + for i := 0; i < numClients; i++ { + var conn net.Conn + var err error + conn, err = tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true, Certificates: []tls.Certificate{cert}}) + require.NoError(t, err) + + msg := fmt.Sprintf("msg%d", i) + expectedRes = append(expectedRes, msg) + + _, err = conn.Write([]byte(msg)) + require.NoError(t, err) + } + + for h.called() < numClients { + time.Sleep(100 * time.Millisecond) + } + + require.False(t, h.isClosed()) + + s.Close() + + require.True(t, h.isClosed()) + require.Equal(t, int32(numClients), atomic.LoadInt32(numAdded)) + require.Equal(t, int32(numClients), atomic.LoadInt32(numRemoved)) + require.Equal(t, numClients, h.called()) + require.Equal(t, expectedRes, h.res()) +} + type mockHandler struct { sync.Mutex diff --git a/src/x/server/testdata/client.crt b/src/x/server/testdata/client.crt new file mode 100644 index 0000000000..366d36e8b8 --- /dev/null +++ b/src/x/server/testdata/client.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPzCCAicCFFJVDW2T6lamcRZYStsBCQTKicKwMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAk5MMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwHhcNMjQw +NDE1MTQ0NjEyWhcNMzQwNDEzMTQ0NjEyWjBeMQswCQYDVQQGEwJOTDETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRcwFQYDVQQDDA5UZXN0Q2xpZW50Q2VydDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAMXgv5tgZD5iBW3LfTC27jPHTLlsAw8ZlBFLmd7potdS3Hy512Xa +kKOaKRKH/0Yla4BKGNh0ZP+xege15CpTFqwtzDPCRkaRvfSxRW19MukeergRDqqI +tEwURHQ5gfub7XB9e5wuNmW2GGs/xgiWsxxm6UMidXkhA5a2//OmDSm2JdcoxJbk +1m1OUzXUuLKLzCIRAbJFFIM1RpvjWSonxpJ2lsNCVQ+LdItxE9mAI+Y+PaDonyzg +ehfAqkl+wxGRevaDFFJr1defxmnJhaYv40AcEpXOWeetZFlspEtrPF7b/3GifRdx +Mlwq27wn3mk4wF1wa12kRNKiz+HHjZrB7UECAwEAATANBgkqhkiG9w0BAQsFAAOC +AQEAKblZ+C4Pdv5grKiCQcfFt53SbIys8uavxmzJNn9nq1QVgBmR/hX8yG8ULeh5 +kKrccwe2WXCBWsIj9cvGCzR+MPMvuIXd3d58yLybmIUBDfJSBC0v5TaNWP2ZdJDG +fH38DfQTeE9XcjpRtCGYtYwReBptSlNiYQlvwkukTNz3mV+0BpvQu/9/R0BNCJAF +9VeLgUcG7T7HC+foCeU7h83Y0xH0OALe33ntcrMPHUclXqImCiC7dl8pb3H7o/xJ +jS3KZxfv6ioxpirvpEbX/EeF+H1HQZUZ8y3+J3K37jMgzNamd/xsaJxEfCq4SE+n +HmgeXgqoNjrM6J0jYAfx1MjbIA== +-----END CERTIFICATE----- diff --git a/src/x/server/testdata/client.key b/src/x/server/testdata/client.key new file mode 100644 index 0000000000..ff226a549e --- /dev/null +++ b/src/x/server/testdata/client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAxeC/m2BkPmIFbct9MLbuM8dMuWwDDxmUEUuZ3umi11LcfLnX +ZdqQo5opEof/RiVrgEoY2HRk/7F6B7XkKlMWrC3MM8JGRpG99LFFbX0y6R56uBEO +qoi0TBREdDmB+5vtcH17nC42ZbYYaz/GCJazHGbpQyJ1eSEDlrb/86YNKbYl1yjE +luTWbU5TNdS4sovMIhEBskUUgzVGm+NZKifGknaWw0JVD4t0i3ET2YAj5j49oOif +LOB6F8CqSX7DEZF69oMUUmvV15/GacmFpi/jQBwSlc5Z561kWWykS2s8Xtv/caJ9 +F3EyXCrbvCfeaTjAXXBrXaRE0qLP4ceNmsHtQQIDAQABAoIBAAtyP8MuJT5SjzvV +rI0317mZCsAjFl42PZFujR0O6MOJ4IU6ftI+fWVpUnzm7wZQvdIy9xL2UK1Vx9hQ +Vj14hvQ4xfosf8IvRgy0gG6f8mT3xWOGYRHOTJemCHuso+85CtgZ+h+DsNPbX7g8 +fSkcBopbDZ07jg4OsdVzCoU+kr5Z1/VDe0O0rTDzGVP686tN9I+DmDYv2HlBUaSZ +SyNZrLJFziqFx4ATTIE3aTLd29pzTl09WiCcZSxhlS+ROnN2iW2xDLNgY9SdUo2a +3r8N+xhYDNQz2paxlsv2x/tCRJvrQDoX7S5QZw4P1pVr6wo6xC+BU2UlpVHifo4M +egyDbAECgYEA5Y9+JDOSQF2cvIV4xTMgVnkLig/ngYesS44xz8HYTRR5s1WW6oCX +3j+OosZBbJpXeBlTK31G9Z92+Om2/Cj2uDvff7EJdhZ5sAdMtUy5X7p9rGTEyi5F +ecZDBfmwHH0lFdwpNtlky1uoaUsVd4IS28WMHoHYDGW241IW8xOlmeECgYEA3Ksb +KR7JCbHx21wh5oPVn5BLpdag7D+M1jBFtjPrzLDNhXlsV9zkjn7xETvTXWU2vNlh +OkoRu4VFEAnsgtxh+gT0lo9ZG9IWi2qP99vPpB3AFYuujBu3GA02/iOo5pri/6R0 +ZNEEreAaaOcT2z6K9HG0KMQ6QpYgXSlzJW8sf2ECgYEAvn2fMA03bIAB4xJi0EkH +qZoScEOYWQ0rdRsOzJbPlc7K2nzImdmRrFRTWVFo0uUUdk2VjX4Mlx/3ir/uHzsi +2GieowhWkI4/9kloZv2+yegoBxkrj5ZsAov57AhxEoLqdkRWUvR8xp9Nleo/awcd +/Q7loh8fF9KDvAjPkHAaOCECgYA98bZNI7wxgYcwGbvWdrmX8iyaIBbKWsiRM7nN +/OM7cYIv7rbwLyzlp1LKkK2zsP7donP9pd82caHCb9a5oV3LjmqOfSz5d08m0cIa +RNUT79oE8lIMOJd8I/GFA8OdAGuqcaLOzjHvEVK4ke1sBTGCjwyQyQzFtljdbg5J +utyV4QKBgHkSBKwrgaump08MJFfK9uFo5sr5jy692XYWMlBVhc+piPwUsoztjI1O +8cvie0mMj6oofSoSyD4FUg9kFUCRMPC0kga/zlHyL1Y/uILX4XUFMl4P/gMbC7kw +0d6BdEsp5SU8Onc9eV/EW+q2Mz7/t5ZHkUrdDQYGgkIBqRZcXlka +-----END RSA PRIVATE KEY----- diff --git a/src/x/server/testdata/rootCA.crt b/src/x/server/testdata/rootCA.crt new file mode 100644 index 0000000000..5d92901e85 --- /dev/null +++ b/src/x/server/testdata/rootCA.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUUwZEZu6XUKW03HPLHGDlt/d/BeUwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCTkwxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDETMBEGA1UEAwwKVGVzdFJvb3RDQTAe +Fw0yNDA0MDQxMTAwMzhaFw0zNDAxMDIxMTAwMzhaMFoxCzAJBgNVBAYTAk5MMRMw +EQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0 +eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDRwn7QLv6zup98kB9VJ1HvHQQMZxt2tqhpFGEc0bXiGrmhFKjV +XZ264ALnUJHQgM8rkRe+dTXg8455FPTMoT+O/BC44pt8CaY1C5kQzVkLaMwaVVxV +vsBfdZ9LOKJQmUXIf1qV/UK3P55Oc3TvwSQW3tU0eMMEOnuNdA4lDdlQnExb7tIJ +4sl0i2w6rB7CoS/0gx2eUPLhPK+vcwCM3d81Odb3ULVznyJ5ZUd5mYY85XXVFQdW +QSyaEE0ekzbmXQzjiqNAt67WEmbadoKdeg8jPG5ka8qBZ21ZGjCHQ8AbzpOn+eDk +n50zwxncpSYVxTilV70PbFSY33PdmdS6XjEnAgMBAAGjUzBRMB0GA1UdDgQWBBSo +kzZddlleTXZYQsBc8efmsZj3WTAfBgNVHSMEGDAWgBSokzZddlleTXZYQsBc8efm +sZj3WTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQACb1ZT3WPK +XdO6jYbZuMN5BBET+mNKQOYrdehY5oiLgr5sDwTwfZvf+q6ic7MpDRBOWMSQuYwp +aUxK7qfw4nSSZb4zDPpzB6Tauh7M5TTPYQwzesn9/JaUBO4vCTwOFGuWxEbXe6PF +VuQOyGQCR9a7juNgoDLiRpWi+xXHz1kMNSeZqV8bn7eo/SIziuSW1JScse91Brt1 +Db9L8iSJpKLDLtwv8fU4/NSORc/672uy4OLxNppsTthh1rk94KbC/FYllC6e8N8n +0iffEn4xE/CECYmAhvplXXS20hCTGbs7tPJ3DGsRfHOyI9RXiayNZuq9GIYKv6eR ++h7NYwXua7oP +-----END CERTIFICATE----- diff --git a/src/x/server/testdata/rootCA.key b/src/x/server/testdata/rootCA.key new file mode 100644 index 0000000000..6d8a527ee2 --- /dev/null +++ b/src/x/server/testdata/rootCA.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDRwn7QLv6zup98 +kB9VJ1HvHQQMZxt2tqhpFGEc0bXiGrmhFKjVXZ264ALnUJHQgM8rkRe+dTXg8455 +FPTMoT+O/BC44pt8CaY1C5kQzVkLaMwaVVxVvsBfdZ9LOKJQmUXIf1qV/UK3P55O +c3TvwSQW3tU0eMMEOnuNdA4lDdlQnExb7tIJ4sl0i2w6rB7CoS/0gx2eUPLhPK+v +cwCM3d81Odb3ULVznyJ5ZUd5mYY85XXVFQdWQSyaEE0ekzbmXQzjiqNAt67WEmba +doKdeg8jPG5ka8qBZ21ZGjCHQ8AbzpOn+eDkn50zwxncpSYVxTilV70PbFSY33Pd +mdS6XjEnAgMBAAECggEAc7MZadGSMibNU9LXRYwDuKXDlufEkU2dRSC1lETkU0w0 +efJAUjhqCSsIzvh1BZIXM8u7UhWq22LcglMZ2ElVGpRU0PZFlzxnTjkQe+EYrZ9n +fVr6LF2kFMN3UvFxLo3snPQKTlM82oEf29v5c0mQk4DkDyZRQ2LSxapvJuclqD0e +iMEG4Yad7p86AmJm7vpe9yp0POhfF5k9ZOAmrov67LMrFPRpWjeRr2ISa2Y/pjYI +vMqL+jGQi72rasNC0JYKyqn+v+LzocZGL4rCQUIpNmKkw81VolwkU8mfe87kneqh +lBsUgmg9nGtNnoFoxxeBNCEQGXf9+BhdGujujycmUQKBgQDo+a+/l/hn3E+7Tqup +nTAuZn80ODpQkfQ/L+jm6G0ENkil62kdyQCUZFfZqpj9dQwSPzCCViXrZymTL35G +1pzZqejPYGANRmiMeQMOOvyUwfIaEz3U9aj7hWl1v4WLUPzEUTTOsG8DBsR/lHBg +ZC081DH8U3m9fYN+Rp6xdT8z+QKBgQDmfXNfrNjFyxnB+kTJovM0k0f6p5HLKjF3 +VNFz0vpNgsa7DHXrmHSBBENi9oj4z72UQo1Q3bTRezR2TLZpsUmkGNEL0PoXTE1U +YVEb/VStLIZeKFv83uD2qvaYw2gA5785SNeinpweNhTC55T0ixojUmVA9eeIT/ex +nBDDJBuWHwKBgDQ12JQIW6vy7I9edwwA5Q5Q/ArY2wC5ZNJQS1KMHfpGrAs68Yiy +RgX7YtCt8bFDbNwd+yIKal8R9Hg+uX7ok6gX8suenof7Em0ApZWn1HkF6dq8GyxB +jYgogtTXgfcRxEO+qyXy1j4IYzrwKir/6D9sknMoxeyYV0KSUvgT/YEJAoGAaCIb +gwlLcqlc/Md+Vn75VDKKXZNhiiGI8bnvW13hWi2Qbaemiwd482UisM5jec4Zf6dF +w1g3PkFkpWHpM/02IR5ZK/aBVw9RDKNfCr88h3TLTDT9wlRL3QXGnaQDFA2f1liz +m7P/IqMaZChOouFJsNWkC2JN9cbzSFoTNKbWk88CgYAS37I31C6xyWlucbztFpmv +YqEPYRpIbRnJUq+F2Kof8dfT44C3rYwRxPHp2LMgyuPxdE491+Gfwsiv587KuBOP +XpCLuWNN38UDndQ5Zcaxl025akSnAF28EitjZCWzw+8TO/52358vpypwSYkdvQNx +or4Q8/AazdVqWUDo3sARPQ== +-----END PRIVATE KEY----- diff --git a/src/x/server/testdata/server.crt b/src/x/server/testdata/server.crt new file mode 100644 index 0000000000..3618a40052 --- /dev/null +++ b/src/x/server/testdata/server.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPzCCAicCFFJVDW2T6lamcRZYStsBCQTKicKvMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAk5MMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwHhcNMjQw +NDA0MTE0ODIwWhcNMzEwNzA3MTE0ODIwWjBeMQswCQYDVQQGEwJOTDETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRcwFQYDVQQDDA5UZXN0U2VydmVyQ2VydDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAOoGyqgrsBCufd9UNt6wr9UJzPaVqdrr/Zble2UPMVZOynAT69W5 +gMdNvrIDU6l1EnRkLyxmP7+beYZvWyU+kN9LyP0pdcgSQdtyhbr1CkqMxhM1MA2B +OFvcEFN2rybaqcLDlq2WUQkmyuVQt7/weLU9jC596PtxUL1e0+wqIItISWpcmemf +cwH9TVlD8Yh6emf87v27afxc0bZCauPsrEPvOC00oU4epXnA7TeDs4RtfqVAuVHX +oEG3oNZ3NHXIIgSht3rNAEfOxBX+7KGvW4hW9ci6W758ECGzHFJThhzO3+w+0YPP +29m+v4CVRaXAGxkVRPiAUDpU8RSPteywTyMCAwEAATANBgkqhkiG9w0BAQsFAAOC +AQEAuQME+WSPw1KKK1jgh1wuiyM5hou1BeagfwOipM3CEblTiHfoERsd2f7Oi6e2 +zPACrvKjVP6bHKjZPEztFHGXDuLdBqK97Vd/o4IFl6p/8EuBeWV85o4Rk1PMhqBI +0jSxsXXhBLgSTkUBgSllwgAbAaxm+L3wBJiQOhh/wEdfNSJ/GeL5477J56Q12/hA +rRCghYRAVVpr44NfG/PbuuMklQHK2l+wkJFBnx+zAAtZF/nb5yCVSy8z/p2TT72a +STfegXYJtE3p4X2+78xajrI7x/PWhNlzJRn3txLW6461vYJIB3ZDVUA2LZtmi3nd +9BUuJVSvsoVwlvrUBF0S1P0vWg== +-----END CERTIFICATE----- diff --git a/src/x/server/testdata/server.key b/src/x/server/testdata/server.key new file mode 100644 index 0000000000..6b6e1cd4cc --- /dev/null +++ b/src/x/server/testdata/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA6gbKqCuwEK5931Q23rCv1QnM9pWp2uv9luV7ZQ8xVk7KcBPr +1bmAx02+sgNTqXUSdGQvLGY/v5t5hm9bJT6Q30vI/Sl1yBJB23KFuvUKSozGEzUw +DYE4W9wQU3avJtqpwsOWrZZRCSbK5VC3v/B4tT2MLn3o+3FQvV7T7Cogi0hJalyZ +6Z9zAf1NWUPxiHp6Z/zu/btp/FzRtkJq4+ysQ+84LTShTh6lecDtN4OzhG1+pUC5 +UdegQbeg1nc0dcgiBKG3es0AR87EFf7soa9biFb1yLpbvnwQIbMcUlOGHM7f7D7R +g8/b2b6/gJVFpcAbGRVE+IBQOlTxFI+17LBPIwIDAQABAoIBAG7rZSYr/rkJWxN8 +wLzvlYctq/27ldPzQjBA+Ck/+CxmD8DrUD3uPEE6cgXBWefZWzPbQBiVJeXU9NZo +9RREATo0m0CSZAa90IUIVKX1+ji3XDJCZVIhSusO9JX6jK2nugpb1ZDUy/fAcS4b +AdxxR2TvlP98Ie77GD+JMHygwugAJPQ4AFZQjO2F/2k3crVt9Frt1VSvf72nkM/C +sDJy4kv+lDZsbeveNPzJo1honiKdCDWpg4duLiLkkG4YvftWHGzpXHoOWnuwnrT5 +jrQTaG10d1jMpnxOwDrxRsFYWtz4kbJm2VtRsgWRIy4A4Vv8VKfFJ0cSu0CChjTE +7yVrExECgYEA9orYMAuuu/c0HGVcXOsA9x6XHtBXr5wK1fI973sGUrtkshTlBgjG +Bc7H+AhVBVdqr2ipm9pFZYSV1OYB2+3AgMQjhoZANR9jr5bmRjJgWuBi5umrFKa4 +n0IQbkwSzjxKvkhpJK25BY2kpjh7/128Q80PMkKnm0LR2gyUWvLFuUUCgYEA8wEJ +PyxHBu93NnbwTP0SUEOYFa654ctiVLbicdcUsbo/Qo4gl/0AIpGmMdeKqx3FWn9I +kCH1xlvb7hp7FNFhDZIiw6CT8V2RQHXdEO9/S0jjOOBg29hxARIgoU+PjjaQG/H+ +SqlDR/ijXUghlllwtP6jCJqy29+Ukpx8WnIkiUcCgYEA4DNXcinNlyWw//m1StJz +KEdrShxRAWO3/uOyKywHjgjYA5I4y4oC+pkXBJofJyJd5HIbe5286hruXpNkMWo+ +W1sCq30+upB3COEDO+bn9PyFNFRevgL3deyQIJ2RpHbzyaLz6uUIkuJCTt6i9JF7 +eXTrUtuGrAzR5ab26tZHtLkCgYBZtW42diAafVjjMHFx2pNiMjH4/zH8G1tHruRV +5nLrUbh8139M3Mep+18BU3MLzMfIquic3Qygl9TPR9NcUhekLOjMk4FKP6KZy2k7 +EU9K+bN4kczwHC1LmeeV5Ys3kwkMZ//ajDYcYbvdef18RbSH/OEvEf0pkpaEsWRu +92hpjQKBgH6Y3dW0bQIhXikjboD00vPn0zPuh35I9OfrPHMqDJiz+xom7Sz9tlGk +NfFll5PTf0gZtU7160uzhCp/OqXT3IdVgVt/G5ey13oYP7pCb8MWiuyNW0yUgtTp +GM5GUnEW5zJ1t2qJOxwoq/FA7D5C7EGRM/QpoMD0tsJoJKYyWkJO +-----END RSA PRIVATE KEY----- diff --git a/src/x/server/tls_mode.go b/src/x/server/tls_mode.go new file mode 100644 index 0000000000..04e8f5a1c3 --- /dev/null +++ b/src/x/server/tls_mode.go @@ -0,0 +1,50 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package server implements a network server. +package server + +import "fmt" + +const ( + // TLSDisabled allows plaintext connections only + TLSDisabled TLSMode = iota + // TLSPermissive allows both TLS and plaintext connections + TLSPermissive + // TLSEnforced allows TLS connections only + TLSEnforced +) + +// TLSMode represents the TLS mode +type TLSMode uint16 + +func (t *TLSMode) UnmarshalText(mode []byte) error { + switch string(mode) { + case "disabled": + *t = TLSDisabled + case "permissive": + *t = TLSPermissive + case "enforced": + *t = TLSEnforced + default: + return fmt.Errorf("unknown tls mode: %s", mode) + } + return nil +} diff --git a/src/x/server/tls_mode_test.go b/src/x/server/tls_mode_test.go new file mode 100644 index 0000000000..590566a487 --- /dev/null +++ b/src/x/server/tls_mode_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package server implements a network server. +package server + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTLSModeUnmarshal(t *testing.T) { + var tlsMode TLSMode + + require.NoError(t, tlsMode.UnmarshalText([]byte("disabled"))) + require.Equal(t, TLSDisabled, tlsMode) + + require.NoError(t, tlsMode.UnmarshalText([]byte("permissive"))) + require.Equal(t, TLSPermissive, tlsMode) + + require.NoError(t, tlsMode.UnmarshalText([]byte("enforced"))) + require.Equal(t, TLSEnforced, tlsMode) + + require.Error(t, tlsMode.UnmarshalText([]byte("unknown"))) +} From 413d9d3c9694b2f866cd505b1377f7f63e4879aa Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Thu, 18 Apr 2024 12:21:43 +0000 Subject: [PATCH 02/47] Add support of TLS to aggregator tcp client --- src/aggregator/client/config.go | 35 ++++++ src/aggregator/client/config_test.go | 19 +++ src/aggregator/client/conn.go | 43 +++++++ src/aggregator/client/conn_options.go | 18 +++ src/aggregator/client/conn_test.go | 83 +++++++++++++ src/aggregator/client/options.go | 1 + src/aggregator/client/testdata/client.crt | 20 ++++ src/aggregator/client/testdata/client.key | 27 +++++ src/aggregator/client/testdata/rootCA.crt | 22 ++++ src/aggregator/client/testdata/rootCA.key | 28 +++++ src/aggregator/client/testdata/server.crt | 20 ++++ src/aggregator/client/testdata/server.key | 27 +++++ src/aggregator/client/tls_options.go | 136 ++++++++++++++++++++++ 13 files changed, 479 insertions(+) create mode 100644 src/aggregator/client/testdata/client.crt create mode 100644 src/aggregator/client/testdata/client.key create mode 100644 src/aggregator/client/testdata/rootCA.crt create mode 100644 src/aggregator/client/testdata/rootCA.key create mode 100644 src/aggregator/client/testdata/server.crt create mode 100644 src/aggregator/client/testdata/server.key create mode 100644 src/aggregator/client/tls_options.go diff --git a/src/aggregator/client/config.go b/src/aggregator/client/config.go index 2c9b2c93c3..1458d288b5 100644 --- a/src/aggregator/client/config.go +++ b/src/aggregator/client/config.go @@ -207,6 +207,37 @@ func (c *Configuration) NewClientOptions( return opts, nil } +// TLSConfiguration contains the TLS configuration +type TLSConfiguration struct { + TLSEnabled bool `yaml:"tlsEnabled"` + InsecureSkipVerify *bool `yaml:"insecureSkipVerify"` + ServerName *string `yaml:"serverName"` + CAFile *string `yaml:"caFile"` + CertFile *string `yaml:"certFile"` + KeyFile *string `yaml:"keyFile"` +} + +// NewTLSOptions creates new TLS options +func (c *TLSConfiguration) NewTLSOptions() TLSOptions { + opts := NewTLSOptions().SetTLSEnabled(c.TLSEnabled) + if c.InsecureSkipVerify != nil { + opts = opts.SetInsecureSkipVerify(*c.InsecureSkipVerify) + } + if c.ServerName != nil { + opts = opts.SetServerName(*c.ServerName) + } + if c.CAFile != nil { + opts = opts.SetCAFile(*c.CAFile) + } + if c.CertFile != nil { + opts = opts.SetCertFile(*c.CertFile) + } + if c.KeyFile != nil { + opts = opts.SetKeyFile(*c.KeyFile) + } + return opts +} + // ConnectionConfiguration contains the connection configuration. type ConnectionConfiguration struct { ConnectionTimeout time.Duration `yaml:"connectionTimeout"` @@ -217,6 +248,7 @@ type ConnectionConfiguration struct { ReconnectThresholdMultiplier int `yaml:"reconnectThresholdMultiplier"` MaxReconnectDuration *time.Duration `yaml:"maxReconnectDuration"` WriteRetries *retry.Configuration `yaml:"writeRetries"` + TLS *TLSConfiguration `yaml:"tls"` } // NewConnectionOptions creates new connection options. @@ -247,6 +279,9 @@ func (c *ConnectionConfiguration) NewConnectionOptions(scope tally.Scope) Connec retryOpts := c.WriteRetries.NewOptions(scope) opts = opts.SetWriteRetryOptions(retryOpts) } + if c.TLS != nil { + opts = opts.SetTLSOptions(c.TLS.NewTLSOptions()) + } return opts } diff --git a/src/aggregator/client/config_test.go b/src/aggregator/client/config_test.go index 3f765f4874..dccf19a31c 100644 --- a/src/aggregator/client/config_test.go +++ b/src/aggregator/client/config_test.go @@ -80,6 +80,13 @@ connection: maxBackoff: 1s maxRetries: 2 jitter: true + tls: + tlsEnabled: false + insecureSkipVerify: true + serverName: TestServer + caFile: /tmp/ca + certFile: /tmp/cert + keyFile: /tmp/key ` func TestConfigUnmarshal(t *testing.T) { @@ -120,6 +127,12 @@ func TestConfigUnmarshal(t *testing.T) { require.Equal(t, time.Second, cfg.Connection.WriteRetries.MaxBackoff) require.Equal(t, 2, cfg.Connection.WriteRetries.MaxRetries) require.Equal(t, true, *cfg.Connection.WriteRetries.Jitter) + require.False(t, cfg.Connection.TLS.TLSEnabled) + require.True(t, *cfg.Connection.TLS.InsecureSkipVerify) + require.Equal(t, "TestServer", *cfg.Connection.TLS.ServerName) + require.Equal(t, "/tmp/ca", *cfg.Connection.TLS.CAFile) + require.Equal(t, "/tmp/cert", *cfg.Connection.TLS.CertFile) + require.Equal(t, "/tmp/key", *cfg.Connection.TLS.KeyFile) require.Nil(t, cfg.Connection.WriteRetries.Forever) } @@ -171,4 +184,10 @@ func TestNewClientOptions(t *testing.T) { require.Equal(t, 2, opts.ConnectionOptions().WriteRetryOptions().MaxRetries()) require.Equal(t, true, opts.ConnectionOptions().WriteRetryOptions().Jitter()) require.Equal(t, false, opts.ConnectionOptions().WriteRetryOptions().Forever()) + require.False(t, opts.ConnectionOptions().TLSOptions().TLSEnabled()) + require.True(t, opts.ConnectionOptions().TLSOptions().InsecureSkipVerify()) + require.Equal(t, "TestServer", opts.ConnectionOptions().TLSOptions().ServerName()) + require.Equal(t, "/tmp/ca", opts.ConnectionOptions().TLSOptions().CAFile()) + require.Equal(t, "/tmp/cert", opts.ConnectionOptions().TLSOptions().CertFile()) + require.Equal(t, "/tmp/key", opts.ConnectionOptions().TLSOptions().KeyFile()) } diff --git a/src/aggregator/client/conn.go b/src/aggregator/client/conn.go index 705ff1d48f..a0f984f5ab 100644 --- a/src/aggregator/client/conn.go +++ b/src/aggregator/client/conn.go @@ -22,9 +22,13 @@ package client import ( "context" + "crypto/tls" + "crypto/x509" "errors" + "fmt" "math/rand" "net" + "os" "sync" "time" @@ -77,6 +81,7 @@ type connection struct { mtx sync.Mutex keepAlive bool dialer xnet.ContextDialerFn + tls TLSOptions } // newConnection creates a new connection. @@ -101,6 +106,7 @@ func newConnection(addr string, opts ConnectionOptions) *connection { xio.ResettableWriterOptions{WriteBufferSize: 0}, ), metrics: newConnectionMetrics(opts.InstrumentOptions().MetricsScope()), + tls: opts.TLSOptions(), } c.connectWithLockFn = c.connectWithLock c.writeWithLockFn = c.writeWithLock @@ -152,6 +158,34 @@ func (c *connection) Close() { c.mtx.Unlock() } +func (c *connection) upgradeToTLS(conn net.Conn) (net.Conn, error) { + certPool := x509.NewCertPool() + if c.tls.CAFile() != "" { + certs, err := os.ReadFile(c.tls.CAFile()) + if err != nil { + return conn, fmt.Errorf("read bundle error: %w", err) + } + if ok := certPool.AppendCertsFromPEM(certs); !ok { + return conn, fmt.Errorf("cannot append cert to cert pool") + } + } + tlsConfig := &tls.Config{ + RootCAs: certPool, + InsecureSkipVerify: c.tls.InsecureSkipVerify(), + ServerName: c.tls.ServerName(), + } + if c.tls.CertFile() != "" && c.tls.KeyFile() != "" { + tlsConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(c.tls.CertFile(), c.tls.KeyFile()) + if err != nil { + return nil, fmt.Errorf("load x509 key pair error: %w", err) + } + return &cert, nil + } + } + return tls.Client(conn, tlsConfig), nil +} + // writeAttemptWithLock attempts to establish a new connection and writes raw bytes // to the connection while holding the write lock. // If the write succeeds, c.conn is guaranteed to be a valid connection on return. @@ -192,6 +226,15 @@ func (c *connection) connectWithLock() error { } } + if c.tls.TLSEnabled() { + conn, err = c.upgradeToTLS(conn) + if err != nil { + c.metrics.connectError.Inc(1) + conn.Close() + return err + } + } + if c.conn != nil { c.conn.Close() // nolint: errcheck } diff --git a/src/aggregator/client/conn_options.go b/src/aggregator/client/conn_options.go index 98e7073f00..5c3c823826 100644 --- a/src/aggregator/client/conn_options.go +++ b/src/aggregator/client/conn_options.go @@ -113,6 +113,12 @@ type ConnectionOptions interface { // RWOptions returns the RW options. RWOptions() xio.Options + // SetTLSOptions sets TLS options + SetTLSOptions(value TLSOptions) ConnectionOptions + + // TLSOptions returns the TLS options + TLSOptions() TLSOptions + // ContextDialer allows customizing the way an aggregator client the aggregator, at the TCP layer. // By default, this is: // (&net.ContextDialer{}).DialContext. This can be used to do a variety of things, such as forwarding a connection @@ -137,6 +143,7 @@ type connectionOptions struct { maxThreshold int multiplier int connKeepAlive bool + tlsOptions TLSOptions dialer xnet.ContextDialerFn } @@ -159,6 +166,7 @@ func NewConnectionOptions() ConnectionOptions { multiplier: defaultReconnectThresholdMultiplier, maxDuration: defaultMaxReconnectDuration, writeRetryOpts: defaultWriteRetryOpts, + tlsOptions: NewTLSOptions(), rwOpts: xio.NewOptions(), dialer: nil, // Will default to net.Dialer{}.DialContext } @@ -274,6 +282,16 @@ func (o *connectionOptions) RWOptions() xio.Options { return o.rwOpts } +func (o *connectionOptions) SetTLSOptions(value TLSOptions) ConnectionOptions { + opts := *o + opts.tlsOptions = value + return &opts +} + +func (o *connectionOptions) TLSOptions() TLSOptions { + return o.tlsOptions +} + func (o *connectionOptions) ContextDialer() xnet.ContextDialerFn { return o.dialer } diff --git a/src/aggregator/client/conn_test.go b/src/aggregator/client/conn_test.go index 6c1e6b330d..6617a58b0d 100644 --- a/src/aggregator/client/conn_test.go +++ b/src/aggregator/client/conn_test.go @@ -22,10 +22,13 @@ package client import ( "context" + "crypto/tls" + "crypto/x509" "errors" "fmt" "math" "net" + "os" "sync" "testing" "time" @@ -405,6 +408,76 @@ func TestConnectWriteToServer(t *testing.T) { require.Nil(t, conn.conn) } +func TestTLSConnectWriteToServer(t *testing.T) { + data := []byte("foobar") + + // Start tls server. + var wg sync.WaitGroup + wg.Add(1) + + serverCert, err := tls.LoadX509KeyPair("./testdata/server.crt", "./testdata/server.key") + require.NoError(t, err) + certPool := x509.NewCertPool() + certs, err := os.ReadFile("./testdata/rootCA.crt") + require.NoError(t, err) + certPool.AppendCertsFromPEM(certs) + l, err := tls.Listen(tcpProtocol, testLocalServerAddr, &tls.Config{ + Certificates: []tls.Certificate{serverCert}, + ClientCAs: certPool, + ClientAuth: tls.RequireAndVerifyClientCert, + }) + require.NoError(t, err) + serverAddr := l.Addr().String() + + go func() { + defer wg.Done() + + // Ignore the first testing connection. + conn, err := l.Accept() + tlsConn, ok := conn.(*tls.Conn) + require.True(t, ok) + tlsConn.Handshake() + require.NoError(t, err) + require.NoError(t, conn.Close()) + + // Read from the second connection. + conn, err = l.Accept() + require.NoError(t, err) + buf := make([]byte, 1024) + n, err := conn.Read(buf) + require.NoError(t, err) + require.Equal(t, data, buf[:n]) + conn.Close() // nolint: errcheck + }() + + clientCert, err := tls.LoadX509KeyPair("./testdata/client.crt", "./testdata/client.key") + require.NoError(t, err) + // Wait until the server starts up. + dialer := net.Dialer{Timeout: time.Minute} + testConn, err := tls.DialWithDialer(&dialer, tcpProtocol, serverAddr, &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{clientCert}, + RootCAs: certPool, + }) + require.NoError(t, err) + require.NoError(t, testConn.Close()) + + // Create a new connection and assert we can write successfully. + opts := testTLSConnectionOptions().SetInitReconnectThreshold(0) + conn := newConnection(serverAddr, opts) + require.NoError(t, conn.Write(data)) + require.Equal(t, 0, conn.numFailures) + require.NotNil(t, conn.conn) + + // Stop the server. + l.Close() // nolint: errcheck + wg.Wait() + + // Close the connection + conn.Close() + require.Nil(t, conn.conn) +} + func testConnectionOptions() ConnectionOptions { return NewConnectionOptions(). SetClockOptions(clock.NewOptions()). @@ -416,6 +489,16 @@ func testConnectionOptions() ConnectionOptions { SetWriteTimeout(100 * time.Millisecond) } +func testTLSConnectionOptions() ConnectionOptions { + tlsOptions := NewTLSOptions(). + SetTLSEnabled(true). + SetInsecureSkipVerify(true). + SetCAFile("./testdata/rootCA.crt"). + SetCertFile("./testdata/client.crt"). + SetKeyFile("./testdata/client.key") + return testConnectionOptions().SetTLSOptions(tlsOptions) +} + func testConnectionProperties() *gopter.Properties { params := gopter.DefaultTestParameters() params.Rng.Seed(testRandomSeeed) diff --git a/src/aggregator/client/options.go b/src/aggregator/client/options.go index f42dc69411..0d9bef6686 100644 --- a/src/aggregator/client/options.go +++ b/src/aggregator/client/options.go @@ -251,6 +251,7 @@ type options struct { maxBatchSize int flushWorkerCount int aggregatorClientType AggregatorClientType + tlsOptions TLSOptions } // NewOptions creates a new set of client options. diff --git a/src/aggregator/client/testdata/client.crt b/src/aggregator/client/testdata/client.crt new file mode 100644 index 0000000000..366d36e8b8 --- /dev/null +++ b/src/aggregator/client/testdata/client.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPzCCAicCFFJVDW2T6lamcRZYStsBCQTKicKwMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAk5MMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwHhcNMjQw +NDE1MTQ0NjEyWhcNMzQwNDEzMTQ0NjEyWjBeMQswCQYDVQQGEwJOTDETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRcwFQYDVQQDDA5UZXN0Q2xpZW50Q2VydDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAMXgv5tgZD5iBW3LfTC27jPHTLlsAw8ZlBFLmd7potdS3Hy512Xa +kKOaKRKH/0Yla4BKGNh0ZP+xege15CpTFqwtzDPCRkaRvfSxRW19MukeergRDqqI +tEwURHQ5gfub7XB9e5wuNmW2GGs/xgiWsxxm6UMidXkhA5a2//OmDSm2JdcoxJbk +1m1OUzXUuLKLzCIRAbJFFIM1RpvjWSonxpJ2lsNCVQ+LdItxE9mAI+Y+PaDonyzg +ehfAqkl+wxGRevaDFFJr1defxmnJhaYv40AcEpXOWeetZFlspEtrPF7b/3GifRdx +Mlwq27wn3mk4wF1wa12kRNKiz+HHjZrB7UECAwEAATANBgkqhkiG9w0BAQsFAAOC +AQEAKblZ+C4Pdv5grKiCQcfFt53SbIys8uavxmzJNn9nq1QVgBmR/hX8yG8ULeh5 +kKrccwe2WXCBWsIj9cvGCzR+MPMvuIXd3d58yLybmIUBDfJSBC0v5TaNWP2ZdJDG +fH38DfQTeE9XcjpRtCGYtYwReBptSlNiYQlvwkukTNz3mV+0BpvQu/9/R0BNCJAF +9VeLgUcG7T7HC+foCeU7h83Y0xH0OALe33ntcrMPHUclXqImCiC7dl8pb3H7o/xJ +jS3KZxfv6ioxpirvpEbX/EeF+H1HQZUZ8y3+J3K37jMgzNamd/xsaJxEfCq4SE+n +HmgeXgqoNjrM6J0jYAfx1MjbIA== +-----END CERTIFICATE----- diff --git a/src/aggregator/client/testdata/client.key b/src/aggregator/client/testdata/client.key new file mode 100644 index 0000000000..ff226a549e --- /dev/null +++ b/src/aggregator/client/testdata/client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAxeC/m2BkPmIFbct9MLbuM8dMuWwDDxmUEUuZ3umi11LcfLnX +ZdqQo5opEof/RiVrgEoY2HRk/7F6B7XkKlMWrC3MM8JGRpG99LFFbX0y6R56uBEO +qoi0TBREdDmB+5vtcH17nC42ZbYYaz/GCJazHGbpQyJ1eSEDlrb/86YNKbYl1yjE +luTWbU5TNdS4sovMIhEBskUUgzVGm+NZKifGknaWw0JVD4t0i3ET2YAj5j49oOif +LOB6F8CqSX7DEZF69oMUUmvV15/GacmFpi/jQBwSlc5Z561kWWykS2s8Xtv/caJ9 +F3EyXCrbvCfeaTjAXXBrXaRE0qLP4ceNmsHtQQIDAQABAoIBAAtyP8MuJT5SjzvV +rI0317mZCsAjFl42PZFujR0O6MOJ4IU6ftI+fWVpUnzm7wZQvdIy9xL2UK1Vx9hQ +Vj14hvQ4xfosf8IvRgy0gG6f8mT3xWOGYRHOTJemCHuso+85CtgZ+h+DsNPbX7g8 +fSkcBopbDZ07jg4OsdVzCoU+kr5Z1/VDe0O0rTDzGVP686tN9I+DmDYv2HlBUaSZ +SyNZrLJFziqFx4ATTIE3aTLd29pzTl09WiCcZSxhlS+ROnN2iW2xDLNgY9SdUo2a +3r8N+xhYDNQz2paxlsv2x/tCRJvrQDoX7S5QZw4P1pVr6wo6xC+BU2UlpVHifo4M +egyDbAECgYEA5Y9+JDOSQF2cvIV4xTMgVnkLig/ngYesS44xz8HYTRR5s1WW6oCX +3j+OosZBbJpXeBlTK31G9Z92+Om2/Cj2uDvff7EJdhZ5sAdMtUy5X7p9rGTEyi5F +ecZDBfmwHH0lFdwpNtlky1uoaUsVd4IS28WMHoHYDGW241IW8xOlmeECgYEA3Ksb +KR7JCbHx21wh5oPVn5BLpdag7D+M1jBFtjPrzLDNhXlsV9zkjn7xETvTXWU2vNlh +OkoRu4VFEAnsgtxh+gT0lo9ZG9IWi2qP99vPpB3AFYuujBu3GA02/iOo5pri/6R0 +ZNEEreAaaOcT2z6K9HG0KMQ6QpYgXSlzJW8sf2ECgYEAvn2fMA03bIAB4xJi0EkH +qZoScEOYWQ0rdRsOzJbPlc7K2nzImdmRrFRTWVFo0uUUdk2VjX4Mlx/3ir/uHzsi +2GieowhWkI4/9kloZv2+yegoBxkrj5ZsAov57AhxEoLqdkRWUvR8xp9Nleo/awcd +/Q7loh8fF9KDvAjPkHAaOCECgYA98bZNI7wxgYcwGbvWdrmX8iyaIBbKWsiRM7nN +/OM7cYIv7rbwLyzlp1LKkK2zsP7donP9pd82caHCb9a5oV3LjmqOfSz5d08m0cIa +RNUT79oE8lIMOJd8I/GFA8OdAGuqcaLOzjHvEVK4ke1sBTGCjwyQyQzFtljdbg5J +utyV4QKBgHkSBKwrgaump08MJFfK9uFo5sr5jy692XYWMlBVhc+piPwUsoztjI1O +8cvie0mMj6oofSoSyD4FUg9kFUCRMPC0kga/zlHyL1Y/uILX4XUFMl4P/gMbC7kw +0d6BdEsp5SU8Onc9eV/EW+q2Mz7/t5ZHkUrdDQYGgkIBqRZcXlka +-----END RSA PRIVATE KEY----- diff --git a/src/aggregator/client/testdata/rootCA.crt b/src/aggregator/client/testdata/rootCA.crt new file mode 100644 index 0000000000..5d92901e85 --- /dev/null +++ b/src/aggregator/client/testdata/rootCA.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUUwZEZu6XUKW03HPLHGDlt/d/BeUwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCTkwxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDETMBEGA1UEAwwKVGVzdFJvb3RDQTAe +Fw0yNDA0MDQxMTAwMzhaFw0zNDAxMDIxMTAwMzhaMFoxCzAJBgNVBAYTAk5MMRMw +EQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0 +eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDRwn7QLv6zup98kB9VJ1HvHQQMZxt2tqhpFGEc0bXiGrmhFKjV +XZ264ALnUJHQgM8rkRe+dTXg8455FPTMoT+O/BC44pt8CaY1C5kQzVkLaMwaVVxV +vsBfdZ9LOKJQmUXIf1qV/UK3P55Oc3TvwSQW3tU0eMMEOnuNdA4lDdlQnExb7tIJ +4sl0i2w6rB7CoS/0gx2eUPLhPK+vcwCM3d81Odb3ULVznyJ5ZUd5mYY85XXVFQdW +QSyaEE0ekzbmXQzjiqNAt67WEmbadoKdeg8jPG5ka8qBZ21ZGjCHQ8AbzpOn+eDk +n50zwxncpSYVxTilV70PbFSY33PdmdS6XjEnAgMBAAGjUzBRMB0GA1UdDgQWBBSo +kzZddlleTXZYQsBc8efmsZj3WTAfBgNVHSMEGDAWgBSokzZddlleTXZYQsBc8efm +sZj3WTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQACb1ZT3WPK +XdO6jYbZuMN5BBET+mNKQOYrdehY5oiLgr5sDwTwfZvf+q6ic7MpDRBOWMSQuYwp +aUxK7qfw4nSSZb4zDPpzB6Tauh7M5TTPYQwzesn9/JaUBO4vCTwOFGuWxEbXe6PF +VuQOyGQCR9a7juNgoDLiRpWi+xXHz1kMNSeZqV8bn7eo/SIziuSW1JScse91Brt1 +Db9L8iSJpKLDLtwv8fU4/NSORc/672uy4OLxNppsTthh1rk94KbC/FYllC6e8N8n +0iffEn4xE/CECYmAhvplXXS20hCTGbs7tPJ3DGsRfHOyI9RXiayNZuq9GIYKv6eR ++h7NYwXua7oP +-----END CERTIFICATE----- diff --git a/src/aggregator/client/testdata/rootCA.key b/src/aggregator/client/testdata/rootCA.key new file mode 100644 index 0000000000..6d8a527ee2 --- /dev/null +++ b/src/aggregator/client/testdata/rootCA.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDRwn7QLv6zup98 +kB9VJ1HvHQQMZxt2tqhpFGEc0bXiGrmhFKjVXZ264ALnUJHQgM8rkRe+dTXg8455 +FPTMoT+O/BC44pt8CaY1C5kQzVkLaMwaVVxVvsBfdZ9LOKJQmUXIf1qV/UK3P55O +c3TvwSQW3tU0eMMEOnuNdA4lDdlQnExb7tIJ4sl0i2w6rB7CoS/0gx2eUPLhPK+v +cwCM3d81Odb3ULVznyJ5ZUd5mYY85XXVFQdWQSyaEE0ekzbmXQzjiqNAt67WEmba +doKdeg8jPG5ka8qBZ21ZGjCHQ8AbzpOn+eDkn50zwxncpSYVxTilV70PbFSY33Pd +mdS6XjEnAgMBAAECggEAc7MZadGSMibNU9LXRYwDuKXDlufEkU2dRSC1lETkU0w0 +efJAUjhqCSsIzvh1BZIXM8u7UhWq22LcglMZ2ElVGpRU0PZFlzxnTjkQe+EYrZ9n +fVr6LF2kFMN3UvFxLo3snPQKTlM82oEf29v5c0mQk4DkDyZRQ2LSxapvJuclqD0e +iMEG4Yad7p86AmJm7vpe9yp0POhfF5k9ZOAmrov67LMrFPRpWjeRr2ISa2Y/pjYI +vMqL+jGQi72rasNC0JYKyqn+v+LzocZGL4rCQUIpNmKkw81VolwkU8mfe87kneqh +lBsUgmg9nGtNnoFoxxeBNCEQGXf9+BhdGujujycmUQKBgQDo+a+/l/hn3E+7Tqup +nTAuZn80ODpQkfQ/L+jm6G0ENkil62kdyQCUZFfZqpj9dQwSPzCCViXrZymTL35G +1pzZqejPYGANRmiMeQMOOvyUwfIaEz3U9aj7hWl1v4WLUPzEUTTOsG8DBsR/lHBg +ZC081DH8U3m9fYN+Rp6xdT8z+QKBgQDmfXNfrNjFyxnB+kTJovM0k0f6p5HLKjF3 +VNFz0vpNgsa7DHXrmHSBBENi9oj4z72UQo1Q3bTRezR2TLZpsUmkGNEL0PoXTE1U +YVEb/VStLIZeKFv83uD2qvaYw2gA5785SNeinpweNhTC55T0ixojUmVA9eeIT/ex +nBDDJBuWHwKBgDQ12JQIW6vy7I9edwwA5Q5Q/ArY2wC5ZNJQS1KMHfpGrAs68Yiy +RgX7YtCt8bFDbNwd+yIKal8R9Hg+uX7ok6gX8suenof7Em0ApZWn1HkF6dq8GyxB +jYgogtTXgfcRxEO+qyXy1j4IYzrwKir/6D9sknMoxeyYV0KSUvgT/YEJAoGAaCIb +gwlLcqlc/Md+Vn75VDKKXZNhiiGI8bnvW13hWi2Qbaemiwd482UisM5jec4Zf6dF +w1g3PkFkpWHpM/02IR5ZK/aBVw9RDKNfCr88h3TLTDT9wlRL3QXGnaQDFA2f1liz +m7P/IqMaZChOouFJsNWkC2JN9cbzSFoTNKbWk88CgYAS37I31C6xyWlucbztFpmv +YqEPYRpIbRnJUq+F2Kof8dfT44C3rYwRxPHp2LMgyuPxdE491+Gfwsiv587KuBOP +XpCLuWNN38UDndQ5Zcaxl025akSnAF28EitjZCWzw+8TO/52358vpypwSYkdvQNx +or4Q8/AazdVqWUDo3sARPQ== +-----END PRIVATE KEY----- diff --git a/src/aggregator/client/testdata/server.crt b/src/aggregator/client/testdata/server.crt new file mode 100644 index 0000000000..3618a40052 --- /dev/null +++ b/src/aggregator/client/testdata/server.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPzCCAicCFFJVDW2T6lamcRZYStsBCQTKicKvMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAk5MMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwHhcNMjQw +NDA0MTE0ODIwWhcNMzEwNzA3MTE0ODIwWjBeMQswCQYDVQQGEwJOTDETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRcwFQYDVQQDDA5UZXN0U2VydmVyQ2VydDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAOoGyqgrsBCufd9UNt6wr9UJzPaVqdrr/Zble2UPMVZOynAT69W5 +gMdNvrIDU6l1EnRkLyxmP7+beYZvWyU+kN9LyP0pdcgSQdtyhbr1CkqMxhM1MA2B +OFvcEFN2rybaqcLDlq2WUQkmyuVQt7/weLU9jC596PtxUL1e0+wqIItISWpcmemf +cwH9TVlD8Yh6emf87v27afxc0bZCauPsrEPvOC00oU4epXnA7TeDs4RtfqVAuVHX +oEG3oNZ3NHXIIgSht3rNAEfOxBX+7KGvW4hW9ci6W758ECGzHFJThhzO3+w+0YPP +29m+v4CVRaXAGxkVRPiAUDpU8RSPteywTyMCAwEAATANBgkqhkiG9w0BAQsFAAOC +AQEAuQME+WSPw1KKK1jgh1wuiyM5hou1BeagfwOipM3CEblTiHfoERsd2f7Oi6e2 +zPACrvKjVP6bHKjZPEztFHGXDuLdBqK97Vd/o4IFl6p/8EuBeWV85o4Rk1PMhqBI +0jSxsXXhBLgSTkUBgSllwgAbAaxm+L3wBJiQOhh/wEdfNSJ/GeL5477J56Q12/hA +rRCghYRAVVpr44NfG/PbuuMklQHK2l+wkJFBnx+zAAtZF/nb5yCVSy8z/p2TT72a +STfegXYJtE3p4X2+78xajrI7x/PWhNlzJRn3txLW6461vYJIB3ZDVUA2LZtmi3nd +9BUuJVSvsoVwlvrUBF0S1P0vWg== +-----END CERTIFICATE----- diff --git a/src/aggregator/client/testdata/server.key b/src/aggregator/client/testdata/server.key new file mode 100644 index 0000000000..6b6e1cd4cc --- /dev/null +++ b/src/aggregator/client/testdata/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA6gbKqCuwEK5931Q23rCv1QnM9pWp2uv9luV7ZQ8xVk7KcBPr +1bmAx02+sgNTqXUSdGQvLGY/v5t5hm9bJT6Q30vI/Sl1yBJB23KFuvUKSozGEzUw +DYE4W9wQU3avJtqpwsOWrZZRCSbK5VC3v/B4tT2MLn3o+3FQvV7T7Cogi0hJalyZ +6Z9zAf1NWUPxiHp6Z/zu/btp/FzRtkJq4+ysQ+84LTShTh6lecDtN4OzhG1+pUC5 +UdegQbeg1nc0dcgiBKG3es0AR87EFf7soa9biFb1yLpbvnwQIbMcUlOGHM7f7D7R +g8/b2b6/gJVFpcAbGRVE+IBQOlTxFI+17LBPIwIDAQABAoIBAG7rZSYr/rkJWxN8 +wLzvlYctq/27ldPzQjBA+Ck/+CxmD8DrUD3uPEE6cgXBWefZWzPbQBiVJeXU9NZo +9RREATo0m0CSZAa90IUIVKX1+ji3XDJCZVIhSusO9JX6jK2nugpb1ZDUy/fAcS4b +AdxxR2TvlP98Ie77GD+JMHygwugAJPQ4AFZQjO2F/2k3crVt9Frt1VSvf72nkM/C +sDJy4kv+lDZsbeveNPzJo1honiKdCDWpg4duLiLkkG4YvftWHGzpXHoOWnuwnrT5 +jrQTaG10d1jMpnxOwDrxRsFYWtz4kbJm2VtRsgWRIy4A4Vv8VKfFJ0cSu0CChjTE +7yVrExECgYEA9orYMAuuu/c0HGVcXOsA9x6XHtBXr5wK1fI973sGUrtkshTlBgjG +Bc7H+AhVBVdqr2ipm9pFZYSV1OYB2+3AgMQjhoZANR9jr5bmRjJgWuBi5umrFKa4 +n0IQbkwSzjxKvkhpJK25BY2kpjh7/128Q80PMkKnm0LR2gyUWvLFuUUCgYEA8wEJ +PyxHBu93NnbwTP0SUEOYFa654ctiVLbicdcUsbo/Qo4gl/0AIpGmMdeKqx3FWn9I +kCH1xlvb7hp7FNFhDZIiw6CT8V2RQHXdEO9/S0jjOOBg29hxARIgoU+PjjaQG/H+ +SqlDR/ijXUghlllwtP6jCJqy29+Ukpx8WnIkiUcCgYEA4DNXcinNlyWw//m1StJz +KEdrShxRAWO3/uOyKywHjgjYA5I4y4oC+pkXBJofJyJd5HIbe5286hruXpNkMWo+ +W1sCq30+upB3COEDO+bn9PyFNFRevgL3deyQIJ2RpHbzyaLz6uUIkuJCTt6i9JF7 +eXTrUtuGrAzR5ab26tZHtLkCgYBZtW42diAafVjjMHFx2pNiMjH4/zH8G1tHruRV +5nLrUbh8139M3Mep+18BU3MLzMfIquic3Qygl9TPR9NcUhekLOjMk4FKP6KZy2k7 +EU9K+bN4kczwHC1LmeeV5Ys3kwkMZ//ajDYcYbvdef18RbSH/OEvEf0pkpaEsWRu +92hpjQKBgH6Y3dW0bQIhXikjboD00vPn0zPuh35I9OfrPHMqDJiz+xom7Sz9tlGk +NfFll5PTf0gZtU7160uzhCp/OqXT3IdVgVt/G5ey13oYP7pCb8MWiuyNW0yUgtTp +GM5GUnEW5zJ1t2qJOxwoq/FA7D5C7EGRM/QpoMD0tsJoJKYyWkJO +-----END RSA PRIVATE KEY----- diff --git a/src/aggregator/client/tls_options.go b/src/aggregator/client/tls_options.go new file mode 100644 index 0000000000..7c9ab06c97 --- /dev/null +++ b/src/aggregator/client/tls_options.go @@ -0,0 +1,136 @@ +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package client + +type TLSOptions interface { + // SetTLSEnabled sets the TLS enabled option + SetTLSEnabled(value bool) TLSOptions + + // TLSEnabled returns the TLS enabled option + TLSEnabled() bool + + // SetInsecureSkipVerify sets the insecure skip verify option + SetInsecureSkipVerify(value bool) TLSOptions + + // InsecureSkipVerify returns the insecure skip verify option + InsecureSkipVerify() bool + + // SetServerName sets the server name option + SetServerName(value string) TLSOptions + + // ServerName returns the server name option + ServerName() string + + // SetCAFile sets the CA file path + SetCAFile(value string) TLSOptions + + // CAFile returns the CA file path + CAFile() string + + // SetCertFile sets the certificate file path + SetCertFile(value string) TLSOptions + + // CertFile returns the certificate file path + CertFile() string + + // SetKeyFile sets the key file path + SetKeyFile(value string) TLSOptions + + // KeyFile returns the key file path + KeyFile() string +} + +type tlsOptions struct { + tlsEnabled bool + insecureSkipVerify bool + serverName string + caFile string + certFile string + keyFile string +} + +// NewTLSOptions creates new TLS options +func NewTLSOptions() TLSOptions { + return &tlsOptions{ + tlsEnabled: false, + insecureSkipVerify: true, + } +} + +func (o *tlsOptions) SetTLSEnabled(value bool) TLSOptions { + opts := *o + opts.tlsEnabled = value + return &opts +} + +func (o *tlsOptions) TLSEnabled() bool { + return o.tlsEnabled +} + +func (o *tlsOptions) SetInsecureSkipVerify(value bool) TLSOptions { + opts := *o + opts.insecureSkipVerify = value + return &opts +} + +func (o *tlsOptions) InsecureSkipVerify() bool { + return o.insecureSkipVerify +} + +func (o *tlsOptions) SetServerName(value string) TLSOptions { + opts := *o + opts.serverName = value + return &opts +} + +func (o *tlsOptions) ServerName() string { + return o.serverName +} + +func (o *tlsOptions) SetCAFile(value string) TLSOptions { + opts := *o + opts.caFile = value + return &opts +} + +func (o *tlsOptions) CAFile() string { + return o.caFile +} + +func (o *tlsOptions) SetCertFile(value string) TLSOptions { + opts := *o + opts.certFile = value + return &opts +} + +func (o *tlsOptions) CertFile() string { + return o.certFile +} + +func (o *tlsOptions) SetKeyFile(value string) TLSOptions { + opts := *o + opts.keyFile = value + return &opts +} + +func (o *tlsOptions) KeyFile() string { + return o.keyFile +} From a57d5a677ad615d656d5e5223ec7248afb7a58b7 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Thu, 18 Apr 2024 12:22:09 +0000 Subject: [PATCH 03/47] Add TLS configuration to aggregator server config --- src/cmd/services/m3aggregator/config/server.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cmd/services/m3aggregator/config/server.go b/src/cmd/services/m3aggregator/config/server.go index e063c6c549..6f595a8c2d 100644 --- a/src/cmd/services/m3aggregator/config/server.go +++ b/src/cmd/services/m3aggregator/config/server.go @@ -79,6 +79,9 @@ type RawTCPServerConfiguration struct { // Protobuf iterator configuration. ProtobufIterator protobufUnaggregatedIteratorConfiguration `yaml:"protobufIterator"` + + // TLS configuration + TLS *xserver.TLSConfiguration `yaml:"tls"` } // NewServerOptions create a new set of raw TCP server options. @@ -97,6 +100,9 @@ func (c *RawTCPServerConfiguration) NewServerOptions( if c.KeepAlivePeriod != nil { serverOpts = serverOpts.SetTCPConnectionKeepAlivePeriod(*c.KeepAlivePeriod) } + if c.TLS != nil { + serverOpts = serverOpts.SetTLSOptions(c.TLS.NewOptions()) + } opts = opts.SetServerOptions(serverOpts) // Set protobuf iterator options. From 3a206dc5af67e6e96c40f70e4f11ec00bd9bc021 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Thu, 18 Apr 2024 12:26:19 +0000 Subject: [PATCH 04/47] Split xserver options into options and TLS options --- src/x/server/options.go | 93 ----------------------------- src/x/server/tls_options.go | 115 ++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 93 deletions(-) create mode 100644 src/x/server/tls_options.go diff --git a/src/x/server/options.go b/src/x/server/options.go index fa05142745..fb5a9df405 100644 --- a/src/x/server/options.go +++ b/src/x/server/options.go @@ -37,33 +37,6 @@ const ( defaultTCPConnectionKeepAlivePeriod = 10 * time.Second ) -type TLSOptions interface { - // SetMode sets the tls mode - SetMode(value TLSMode) TLSOptions - // Mode returns the tls mode - Mode() TLSMode - - // SetMutualTLSEnabled sets the mutual tls enabled option - SetMutualTLSEnabled(value bool) TLSOptions - // MutualTLSEnabled returns the mutual tls enabled option - MutualTLSEnabled() bool - - // SetCertFile sets the certificate file path - SetCertFile(value string) TLSOptions - // CertFile returns the certificate file path - CertFile() string - - // SetKeyFile sets the private key file path - SetKeyFile(value string) TLSOptions - // KeyFile returns the private key file path - KeyFile() string - - // SetClientCAFile sets the CA file path - SetClientCAFile(value string) TLSOptions - // ClientCAFile returns the CA file path - ClientCAFile() string -} - // Options provide a set of server options type Options interface { // SetInstrumentOptions sets the instrument options @@ -186,69 +159,3 @@ func (o *options) SetTLSOptions(value TLSOptions) Options { func (o *options) TLSOptions() TLSOptions { return o.tlsOptions } - -type tlsOptions struct { - mode TLSMode - mTLSEnabled bool - certFile string - keyFile string - clientCAFile string -} - -// NewTLSOptions creates a new set of tls options -func NewTLSOptions() TLSOptions { - return &tlsOptions{ - mode: TLSDisabled, - mTLSEnabled: false, - } -} - -func (o *tlsOptions) SetMode(value TLSMode) TLSOptions { - opts := *o - opts.mode = value - return &opts -} - -func (o *tlsOptions) Mode() TLSMode { - return o.mode -} - -func (o *tlsOptions) SetMutualTLSEnabled(value bool) TLSOptions { - opts := *o - opts.mTLSEnabled = value - return &opts -} - -func (o *tlsOptions) MutualTLSEnabled() bool { - return o.mTLSEnabled -} - -func (o *tlsOptions) SetCertFile(value string) TLSOptions { - opts := *o - opts.certFile = value - return &opts -} - -func (o *tlsOptions) CertFile() string { - return o.certFile -} - -func (o *tlsOptions) SetKeyFile(value string) TLSOptions { - opts := *o - opts.keyFile = value - return &opts -} - -func (o *tlsOptions) KeyFile() string { - return o.keyFile -} - -func (o *tlsOptions) SetClientCAFile(value string) TLSOptions { - opts := *o - opts.clientCAFile = value - return &opts -} - -func (o *tlsOptions) ClientCAFile() string { - return o.clientCAFile -} diff --git a/src/x/server/tls_options.go b/src/x/server/tls_options.go new file mode 100644 index 0000000000..c8aeb0743c --- /dev/null +++ b/src/x/server/tls_options.go @@ -0,0 +1,115 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package server + +// TLSOptions provide a set of TLS options +type TLSOptions interface { + // SetMode sets the tls mode + SetMode(value TLSMode) TLSOptions + // Mode returns the tls mode + Mode() TLSMode + + // SetMutualTLSEnabled sets the mutual tls enabled option + SetMutualTLSEnabled(value bool) TLSOptions + // MutualTLSEnabled returns the mutual tls enabled option + MutualTLSEnabled() bool + + // SetCertFile sets the certificate file path + SetCertFile(value string) TLSOptions + // CertFile returns the certificate file path + CertFile() string + + // SetKeyFile sets the private key file path + SetKeyFile(value string) TLSOptions + // KeyFile returns the private key file path + KeyFile() string + + // SetClientCAFile sets the CA file path + SetClientCAFile(value string) TLSOptions + // ClientCAFile returns the CA file path + ClientCAFile() string +} + +type tlsOptions struct { + mode TLSMode + mTLSEnabled bool + certFile string + keyFile string + clientCAFile string +} + +// NewTLSOptions creates a new set of tls options +func NewTLSOptions() TLSOptions { + return &tlsOptions{ + mode: TLSDisabled, + mTLSEnabled: false, + } +} + +func (o *tlsOptions) SetMode(value TLSMode) TLSOptions { + opts := *o + opts.mode = value + return &opts +} + +func (o *tlsOptions) Mode() TLSMode { + return o.mode +} + +func (o *tlsOptions) SetMutualTLSEnabled(value bool) TLSOptions { + opts := *o + opts.mTLSEnabled = value + return &opts +} + +func (o *tlsOptions) MutualTLSEnabled() bool { + return o.mTLSEnabled +} + +func (o *tlsOptions) SetCertFile(value string) TLSOptions { + opts := *o + opts.certFile = value + return &opts +} + +func (o *tlsOptions) CertFile() string { + return o.certFile +} + +func (o *tlsOptions) SetKeyFile(value string) TLSOptions { + opts := *o + opts.keyFile = value + return &opts +} + +func (o *tlsOptions) KeyFile() string { + return o.keyFile +} + +func (o *tlsOptions) SetClientCAFile(value string) TLSOptions { + opts := *o + opts.clientCAFile = value + return &opts +} + +func (o *tlsOptions) ClientCAFile() string { + return o.clientCAFile +} From f32c307e12f1534bf815484231850cb0752c9d97 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 23 Apr 2024 15:55:34 +0000 Subject: [PATCH 05/47] Description added to the IsTLS function --- src/x/server/buffered_conn.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/x/server/buffered_conn.go b/src/x/server/buffered_conn.go index 0aa84edc73..3a9ac469db 100644 --- a/src/x/server/buffered_conn.go +++ b/src/x/server/buffered_conn.go @@ -49,7 +49,10 @@ type bufferedConn struct { r *bufio.Reader } -// IsTLS returns is the connection is TLS or not +// IsTLS returns is the connection is TLS or not. +// It peeks at the first byte and checks +// if it is equal to the TLS handshake first byte +// https://www.rfc-editor.org/rfc/rfc5246#appendix-A.1 func (b *bufferedConn) IsTLS() (bool, error) { connBytes, err := b.Peek(1) if err != nil { From 92e092004afb34041f14b55d1ca6112790bf49ab Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 23 Apr 2024 15:56:36 +0000 Subject: [PATCH 06/47] Pointers replaced with values in client TLS configuration --- src/aggregator/client/config.go | 36 ++++++++++------------------ src/aggregator/client/config_test.go | 10 ++++---- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/src/aggregator/client/config.go b/src/aggregator/client/config.go index 1458d288b5..a8cd98958b 100644 --- a/src/aggregator/client/config.go +++ b/src/aggregator/client/config.go @@ -209,33 +209,23 @@ func (c *Configuration) NewClientOptions( // TLSConfiguration contains the TLS configuration type TLSConfiguration struct { - TLSEnabled bool `yaml:"tlsEnabled"` - InsecureSkipVerify *bool `yaml:"insecureSkipVerify"` - ServerName *string `yaml:"serverName"` - CAFile *string `yaml:"caFile"` - CertFile *string `yaml:"certFile"` - KeyFile *string `yaml:"keyFile"` + TLSEnabled bool `yaml:"tlsEnabled"` + InsecureSkipVerify bool `yaml:"insecureSkipVerify"` + ServerName string `yaml:"serverName"` + CAFile string `yaml:"caFile"` + CertFile string `yaml:"certFile"` + KeyFile string `yaml:"keyFile"` } // NewTLSOptions creates new TLS options func (c *TLSConfiguration) NewTLSOptions() TLSOptions { - opts := NewTLSOptions().SetTLSEnabled(c.TLSEnabled) - if c.InsecureSkipVerify != nil { - opts = opts.SetInsecureSkipVerify(*c.InsecureSkipVerify) - } - if c.ServerName != nil { - opts = opts.SetServerName(*c.ServerName) - } - if c.CAFile != nil { - opts = opts.SetCAFile(*c.CAFile) - } - if c.CertFile != nil { - opts = opts.SetCertFile(*c.CertFile) - } - if c.KeyFile != nil { - opts = opts.SetKeyFile(*c.KeyFile) - } - return opts + return NewTLSOptions(). + SetTLSEnabled(c.TLSEnabled). + SetInsecureSkipVerify(c.InsecureSkipVerify). + SetServerName(c.ServerName). + SetCAFile(c.CAFile). + SetCertFile(c.CertFile). + SetKeyFile(c.KeyFile) } // ConnectionConfiguration contains the connection configuration. diff --git a/src/aggregator/client/config_test.go b/src/aggregator/client/config_test.go index dccf19a31c..c712194c04 100644 --- a/src/aggregator/client/config_test.go +++ b/src/aggregator/client/config_test.go @@ -128,11 +128,11 @@ func TestConfigUnmarshal(t *testing.T) { require.Equal(t, 2, cfg.Connection.WriteRetries.MaxRetries) require.Equal(t, true, *cfg.Connection.WriteRetries.Jitter) require.False(t, cfg.Connection.TLS.TLSEnabled) - require.True(t, *cfg.Connection.TLS.InsecureSkipVerify) - require.Equal(t, "TestServer", *cfg.Connection.TLS.ServerName) - require.Equal(t, "/tmp/ca", *cfg.Connection.TLS.CAFile) - require.Equal(t, "/tmp/cert", *cfg.Connection.TLS.CertFile) - require.Equal(t, "/tmp/key", *cfg.Connection.TLS.KeyFile) + require.True(t, cfg.Connection.TLS.InsecureSkipVerify) + require.Equal(t, "TestServer", cfg.Connection.TLS.ServerName) + require.Equal(t, "/tmp/ca", cfg.Connection.TLS.CAFile) + require.Equal(t, "/tmp/cert", cfg.Connection.TLS.CertFile) + require.Equal(t, "/tmp/key", cfg.Connection.TLS.KeyFile) require.Nil(t, cfg.Connection.WriteRetries.Forever) } From 4c798f8983f8a1d254eddf9c2b60f16f0edb60a5 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 23 Apr 2024 16:02:53 +0000 Subject: [PATCH 07/47] Close a connection before returning an error if an upgrade to TLS failed --- src/x/server/server.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/x/server/server.go b/src/x/server/server.go index 950979b357..76fc6cf3b5 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -151,10 +151,12 @@ func (s *server) upgradeToTLS(conn BufferedConn) (BufferedConn, error) { if s.tlsOpts.ClientCAFile() != "" { certs, err := os.ReadFile(s.tlsOpts.ClientCAFile()) if err != nil { - return conn, fmt.Errorf("read bundle error: %w", err) + conn.Close() + return nil, fmt.Errorf("read bundle error: %w", err) } if ok := certPool.AppendCertsFromPEM(certs); !ok { - return conn, fmt.Errorf("cannot append cert to cert pool") + conn.Close() + return nil, fmt.Errorf("cannot append cert to cert pool") } } clientAuthType := tls.NoClientCert @@ -181,12 +183,13 @@ func (s *server) maybeUpgradeToTLS(conn BufferedConn) (BufferedConn, error) { case TLSPermissive: isTLSConnection, err := conn.IsTLS() if err != nil { - return conn, err + conn.Close() + return nil, err } if isTLSConnection { conn, err = s.upgradeToTLS(conn) if err != nil { - return conn, err + return nil, err } } case TLSEnforced: @@ -194,14 +197,16 @@ func (s *server) maybeUpgradeToTLS(conn BufferedConn) (BufferedConn, error) { var isTLSConnection bool isTLSConnection, err = conn.IsTLS() if err != nil { - return conn, err + conn.Close() + return nil, err } if !isTLSConnection { - return conn, fmt.Errorf("Not a tls connection") + conn.Close() + return nil, fmt.Errorf("not a tls connection") } conn, err = s.upgradeToTLS(conn) if err != nil { - return conn, err + return nil, err } } return conn, nil @@ -219,7 +224,6 @@ func (s *server) serve() { } conn, err := s.maybeUpgradeToTLS(conn) if err != nil { - conn.Close() continue } if !s.addConnectionFn(conn) { From 8c4513aeee6af9f44c1bc31752bf41f363c38489 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Fri, 26 Apr 2024 12:59:28 +0000 Subject: [PATCH 08/47] TLSEnabled renamed to Enabled --- src/aggregator/client/config.go | 4 ++-- src/aggregator/client/config_test.go | 6 +++--- src/aggregator/client/conn.go | 2 +- src/aggregator/client/conn_test.go | 2 +- src/aggregator/client/tls_options.go | 20 ++++++++++---------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/aggregator/client/config.go b/src/aggregator/client/config.go index a8cd98958b..26a0d6c835 100644 --- a/src/aggregator/client/config.go +++ b/src/aggregator/client/config.go @@ -209,7 +209,7 @@ func (c *Configuration) NewClientOptions( // TLSConfiguration contains the TLS configuration type TLSConfiguration struct { - TLSEnabled bool `yaml:"tlsEnabled"` + Enabled bool `yaml:"enabled"` InsecureSkipVerify bool `yaml:"insecureSkipVerify"` ServerName string `yaml:"serverName"` CAFile string `yaml:"caFile"` @@ -220,7 +220,7 @@ type TLSConfiguration struct { // NewTLSOptions creates new TLS options func (c *TLSConfiguration) NewTLSOptions() TLSOptions { return NewTLSOptions(). - SetTLSEnabled(c.TLSEnabled). + SetEnabled(c.Enabled). SetInsecureSkipVerify(c.InsecureSkipVerify). SetServerName(c.ServerName). SetCAFile(c.CAFile). diff --git a/src/aggregator/client/config_test.go b/src/aggregator/client/config_test.go index c712194c04..7425c3925b 100644 --- a/src/aggregator/client/config_test.go +++ b/src/aggregator/client/config_test.go @@ -81,7 +81,7 @@ connection: maxRetries: 2 jitter: true tls: - tlsEnabled: false + enabled: true insecureSkipVerify: true serverName: TestServer caFile: /tmp/ca @@ -127,7 +127,7 @@ func TestConfigUnmarshal(t *testing.T) { require.Equal(t, time.Second, cfg.Connection.WriteRetries.MaxBackoff) require.Equal(t, 2, cfg.Connection.WriteRetries.MaxRetries) require.Equal(t, true, *cfg.Connection.WriteRetries.Jitter) - require.False(t, cfg.Connection.TLS.TLSEnabled) + require.True(t, cfg.Connection.TLS.Enabled) require.True(t, cfg.Connection.TLS.InsecureSkipVerify) require.Equal(t, "TestServer", cfg.Connection.TLS.ServerName) require.Equal(t, "/tmp/ca", cfg.Connection.TLS.CAFile) @@ -184,7 +184,7 @@ func TestNewClientOptions(t *testing.T) { require.Equal(t, 2, opts.ConnectionOptions().WriteRetryOptions().MaxRetries()) require.Equal(t, true, opts.ConnectionOptions().WriteRetryOptions().Jitter()) require.Equal(t, false, opts.ConnectionOptions().WriteRetryOptions().Forever()) - require.False(t, opts.ConnectionOptions().TLSOptions().TLSEnabled()) + require.True(t, opts.ConnectionOptions().TLSOptions().Enabled()) require.True(t, opts.ConnectionOptions().TLSOptions().InsecureSkipVerify()) require.Equal(t, "TestServer", opts.ConnectionOptions().TLSOptions().ServerName()) require.Equal(t, "/tmp/ca", opts.ConnectionOptions().TLSOptions().CAFile()) diff --git a/src/aggregator/client/conn.go b/src/aggregator/client/conn.go index a0f984f5ab..a5024ce0e1 100644 --- a/src/aggregator/client/conn.go +++ b/src/aggregator/client/conn.go @@ -226,7 +226,7 @@ func (c *connection) connectWithLock() error { } } - if c.tls.TLSEnabled() { + if c.tls.Enabled() { conn, err = c.upgradeToTLS(conn) if err != nil { c.metrics.connectError.Inc(1) diff --git a/src/aggregator/client/conn_test.go b/src/aggregator/client/conn_test.go index 6617a58b0d..99bef41724 100644 --- a/src/aggregator/client/conn_test.go +++ b/src/aggregator/client/conn_test.go @@ -491,7 +491,7 @@ func testConnectionOptions() ConnectionOptions { func testTLSConnectionOptions() ConnectionOptions { tlsOptions := NewTLSOptions(). - SetTLSEnabled(true). + SetEnabled(true). SetInsecureSkipVerify(true). SetCAFile("./testdata/rootCA.crt"). SetCertFile("./testdata/client.crt"). diff --git a/src/aggregator/client/tls_options.go b/src/aggregator/client/tls_options.go index 7c9ab06c97..1133e1d0bb 100644 --- a/src/aggregator/client/tls_options.go +++ b/src/aggregator/client/tls_options.go @@ -21,11 +21,11 @@ package client type TLSOptions interface { - // SetTLSEnabled sets the TLS enabled option - SetTLSEnabled(value bool) TLSOptions + // SetEnabled sets the TLS enabled option + SetEnabled(value bool) TLSOptions - // TLSEnabled returns the TLS enabled option - TLSEnabled() bool + // Enabled returns the TLS enabled option + Enabled() bool // SetInsecureSkipVerify sets the insecure skip verify option SetInsecureSkipVerify(value bool) TLSOptions @@ -59,7 +59,7 @@ type TLSOptions interface { } type tlsOptions struct { - tlsEnabled bool + enabled bool insecureSkipVerify bool serverName string caFile string @@ -70,19 +70,19 @@ type tlsOptions struct { // NewTLSOptions creates new TLS options func NewTLSOptions() TLSOptions { return &tlsOptions{ - tlsEnabled: false, + enabled: false, insecureSkipVerify: true, } } -func (o *tlsOptions) SetTLSEnabled(value bool) TLSOptions { +func (o *tlsOptions) SetEnabled(value bool) TLSOptions { opts := *o - opts.tlsEnabled = value + opts.enabled = value return &opts } -func (o *tlsOptions) TLSEnabled() bool { - return o.tlsEnabled +func (o *tlsOptions) Enabled() bool { + return o.enabled } func (o *tlsOptions) SetInsecureSkipVerify(value bool) TLSOptions { From 446d2efeb6b473a15efd3992defb43b295d37880 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Wed, 22 May 2024 13:13:41 +0000 Subject: [PATCH 09/47] Integration tests --- .../aggregator_tls/client.crt | 20 ++ .../aggregator_tls/client.key | 27 ++ .../aggregator_tls/docker-compose.yml | 61 ++++ .../aggregator_tls/m3aggregator.yml | 262 ++++++++++++++++++ .../aggregator_tls/m3coordinator.yml | 78 ++++++ .../aggregator_tls/rootCA.crt | 22 ++ .../aggregator_tls/server.crt | 20 ++ .../aggregator_tls/server.key | 27 ++ .../aggregator_tls/test.sh | 125 +++++++++ scripts/docker-integration-tests/run.sh | 1 + 10 files changed, 643 insertions(+) create mode 100644 scripts/docker-integration-tests/aggregator_tls/client.crt create mode 100644 scripts/docker-integration-tests/aggregator_tls/client.key create mode 100644 scripts/docker-integration-tests/aggregator_tls/docker-compose.yml create mode 100644 scripts/docker-integration-tests/aggregator_tls/m3aggregator.yml create mode 100644 scripts/docker-integration-tests/aggregator_tls/m3coordinator.yml create mode 100644 scripts/docker-integration-tests/aggregator_tls/rootCA.crt create mode 100644 scripts/docker-integration-tests/aggregator_tls/server.crt create mode 100644 scripts/docker-integration-tests/aggregator_tls/server.key create mode 100755 scripts/docker-integration-tests/aggregator_tls/test.sh diff --git a/scripts/docker-integration-tests/aggregator_tls/client.crt b/scripts/docker-integration-tests/aggregator_tls/client.crt new file mode 100644 index 0000000000..366d36e8b8 --- /dev/null +++ b/scripts/docker-integration-tests/aggregator_tls/client.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPzCCAicCFFJVDW2T6lamcRZYStsBCQTKicKwMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAk5MMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwHhcNMjQw +NDE1MTQ0NjEyWhcNMzQwNDEzMTQ0NjEyWjBeMQswCQYDVQQGEwJOTDETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRcwFQYDVQQDDA5UZXN0Q2xpZW50Q2VydDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAMXgv5tgZD5iBW3LfTC27jPHTLlsAw8ZlBFLmd7potdS3Hy512Xa +kKOaKRKH/0Yla4BKGNh0ZP+xege15CpTFqwtzDPCRkaRvfSxRW19MukeergRDqqI +tEwURHQ5gfub7XB9e5wuNmW2GGs/xgiWsxxm6UMidXkhA5a2//OmDSm2JdcoxJbk +1m1OUzXUuLKLzCIRAbJFFIM1RpvjWSonxpJ2lsNCVQ+LdItxE9mAI+Y+PaDonyzg +ehfAqkl+wxGRevaDFFJr1defxmnJhaYv40AcEpXOWeetZFlspEtrPF7b/3GifRdx +Mlwq27wn3mk4wF1wa12kRNKiz+HHjZrB7UECAwEAATANBgkqhkiG9w0BAQsFAAOC +AQEAKblZ+C4Pdv5grKiCQcfFt53SbIys8uavxmzJNn9nq1QVgBmR/hX8yG8ULeh5 +kKrccwe2WXCBWsIj9cvGCzR+MPMvuIXd3d58yLybmIUBDfJSBC0v5TaNWP2ZdJDG +fH38DfQTeE9XcjpRtCGYtYwReBptSlNiYQlvwkukTNz3mV+0BpvQu/9/R0BNCJAF +9VeLgUcG7T7HC+foCeU7h83Y0xH0OALe33ntcrMPHUclXqImCiC7dl8pb3H7o/xJ +jS3KZxfv6ioxpirvpEbX/EeF+H1HQZUZ8y3+J3K37jMgzNamd/xsaJxEfCq4SE+n +HmgeXgqoNjrM6J0jYAfx1MjbIA== +-----END CERTIFICATE----- diff --git a/scripts/docker-integration-tests/aggregator_tls/client.key b/scripts/docker-integration-tests/aggregator_tls/client.key new file mode 100644 index 0000000000..ff226a549e --- /dev/null +++ b/scripts/docker-integration-tests/aggregator_tls/client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAxeC/m2BkPmIFbct9MLbuM8dMuWwDDxmUEUuZ3umi11LcfLnX +ZdqQo5opEof/RiVrgEoY2HRk/7F6B7XkKlMWrC3MM8JGRpG99LFFbX0y6R56uBEO +qoi0TBREdDmB+5vtcH17nC42ZbYYaz/GCJazHGbpQyJ1eSEDlrb/86YNKbYl1yjE +luTWbU5TNdS4sovMIhEBskUUgzVGm+NZKifGknaWw0JVD4t0i3ET2YAj5j49oOif +LOB6F8CqSX7DEZF69oMUUmvV15/GacmFpi/jQBwSlc5Z561kWWykS2s8Xtv/caJ9 +F3EyXCrbvCfeaTjAXXBrXaRE0qLP4ceNmsHtQQIDAQABAoIBAAtyP8MuJT5SjzvV +rI0317mZCsAjFl42PZFujR0O6MOJ4IU6ftI+fWVpUnzm7wZQvdIy9xL2UK1Vx9hQ +Vj14hvQ4xfosf8IvRgy0gG6f8mT3xWOGYRHOTJemCHuso+85CtgZ+h+DsNPbX7g8 +fSkcBopbDZ07jg4OsdVzCoU+kr5Z1/VDe0O0rTDzGVP686tN9I+DmDYv2HlBUaSZ +SyNZrLJFziqFx4ATTIE3aTLd29pzTl09WiCcZSxhlS+ROnN2iW2xDLNgY9SdUo2a +3r8N+xhYDNQz2paxlsv2x/tCRJvrQDoX7S5QZw4P1pVr6wo6xC+BU2UlpVHifo4M +egyDbAECgYEA5Y9+JDOSQF2cvIV4xTMgVnkLig/ngYesS44xz8HYTRR5s1WW6oCX +3j+OosZBbJpXeBlTK31G9Z92+Om2/Cj2uDvff7EJdhZ5sAdMtUy5X7p9rGTEyi5F +ecZDBfmwHH0lFdwpNtlky1uoaUsVd4IS28WMHoHYDGW241IW8xOlmeECgYEA3Ksb +KR7JCbHx21wh5oPVn5BLpdag7D+M1jBFtjPrzLDNhXlsV9zkjn7xETvTXWU2vNlh +OkoRu4VFEAnsgtxh+gT0lo9ZG9IWi2qP99vPpB3AFYuujBu3GA02/iOo5pri/6R0 +ZNEEreAaaOcT2z6K9HG0KMQ6QpYgXSlzJW8sf2ECgYEAvn2fMA03bIAB4xJi0EkH +qZoScEOYWQ0rdRsOzJbPlc7K2nzImdmRrFRTWVFo0uUUdk2VjX4Mlx/3ir/uHzsi +2GieowhWkI4/9kloZv2+yegoBxkrj5ZsAov57AhxEoLqdkRWUvR8xp9Nleo/awcd +/Q7loh8fF9KDvAjPkHAaOCECgYA98bZNI7wxgYcwGbvWdrmX8iyaIBbKWsiRM7nN +/OM7cYIv7rbwLyzlp1LKkK2zsP7donP9pd82caHCb9a5oV3LjmqOfSz5d08m0cIa +RNUT79oE8lIMOJd8I/GFA8OdAGuqcaLOzjHvEVK4ke1sBTGCjwyQyQzFtljdbg5J +utyV4QKBgHkSBKwrgaump08MJFfK9uFo5sr5jy692XYWMlBVhc+piPwUsoztjI1O +8cvie0mMj6oofSoSyD4FUg9kFUCRMPC0kga/zlHyL1Y/uILX4XUFMl4P/gMbC7kw +0d6BdEsp5SU8Onc9eV/EW+q2Mz7/t5ZHkUrdDQYGgkIBqRZcXlka +-----END RSA PRIVATE KEY----- diff --git a/scripts/docker-integration-tests/aggregator_tls/docker-compose.yml b/scripts/docker-integration-tests/aggregator_tls/docker-compose.yml new file mode 100644 index 0000000000..3d7b9b1236 --- /dev/null +++ b/scripts/docker-integration-tests/aggregator_tls/docker-compose.yml @@ -0,0 +1,61 @@ +version: "3.5" +services: + dbnode01: + expose: + - "9000-9004" + - "2379-2380" + - "7201" + ports: + - "0.0.0.0:9000-9004:9000-9004" + - "0.0.0.0:2379-2380:2379-2380" + - "0.0.0.0:7201:7201" + networks: + - backend + image: "m3dbnode_integration:${REVISION}" + m3coordinator01: + expose: + - "7202" + - "7203" + - "7204" + ports: + - "0.0.0.0:7202:7202" + - "0.0.0.0:7203:7203" + - "0.0.0.0:7204:7204" + networks: + - backend + image: "m3coordinator_integration:${REVISION}" + volumes: + - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + - "./client.crt:/tmp/client.crt" + - "./client.key:/tmp/client.key" + - "./rootCA.crt:/tmp/rootCA.crt" + m3aggregator01: + expose: + - "6001" + - "6000" + ports: + - "127.0.0.1:6001:6001" + - "127.0.0.1:6000:6000" + networks: + - backend + environment: + - M3AGGREGATOR_HOST_ID=m3aggregator01 + image: "m3aggregator_integration:${REVISION}" + volumes: + - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + - "./server.crt:/tmp/server.crt" + - "./server.key:/tmp/server.key" + - "./rootCA.crt:/tmp/rootCA.crt" + m3aggregator02: + networks: + - backend + environment: + - M3AGGREGATOR_HOST_ID=m3aggregator02 + image: "m3aggregator_integration:${REVISION}" + volumes: + - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + - "./server.crt:/tmp/server.crt" + - "./server.key:/tmp/server.key" + - "./rootCA.crt:/tmp/rootCA.crt" +networks: + backend: diff --git a/scripts/docker-integration-tests/aggregator_tls/m3aggregator.yml b/scripts/docker-integration-tests/aggregator_tls/m3aggregator.yml new file mode 100644 index 0000000000..d71e1bb251 --- /dev/null +++ b/scripts/docker-integration-tests/aggregator_tls/m3aggregator.yml @@ -0,0 +1,262 @@ +logging: + level: info + +metrics: + scope: + prefix: m3aggregator + prometheus: + onError: none + handlerPath: /metrics + sanitization: prometheus + samplingRate: 1.0 + extended: none + +http: + listenAddress: 0.0.0.0:6001 + readTimeout: 60s + writeTimeout: 60s + +rawtcp: + listenAddress: 0.0.0.0:6000 + keepAliveEnabled: true + keepAlivePeriod: 1m + tls: + mode: enforced + mTLSEnabled: true + certFile: /tmp/server.crt + keyFile: /tmp/server.key + clientCAFile: /tmp/rootCA.crt + retry: + initialBackoff: 5ms + backoffFactor: 2.0 + maxBackoff: 1s + forever: true + jitter: true + readBufferSize: 65536 + protobufIterator: + initBufferSize: 1440 + maxMessageSize: 50000000 # max message size is 50MB + bytesPool: + buckets: + - count: 1024 + capacity: 2048 + - count: 512 + capacity: 4096 + - count: 256 + capacity: 8192 + - count: 128 + capacity: 16384 + - count: 64 + capacity: 32768 + - count: 32 + capacity: 65536 + watermark: + low: 0.001 + high: 0.002 + +kvClient: + etcd: + env: override_test_env + zone: embedded + service: m3aggregator + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - dbnode01:2379 + +runtimeOptions: + kvConfig: + environment: override_test_env + zone: embedded + writeValuesPerMetricLimitPerSecondKey: write-values-per-metric-limit-per-second + writeValuesPerMetricLimitPerSecond: 0 + writeNewMetricLimitClusterPerSecondKey: write-new-metric-limit-cluster-per-second + writeNewMetricLimitClusterPerSecond: 0 + writeNewMetricNoLimitWarmupDuration: 0 + +aggregator: + hostID: + resolver: environment + envVarName: M3AGGREGATOR_HOST_ID + instanceID: + type: host_id + metricPrefix: "" + counterPrefix: "" + timerPrefix: "" + gaugePrefix: "" + aggregationTypes: + counterTransformFnType: empty + timerTransformFnType: suffix + gaugeTransformFnType: empty + aggregationTypesPool: + size: 1024 + quantilesPool: + buckets: + - count: 256 + capacity: 4 + - count: 128 + capacity: 8 + stream: + eps: 0.001 + capacity: 32 + streamPool: + size: 4096 + samplePool: + size: 4096 + floatsPool: + buckets: + - count: 4096 + capacity: 16 + - count: 2048 + capacity: 32 + - count: 1024 + capacity: 64 + client: + placementKV: + namespace: /placement + zone: embedded + environment: override_test_env + placementWatcher: + key: m3aggregator + initWatchTimeout: 15s + hashType: murmur32 + shardCutoffLingerDuration: 1m + encoder: + initBufferSize: 100 + maxMessageSize: 50000000 + bytesPool: + buckets: + - capacity: 16 + count: 10 + - capacity: 32 + count: 20 + watermark: + low: 0.001 + high: 0.01 + maxTimerBatchSize: 140 + queueSize: 1000 + queueDropType: oldest + connection: + connectionTimeout: 1s + connectionKeepAlive: true + writeTimeout: 1s + initReconnectThreshold: 2 + maxReconnectThreshold: 5000 + reconnectThresholdMultiplier: 2 + maxReconnectDuration: 1m + placementManager: + kvConfig: + namespace: /placement + environment: override_test_env + zone: embedded + placementWatcher: + key: m3aggregator + initWatchTimeout: 10s + hashType: murmur32 + bufferDurationBeforeShardCutover: 10m + bufferDurationAfterShardCutoff: 10m + resignTimeout: 1m + flushTimesManager: + kvConfig: + environment: override_test_env + zone: embedded + flushTimesKeyFmt: shardset/%d/flush + flushTimesPersistRetrier: + initialBackoff: 100ms + backoffFactor: 2.0 + maxBackoff: 2s + maxRetries: 3 + electionManager: + election: + leaderTimeout: 10s + resignTimeout: 10s + ttlSeconds: 10 + serviceID: + name: m3aggregator + environment: override_test_env + zone: embedded + electionKeyFmt: shardset/%d/lock + campaignRetrier: + initialBackoff: 100ms + backoffFactor: 2.0 + maxBackoff: 2s + forever: true + jitter: true + changeRetrier: + initialBackoff: 100ms + backoffFactor: 2.0 + maxBackoff: 5s + forever: true + jitter: true + resignRetrier: + initialBackoff: 100ms + backoffFactor: 2.0 + maxBackoff: 5s + forever: true + jitter: true + campaignStateCheckInterval: 1s + shardCutoffCheckOffset: 30s + flushManager: + checkEvery: 1s + jitterEnabled: true + maxJitters: + - flushInterval: 5s + maxJitterPercent: 1.0 + - flushInterval: 10s + maxJitterPercent: 0.5 + - flushInterval: 1m + maxJitterPercent: 0.5 + - flushInterval: 10m + maxJitterPercent: 0.5 + - flushInterval: 1h + maxJitterPercent: 0.25 + numWorkersPerCPU: 0.5 + maxBufferSize: 5m + forcedFlushWindowSize: 10s + flush: + handlers: + - dynamicBackend: + name: m3msg + hashType: murmur32 + producer: + buffer: + maxBufferSize: 1000000000 # max buffer before m3msg start dropping data. + writer: + topicName: aggregated_metrics + topicServiceOverride: + zone: embedded + environment: override_test_env + messageRetry: + initialBackoff: 1m + maxBackoff: 2m + messageQueueNewWritesScanInterval: 1s + ackErrorRetry: + initialBackoff: 2s + maxBackoff: 10s + connection: + dialTimeout: 5s + writeTimeout: 5s + retry: + initialBackoff: 1s + maxBackoff: 10s + flushInterval: 1s + writeBufferSize: 16384 + readBufferSize: 256 + forwarding: + maxSingleDelay: 5s + entryTTL: 6h + entryCheckInterval: 10m + maxTimerBatchSizePerWrite: 140 + defaultStoragePolicies: + - 10s:2d + maxNumCachedSourceSets: 2 + discardNaNAggregatedValues: true + entryPool: + size: 4096 + counterElemPool: + size: 4096 + timerElemPool: + size: 4096 + gaugeElemPool: + size: 4096 diff --git a/scripts/docker-integration-tests/aggregator_tls/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_tls/m3coordinator.yml new file mode 100644 index 0000000000..7df48ab953 --- /dev/null +++ b/scripts/docker-integration-tests/aggregator_tls/m3coordinator.yml @@ -0,0 +1,78 @@ +listenAddress: 0.0.0.0:7202 + +carbon: + ingester: + listenAddress: "0.0.0.0:7204" + rules: + - pattern: .* + aggregation: + type: mean + policies: + - resolution: 10s + retention: 6h + +clusters: + - client: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - dbnode01:2379 + +downsample: + remoteAggregator: + client: + placementKV: + namespace: /placement + environment: override_test_env + placementWatcher: + key: m3aggregator + initWatchTimeout: 10s + hashType: murmur32 + shardCutoffLingerDuration: 1m + forceFlushEvery: 1s + flushWorkerCount: 4 + maxTimerBatchSize: 1120 + queueSize: 100 + queueDropType: oldest + encoder: + initBufferSize: 2048 + maxMessageSize: 10485760 + bytesPool: + buckets: + - capacity: 2048 + count: 4096 + - capacity: 4096 + count: 4096 + watermark: + low: 0.7 + high: 1.0 + connection: + writeTimeout: 250ms + tls: + enabled: true + insecureSkipVerify: true + caFile: /tmp/rootCA.crt + certFile: /tmp/client.crt + keyFile: /tmp/client.key + +ingest: + ingester: + workerPoolSize: 100 + opPool: + size: 10000 + retry: + maxRetries: 3 + jitter: true + logSampleRate: 0.01 + m3msg: + server: + listenAddress: "0.0.0.0:7507" + retry: + maxBackoff: 10s + jitter: true diff --git a/scripts/docker-integration-tests/aggregator_tls/rootCA.crt b/scripts/docker-integration-tests/aggregator_tls/rootCA.crt new file mode 100644 index 0000000000..5d92901e85 --- /dev/null +++ b/scripts/docker-integration-tests/aggregator_tls/rootCA.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlTCCAn2gAwIBAgIUUwZEZu6XUKW03HPLHGDlt/d/BeUwDQYJKoZIhvcNAQEL +BQAwWjELMAkGA1UEBhMCTkwxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDETMBEGA1UEAwwKVGVzdFJvb3RDQTAe +Fw0yNDA0MDQxMTAwMzhaFw0zNDAxMDIxMTAwMzhaMFoxCzAJBgNVBAYTAk5MMRMw +EQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0 +eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDRwn7QLv6zup98kB9VJ1HvHQQMZxt2tqhpFGEc0bXiGrmhFKjV +XZ264ALnUJHQgM8rkRe+dTXg8455FPTMoT+O/BC44pt8CaY1C5kQzVkLaMwaVVxV +vsBfdZ9LOKJQmUXIf1qV/UK3P55Oc3TvwSQW3tU0eMMEOnuNdA4lDdlQnExb7tIJ +4sl0i2w6rB7CoS/0gx2eUPLhPK+vcwCM3d81Odb3ULVznyJ5ZUd5mYY85XXVFQdW +QSyaEE0ekzbmXQzjiqNAt67WEmbadoKdeg8jPG5ka8qBZ21ZGjCHQ8AbzpOn+eDk +n50zwxncpSYVxTilV70PbFSY33PdmdS6XjEnAgMBAAGjUzBRMB0GA1UdDgQWBBSo +kzZddlleTXZYQsBc8efmsZj3WTAfBgNVHSMEGDAWgBSokzZddlleTXZYQsBc8efm +sZj3WTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQACb1ZT3WPK +XdO6jYbZuMN5BBET+mNKQOYrdehY5oiLgr5sDwTwfZvf+q6ic7MpDRBOWMSQuYwp +aUxK7qfw4nSSZb4zDPpzB6Tauh7M5TTPYQwzesn9/JaUBO4vCTwOFGuWxEbXe6PF +VuQOyGQCR9a7juNgoDLiRpWi+xXHz1kMNSeZqV8bn7eo/SIziuSW1JScse91Brt1 +Db9L8iSJpKLDLtwv8fU4/NSORc/672uy4OLxNppsTthh1rk94KbC/FYllC6e8N8n +0iffEn4xE/CECYmAhvplXXS20hCTGbs7tPJ3DGsRfHOyI9RXiayNZuq9GIYKv6eR ++h7NYwXua7oP +-----END CERTIFICATE----- diff --git a/scripts/docker-integration-tests/aggregator_tls/server.crt b/scripts/docker-integration-tests/aggregator_tls/server.crt new file mode 100644 index 0000000000..3618a40052 --- /dev/null +++ b/scripts/docker-integration-tests/aggregator_tls/server.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPzCCAicCFFJVDW2T6lamcRZYStsBCQTKicKvMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAk5MMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwHhcNMjQw +NDA0MTE0ODIwWhcNMzEwNzA3MTE0ODIwWjBeMQswCQYDVQQGEwJOTDETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRcwFQYDVQQDDA5UZXN0U2VydmVyQ2VydDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAOoGyqgrsBCufd9UNt6wr9UJzPaVqdrr/Zble2UPMVZOynAT69W5 +gMdNvrIDU6l1EnRkLyxmP7+beYZvWyU+kN9LyP0pdcgSQdtyhbr1CkqMxhM1MA2B +OFvcEFN2rybaqcLDlq2WUQkmyuVQt7/weLU9jC596PtxUL1e0+wqIItISWpcmemf +cwH9TVlD8Yh6emf87v27afxc0bZCauPsrEPvOC00oU4epXnA7TeDs4RtfqVAuVHX +oEG3oNZ3NHXIIgSht3rNAEfOxBX+7KGvW4hW9ci6W758ECGzHFJThhzO3+w+0YPP +29m+v4CVRaXAGxkVRPiAUDpU8RSPteywTyMCAwEAATANBgkqhkiG9w0BAQsFAAOC +AQEAuQME+WSPw1KKK1jgh1wuiyM5hou1BeagfwOipM3CEblTiHfoERsd2f7Oi6e2 +zPACrvKjVP6bHKjZPEztFHGXDuLdBqK97Vd/o4IFl6p/8EuBeWV85o4Rk1PMhqBI +0jSxsXXhBLgSTkUBgSllwgAbAaxm+L3wBJiQOhh/wEdfNSJ/GeL5477J56Q12/hA +rRCghYRAVVpr44NfG/PbuuMklQHK2l+wkJFBnx+zAAtZF/nb5yCVSy8z/p2TT72a +STfegXYJtE3p4X2+78xajrI7x/PWhNlzJRn3txLW6461vYJIB3ZDVUA2LZtmi3nd +9BUuJVSvsoVwlvrUBF0S1P0vWg== +-----END CERTIFICATE----- diff --git a/scripts/docker-integration-tests/aggregator_tls/server.key b/scripts/docker-integration-tests/aggregator_tls/server.key new file mode 100644 index 0000000000..6b6e1cd4cc --- /dev/null +++ b/scripts/docker-integration-tests/aggregator_tls/server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA6gbKqCuwEK5931Q23rCv1QnM9pWp2uv9luV7ZQ8xVk7KcBPr +1bmAx02+sgNTqXUSdGQvLGY/v5t5hm9bJT6Q30vI/Sl1yBJB23KFuvUKSozGEzUw +DYE4W9wQU3avJtqpwsOWrZZRCSbK5VC3v/B4tT2MLn3o+3FQvV7T7Cogi0hJalyZ +6Z9zAf1NWUPxiHp6Z/zu/btp/FzRtkJq4+ysQ+84LTShTh6lecDtN4OzhG1+pUC5 +UdegQbeg1nc0dcgiBKG3es0AR87EFf7soa9biFb1yLpbvnwQIbMcUlOGHM7f7D7R +g8/b2b6/gJVFpcAbGRVE+IBQOlTxFI+17LBPIwIDAQABAoIBAG7rZSYr/rkJWxN8 +wLzvlYctq/27ldPzQjBA+Ck/+CxmD8DrUD3uPEE6cgXBWefZWzPbQBiVJeXU9NZo +9RREATo0m0CSZAa90IUIVKX1+ji3XDJCZVIhSusO9JX6jK2nugpb1ZDUy/fAcS4b +AdxxR2TvlP98Ie77GD+JMHygwugAJPQ4AFZQjO2F/2k3crVt9Frt1VSvf72nkM/C +sDJy4kv+lDZsbeveNPzJo1honiKdCDWpg4duLiLkkG4YvftWHGzpXHoOWnuwnrT5 +jrQTaG10d1jMpnxOwDrxRsFYWtz4kbJm2VtRsgWRIy4A4Vv8VKfFJ0cSu0CChjTE +7yVrExECgYEA9orYMAuuu/c0HGVcXOsA9x6XHtBXr5wK1fI973sGUrtkshTlBgjG +Bc7H+AhVBVdqr2ipm9pFZYSV1OYB2+3AgMQjhoZANR9jr5bmRjJgWuBi5umrFKa4 +n0IQbkwSzjxKvkhpJK25BY2kpjh7/128Q80PMkKnm0LR2gyUWvLFuUUCgYEA8wEJ +PyxHBu93NnbwTP0SUEOYFa654ctiVLbicdcUsbo/Qo4gl/0AIpGmMdeKqx3FWn9I +kCH1xlvb7hp7FNFhDZIiw6CT8V2RQHXdEO9/S0jjOOBg29hxARIgoU+PjjaQG/H+ +SqlDR/ijXUghlllwtP6jCJqy29+Ukpx8WnIkiUcCgYEA4DNXcinNlyWw//m1StJz +KEdrShxRAWO3/uOyKywHjgjYA5I4y4oC+pkXBJofJyJd5HIbe5286hruXpNkMWo+ +W1sCq30+upB3COEDO+bn9PyFNFRevgL3deyQIJ2RpHbzyaLz6uUIkuJCTt6i9JF7 +eXTrUtuGrAzR5ab26tZHtLkCgYBZtW42diAafVjjMHFx2pNiMjH4/zH8G1tHruRV +5nLrUbh8139M3Mep+18BU3MLzMfIquic3Qygl9TPR9NcUhekLOjMk4FKP6KZy2k7 +EU9K+bN4kczwHC1LmeeV5Ys3kwkMZ//ajDYcYbvdef18RbSH/OEvEf0pkpaEsWRu +92hpjQKBgH6Y3dW0bQIhXikjboD00vPn0zPuh35I9OfrPHMqDJiz+xom7Sz9tlGk +NfFll5PTf0gZtU7160uzhCp/OqXT3IdVgVt/G5ey13oYP7pCb8MWiuyNW0yUgtTp +GM5GUnEW5zJ1t2qJOxwoq/FA7D5C7EGRM/QpoMD0tsJoJKYyWkJO +-----END RSA PRIVATE KEY----- diff --git a/scripts/docker-integration-tests/aggregator_tls/test.sh b/scripts/docker-integration-tests/aggregator_tls/test.sh new file mode 100755 index 0000000000..b18c5fdf37 --- /dev/null +++ b/scripts/docker-integration-tests/aggregator_tls/test.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +set -xe + +source "$M3_PATH"/scripts/docker-integration-tests/common.sh +REVISION=$(git rev-parse HEAD) +COMPOSE_FILE="$M3_PATH"/scripts/docker-integration-tests/aggregator_tls/docker-compose.yml +export REVISION + +echo "Run m3dbnode" +docker-compose -f ${COMPOSE_FILE} up -d dbnode01 + +# Stop containers on exit +METRIC_EMIT_PID="-1" +function defer { + docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes + if [ "$METRIC_EMIT_PID" != "-1" ]; then + echo "Kill metric emit process" + kill $METRIC_EMIT_PID + fi +} +trap defer EXIT + +echo "Setup DB node" +AGG_RESOLUTION=10s AGG_RETENTION=6h setup_single_m3db_node + +echo "Initializing aggregator topology" +curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/services/m3aggregator/placement/init -d '{ + "num_shards": 64, + "replication_factor": 2, + "instances": [ + { + "id": "m3aggregator01", + "isolation_group": "availability-zone-a", + "zone": "embedded", + "weight": 100, + "endpoint": "m3aggregator01:6000", + "hostname": "m3aggregator01", + "port": 6000 + }, + { + "id": "m3aggregator02", + "isolation_group": "availability-zone-b", + "zone": "embedded", + "weight": 100, + "endpoint": "m3aggregator02:6000", + "hostname": "m3aggregator02", + "port": 6000 + } + ] +}' + +echo "Initializing m3msg topic for m3coordinator ingestion from m3aggregators" +curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/topic/init -d '{ + "numberOfShards": 64 +}' + +echo "Initializing m3coordinator topology" +curl -vvvsSf -X POST localhost:7201/api/v1/services/m3coordinator/placement/init -d '{ + "instances": [ + { + "id": "m3coordinator01", + "zone": "embedded", + "endpoint": "m3coordinator01:7507", + "hostname": "m3coordinator01", + "port": 7507 + } + ] +}' +echo "Done initializing m3coordinator topology" + +echo "Validating m3coordinator topology" +[ "$(curl -sSf localhost:7201/api/v1/services/m3coordinator/placement | jq .placement.instances.m3coordinator01.id)" == '"m3coordinator01"' ] +echo "Done validating topology" + +# Do this after placement for m3coordinator is created. +echo "Adding m3coordinator as a consumer to the aggregator topic" +curl -vvvsSf -X POST -H "Cluster-Environment-Name: override_test_env" localhost:7201/api/v1/topic -d '{ + "consumerService": { + "serviceId": { + "name": "m3coordinator", + "environment": "default_env", + "zone": "embedded" + }, + "consumptionType": "SHARED", + "messageTtlNanos": "600000000000" + } +}' # msgs will be discarded after 600000000000ns = 10mins + +echo "Running m3coordinator container" +echo "> port 7202 is coordinator API" +echo "> port 7203 is coordinator metrics" +echo "> port 7204 is coordinator graphite ingest" +echo "> port 7507 is coordinator m3msg ingest from aggregator ingest" +docker-compose -f ${COMPOSE_FILE} up -d m3coordinator01 +COORDINATOR_API="localhost:7202" + +echo "Running m3aggregator containers" +docker-compose -f ${COMPOSE_FILE} up -d m3aggregator01 +docker-compose -f ${COMPOSE_FILE} up -d m3aggregator02 + +echo "Verifying aggregation with remote aggregators" + +function read_carbon { + target=$1 + expected_val=$2 + end=$(date +%s) + start=$(($end-1000)) + RESPONSE=$(curl -sSfg "http://${COORDINATOR_API}/api/v1/graphite/render?target=$target&from=$start&until=$end") + test "$(echo "$RESPONSE" | jq ".[0].datapoints | .[][0] | select(. != null)" | tail -n 1)" = "$expected_val" + return $? +} + +# Send metric values 40 and 44 every second +echo "Sending unaggregated carbon metrics to m3coordinator" +bash -c 'while true; do t=$(date +%s); echo "foo.bar.baz 40 $t" | nc -q 0 0.0.0.0 7204; echo "foo.bar.baz 44 $t" | nc -q 0 0.0.0.0 7204; sleep 1; done' & + +# Track PID to kill on exit +METRIC_EMIT_PID="$!" + +# Read back the averaged averaged metric, we configured graphite +# aggregation policy to average each tile and we are emitting +# values 40 and 44 to get an average of 42 each tile +echo "Read back aggregated averaged metric" +ATTEMPTS=10 TIMEOUT=1 retry_with_backoff read_carbon foo.bar.* 42 diff --git a/scripts/docker-integration-tests/run.sh b/scripts/docker-integration-tests/run.sh index bead15b207..2de5258799 100755 --- a/scripts/docker-integration-tests/run.sh +++ b/scripts/docker-integration-tests/run.sh @@ -8,6 +8,7 @@ TESTS=( scripts/docker-integration-tests/carbon/test.sh scripts/docker-integration-tests/aggregator/test.sh scripts/docker-integration-tests/aggregator_legacy/test.sh + scripts/docker-integration-tests/aggregator_tls/test.sh scripts/docker-integration-tests/query_fanout/test.sh scripts/docker-integration-tests/repair/test.sh scripts/docker-integration-tests/replication/test.sh From d992b6f825d449c1e5043600c266b54cb20992a5 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 28 May 2024 13:09:07 +0000 Subject: [PATCH 10/47] Cert pool moved to the client struct --- src/aggregator/client/conn.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/aggregator/client/conn.go b/src/aggregator/client/conn.go index a5024ce0e1..45eb5322a4 100644 --- a/src/aggregator/client/conn.go +++ b/src/aggregator/client/conn.go @@ -82,6 +82,7 @@ type connection struct { keepAlive bool dialer xnet.ContextDialerFn tls TLSOptions + certPool *x509.CertPool } // newConnection creates a new connection. @@ -105,8 +106,9 @@ func newConnection(addr string, opts ConnectionOptions) *connection { uninitWriter, xio.ResettableWriterOptions{WriteBufferSize: 0}, ), - metrics: newConnectionMetrics(opts.InstrumentOptions().MetricsScope()), - tls: opts.TLSOptions(), + metrics: newConnectionMetrics(opts.InstrumentOptions().MetricsScope()), + tls: opts.TLSOptions(), + certPool: x509.NewCertPool(), } c.connectWithLockFn = c.connectWithLock c.writeWithLockFn = c.writeWithLock @@ -159,18 +161,17 @@ func (c *connection) Close() { } func (c *connection) upgradeToTLS(conn net.Conn) (net.Conn, error) { - certPool := x509.NewCertPool() if c.tls.CAFile() != "" { certs, err := os.ReadFile(c.tls.CAFile()) if err != nil { return conn, fmt.Errorf("read bundle error: %w", err) } - if ok := certPool.AppendCertsFromPEM(certs); !ok { + if ok := c.certPool.AppendCertsFromPEM(certs); !ok { return conn, fmt.Errorf("cannot append cert to cert pool") } } tlsConfig := &tls.Config{ - RootCAs: certPool, + RootCAs: c.certPool, InsecureSkipVerify: c.tls.InsecureSkipVerify(), ServerName: c.tls.ServerName(), } From 74d9e7742d274806bb0782cd67aa2320fa80bc11 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 28 May 2024 13:09:18 +0000 Subject: [PATCH 11/47] Cert pool moved to the server struct --- src/x/server/server.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/x/server/server.go b/src/x/server/server.go index 76fc6cf3b5..2d33dacc10 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -96,6 +96,7 @@ type server struct { handler Handler listenerOpts xnet.ListenerOptions tlsOpts TLSOptions + certPool *x509.CertPool addConnectionFn addConnectionFn removeConnectionFn removeConnectionFn @@ -118,6 +119,7 @@ func NewServer(address string, handler Handler, opts Options) Server { handler: handler, listenerOpts: opts.ListenerOptions(), tlsOpts: opts.TLSOptions(), + certPool: x509.NewCertPool(), } // Set up the connection functions. @@ -147,14 +149,13 @@ func (s *server) Serve(l net.Listener) error { } func (s *server) upgradeToTLS(conn BufferedConn) (BufferedConn, error) { - certPool := x509.NewCertPool() if s.tlsOpts.ClientCAFile() != "" { certs, err := os.ReadFile(s.tlsOpts.ClientCAFile()) if err != nil { conn.Close() return nil, fmt.Errorf("read bundle error: %w", err) } - if ok := certPool.AppendCertsFromPEM(certs); !ok { + if ok := s.certPool.AppendCertsFromPEM(certs); !ok { conn.Close() return nil, fmt.Errorf("cannot append cert to cert pool") } @@ -164,7 +165,7 @@ func (s *server) upgradeToTLS(conn BufferedConn) (BufferedConn, error) { clientAuthType = tls.RequireAndVerifyClientCert } tlsConfig := &tls.Config{ - ClientCAs: certPool, + ClientCAs: s.certPool, GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { cert, err := tls.LoadX509KeyPair(s.tlsOpts.CertFile(), s.tlsOpts.KeyFile()) if err != nil { From 09e549d5c08ed5e8c5366810182525e0b7004834 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 28 May 2024 13:22:25 +0000 Subject: [PATCH 12/47] Prevent server tests from being looped forever --- src/x/server/server_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index fb7e146d3d..f9c6ebd28a 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -155,7 +155,8 @@ func TestTLSPermissiveServerListenAndClose(t *testing.T) { require.NoError(t, err) } - for h.called() < numClients { + numChecks := 0 + for h.called() < numClients && numChecks < 5 { time.Sleep(100 * time.Millisecond) } @@ -199,8 +200,10 @@ func TestTLSEnforcedServerListenAndClose(t *testing.T) { require.NoError(t, err) } - for h.called() < numClients/2 { + numChecks := 0 + for h.called() < numClients/2 && numChecks < 5 { time.Sleep(100 * time.Millisecond) + numChecks++ } require.False(t, h.isClosed()) @@ -241,8 +244,10 @@ func TestMutualTLSServerListenAndClose(t *testing.T) { require.NoError(t, err) } - for h.called() < numClients { + numChecks := 0 + for h.called() < numClients && numChecks < 5 { time.Sleep(100 * time.Millisecond) + numChecks++ } require.False(t, h.isClosed()) From d841bcf4646dac4b97662b973d77e073c2564845 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 28 May 2024 13:37:21 +0000 Subject: [PATCH 13/47] testPlainTCPServer function added --- src/x/server/server_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index f9c6ebd28a..e77f412236 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -39,6 +39,10 @@ const ( testListenAddress = "127.0.0.1:0" ) +func testPlainTCPServer(addr string) (*server, *mockHandler, *int32, *int32) { + return testServer(addr, TLSDisabled, false) +} + // nolint: unparam func testServer(addr string, tlsMode TLSMode, mTLSEnabled bool) (*server, *mockHandler, *int32, *int32) { var ( @@ -74,7 +78,7 @@ func testServer(addr string, tlsMode TLSMode, mTLSEnabled bool) (*server, *mockH } func TestServerListenAndClose(t *testing.T) { - s, h, numAdded, numRemoved := testServer(testListenAddress, TLSDisabled, false) + s, h, numAdded, numRemoved := testPlainTCPServer(testListenAddress) var ( numClients = 9 @@ -112,7 +116,7 @@ func TestServerListenAndClose(t *testing.T) { } func TestServe(t *testing.T) { - s, _, _, _ := testServer(testListenAddress, TLSDisabled, false) + s, _, _, _ := testPlainTCPServer(testListenAddress) l, err := net.Listen("tcp", testListenAddress) require.NoError(t, err) From 702563d93b7d5b0e6e058297082a1f41659936aa Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 28 May 2024 14:25:41 +0000 Subject: [PATCH 14/47] maybeUpgradeToTLS refactored --- src/x/server/server.go | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/src/x/server/server.go b/src/x/server/server.go index 2d33dacc10..0ccb9f5a69 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -180,35 +180,22 @@ func (s *server) upgradeToTLS(conn BufferedConn) (BufferedConn, error) { } func (s *server) maybeUpgradeToTLS(conn BufferedConn) (BufferedConn, error) { - switch s.tlsOpts.Mode() { - case TLSPermissive: - isTLSConnection, err := conn.IsTLS() - if err != nil { - conn.Close() - return nil, err - } - if isTLSConnection { - conn, err = s.upgradeToTLS(conn) - if err != nil { - return nil, err - } - } - case TLSEnforced: - var err error - var isTLSConnection bool - isTLSConnection, err = conn.IsTLS() - if err != nil { - conn.Close() - return nil, err - } - if !isTLSConnection { - conn.Close() - return nil, fmt.Errorf("not a tls connection") - } + if s.tlsOpts.Mode() == TLSDisabled { + return conn, nil + } + isTLSConnection, err := conn.IsTLS() + if err != nil { + conn.Close() + return nil, err + } + if isTLSConnection { conn, err = s.upgradeToTLS(conn) if err != nil { return nil, err } + } else if s.tlsOpts.Mode() == TLSEnforced { + conn.Close() + return nil, fmt.Errorf("not a tls connection") } return conn, nil } From b89c701b33cac47c7f079b72342a954d92f815e2 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Thu, 30 May 2024 12:09:01 +0000 Subject: [PATCH 15/47] SecuredConnection refactored --- .../{buffered_conn.go => secured_conn.go} | 52 +++++++++++++------ ...ered_conn_test.go => secured_conn_test.go} | 8 +-- src/x/server/server.go | 37 +++++++------ src/x/server/server_test.go | 4 +- 4 files changed, 60 insertions(+), 41 deletions(-) rename src/x/server/{buffered_conn.go => secured_conn.go} (63%) rename src/x/server/{buffered_conn_test.go => secured_conn_test.go} (96%) diff --git a/src/x/server/buffered_conn.go b/src/x/server/secured_conn.go similarity index 63% rename from src/x/server/buffered_conn.go rename to src/x/server/secured_conn.go index 3a9ac469db..81e2b4c92b 100644 --- a/src/x/server/buffered_conn.go +++ b/src/x/server/secured_conn.go @@ -23,56 +23,76 @@ package server import ( "bufio" + "crypto/tls" "net" ) // TLSHandshakeFirstByte is the first byte of a tls connection handshake const TLSHandshakeFirstByte = 0x16 -func newBufferedConn(conn net.Conn) BufferedConn { - return &bufferedConn{ - r: bufio.NewReader(conn), - Conn: conn, +func newSecuredConn(conn net.Conn) SecuredConn { + return &securedConn{ + r: bufio.NewReader(conn), + Conn: conn, + isTLS: nil, } } -// BufferedConn represents the buffered connection -type BufferedConn interface { +// SecuredConn represents the secured connection +type SecuredConn interface { net.Conn IsTLS() (bool, error) - Peek(int) ([]byte, error) GetConn() net.Conn + UpgradeToTLS(*tls.Config) SecuredConn } -type bufferedConn struct { +type securedConn struct { net.Conn - r *bufio.Reader + r *bufio.Reader + isTLS *bool } // IsTLS returns is the connection is TLS or not. // It peeks at the first byte and checks // if it is equal to the TLS handshake first byte // https://www.rfc-editor.org/rfc/rfc5246#appendix-A.1 -func (b *bufferedConn) IsTLS() (bool, error) { - connBytes, err := b.Peek(1) +func (b *securedConn) IsTLS() (bool, error) { + if b.isTLS != nil { + return *b.isTLS, nil + } + connBytes, err := b.r.Peek(1) if err != nil { return false, err } isTLS := len(connBytes) > 0 && connBytes[0] == TLSHandshakeFirstByte + b.isTLS = &isTLS return isTLS, nil } -// Peek returns the next n bytes without advancing the reader -func (b *bufferedConn) Peek(n int) ([]byte, error) { - return b.r.Peek(n) +func (b *securedConn) UpgradeToTLS(tlsConfig *tls.Config) SecuredConn { + tlsConn := tls.Server(b, tlsConfig) + t := true + return &securedConn{ + r: bufio.NewReader(tlsConn), + Conn: tlsConn, + isTLS: &t, + } } // Read reads n bytes -func (b *bufferedConn) Read(n []byte) (int, error) { +func (b *securedConn) Read(n []byte) (int, error) { + // Before reading we need to ensure if we know the type of the connection. + // After reading data it will be impossible to determine + // if the connection has the TLS layer or not. + if b.isTLS == nil { + if _, err := b.IsTLS(); err != nil { + return 0, err + } + } return b.r.Read(n) } // GetConn returns net.Conn connection -func (b *bufferedConn) GetConn() net.Conn { +func (b *securedConn) GetConn() net.Conn { return b.Conn } diff --git a/src/x/server/buffered_conn_test.go b/src/x/server/secured_conn_test.go similarity index 96% rename from src/x/server/buffered_conn_test.go rename to src/x/server/secured_conn_test.go index 137a9fd4a7..70d89952be 100644 --- a/src/x/server/buffered_conn_test.go +++ b/src/x/server/secured_conn_test.go @@ -57,10 +57,10 @@ func TestPlainTCPConnection(t *testing.T) { _, err = clientConn.Write([]byte("not a tls connection")) require.NoError(t, err) - var conn BufferedConn + var conn SecuredConn select { case newConn := <-connCh: - conn = newBufferedConn(newConn) + conn = newSecuredConn(newConn) case newErr := <-errCh: err = newErr } @@ -85,10 +85,10 @@ func TestTLSConnection(t *testing.T) { defer tlsConn.Close() go tlsConn.Handshake() - var conn BufferedConn + var conn SecuredConn select { case newConn := <-connCh: - conn = newBufferedConn(newConn) + conn = newSecuredConn(newConn) case newErr := <-errCh: err = newErr } diff --git a/src/x/server/server.go b/src/x/server/server.go index 0ccb9f5a69..700449b87c 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -148,15 +148,13 @@ func (s *server) Serve(l net.Listener) error { return nil } -func (s *server) upgradeToTLS(conn BufferedConn) (BufferedConn, error) { +func (s *server) getTLSConfig() (*tls.Config, error) { if s.tlsOpts.ClientCAFile() != "" { certs, err := os.ReadFile(s.tlsOpts.ClientCAFile()) if err != nil { - conn.Close() return nil, fmt.Errorf("read bundle error: %w", err) } if ok := s.certPool.AppendCertsFromPEM(certs); !ok { - conn.Close() return nil, fmt.Errorf("cannot append cert to cert pool") } } @@ -175,26 +173,27 @@ func (s *server) upgradeToTLS(conn BufferedConn) (BufferedConn, error) { }, ClientAuth: clientAuthType, } - tlsConn := tls.Server(conn, tlsConfig) - return newBufferedConn(tlsConn), nil + return tlsConfig, nil } -func (s *server) maybeUpgradeToTLS(conn BufferedConn) (BufferedConn, error) { +func (s *server) maybeUpgradeToTLS(conn SecuredConn) (SecuredConn, error) { if s.tlsOpts.Mode() == TLSDisabled { return conn, nil } isTLSConnection, err := conn.IsTLS() if err != nil { - conn.Close() return nil, err } if isTLSConnection { - conn, err = s.upgradeToTLS(conn) + tlsConfig, err := s.getTLSConfig() + if err != nil { + return nil, err + } + conn = conn.UpgradeToTLS(tlsConfig) if err != nil { return nil, err } } else if s.tlsOpts.Mode() == TLSEnforced { - conn.Close() return nil, fmt.Errorf("not a tls connection") } return conn, nil @@ -203,27 +202,27 @@ func (s *server) maybeUpgradeToTLS(conn BufferedConn) (BufferedConn, error) { func (s *server) serve() { connCh, errCh := xnet.StartForeverAcceptLoop(s.listener, s.retryOpts) for conn := range connCh { - conn := newBufferedConn(conn) + conn := newSecuredConn(conn) if tcpConn, ok := conn.GetConn().(*net.TCPConn); ok { tcpConn.SetKeepAlive(s.tcpConnectionKeepAlive) if s.tcpConnectionKeepAlivePeriod != 0 { tcpConn.SetKeepAlivePeriod(s.tcpConnectionKeepAlivePeriod) } } - conn, err := s.maybeUpgradeToTLS(conn) - if err != nil { - continue - } if !s.addConnectionFn(conn) { conn.Close() } else { s.wgConns.Add(1) go func() { - s.handler.Handle(conn) - - conn.Close() - s.removeConnectionFn(conn) - s.wgConns.Done() + defer conn.Close() + defer s.removeConnectionFn(conn) + defer s.wgConns.Done() + + securedConn, err := s.maybeUpgradeToTLS(conn) + if err != nil { + return + } + s.handler.Handle(securedConn) }() } } diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index e77f412236..2f156796c3 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -215,8 +215,8 @@ func TestTLSEnforcedServerListenAndClose(t *testing.T) { s.Close() require.True(t, h.isClosed()) - require.Equal(t, int32(numClients)/2, atomic.LoadInt32(numAdded)) - require.Equal(t, int32(numClients)/2, atomic.LoadInt32(numRemoved)) + require.Equal(t, int32(numClients), atomic.LoadInt32(numAdded)) + require.Equal(t, int32(numClients), atomic.LoadInt32(numRemoved)) require.Equal(t, numClients/2, h.called()) require.Equal(t, expectedRes, h.res()) } From 94a5706e70b6093b123aba7c7399506d319d2cc6 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Mon, 24 Jun 2024 12:50:03 +0000 Subject: [PATCH 16/47] TLS config TTL --- src/x/server/config.go | 6 +++- src/x/server/config_test.go | 3 ++ src/x/server/server.go | 62 +++++++++++++++++++++++++------------ src/x/server/tls_options.go | 28 ++++++++++++++--- 4 files changed, 74 insertions(+), 25 deletions(-) diff --git a/src/x/server/config.go b/src/x/server/config.go index 4d1837603e..c491fe76f7 100644 --- a/src/x/server/config.go +++ b/src/x/server/config.go @@ -86,6 +86,9 @@ type TLSConfiguration struct { // ClientCAFile path to a CA file for verifying clients ClientCAFile string `yaml:"clientCAFile"` + + // CertificatesTTL is a time duration certificates are stored in memory + CertificatesTTL time.Duration `yaml:"certificatesTTL"` } // NewOptions creates TLS options @@ -94,7 +97,8 @@ func (c TLSConfiguration) NewOptions() TLSOptions { SetMutualTLSEnabled(c.MutualTLSEnabled). SetCertFile(c.CertFile). SetKeyFile(c.KeyFile). - SetClientCAFile(c.ClientCAFile) + SetClientCAFile(c.ClientCAFile). + SetCertificatesTTL(c.CertificatesTTL) var tlsMode TLSMode if err := tlsMode.UnmarshalText([]byte(c.Mode)); err == nil { opts = opts.SetMode(tlsMode) diff --git a/src/x/server/config_test.go b/src/x/server/config_test.go index 8a525f081d..1c9a046d3d 100644 --- a/src/x/server/config_test.go +++ b/src/x/server/config_test.go @@ -41,6 +41,7 @@ tls: certFile: /tmp/cert keyFile: /tmp/key clientCAFile: /tmp/ca + certificatesTTL: 10m ` var cfg Configuration @@ -54,6 +55,7 @@ tls: require.Equal(t, "/tmp/cert", cfg.TLS.CertFile) require.Equal(t, "/tmp/key", cfg.TLS.KeyFile) require.Equal(t, "/tmp/ca", cfg.TLS.ClientCAFile) + require.Equal(t, 10*time.Minute, cfg.TLS.CertificatesTTL) opts := cfg.NewOptions(instrument.NewOptions()) require.Equal(t, 5*time.Second, opts.TCPConnectionKeepAlivePeriod()) @@ -64,6 +66,7 @@ tls: require.Equal(t, "/tmp/cert", opts.TLSOptions().CertFile()) require.Equal(t, "/tmp/key", opts.TLSOptions().KeyFile()) require.Equal(t, "/tmp/ca", opts.TLSOptions().ClientCAFile()) + require.Equal(t, 10*time.Minute, opts.TLSOptions().CertificatesTTL()) require.NotNil(t, cfg.NewServer(nil, instrument.NewOptions())) } diff --git a/src/x/server/server.go b/src/x/server/server.go index 700449b87c..8f97637e0e 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -87,16 +87,18 @@ type server struct { tcpConnectionKeepAlive bool tcpConnectionKeepAlivePeriod time.Duration - closed bool - closedChan chan struct{} - numConns int32 - conns []net.Conn - wgConns sync.WaitGroup - metrics serverMetrics - handler Handler - listenerOpts xnet.ListenerOptions - tlsOpts TLSOptions - certPool *x509.CertPool + closed bool + closedChan chan struct{} + numConns int32 + conns []net.Conn + wgConns sync.WaitGroup + metrics serverMetrics + handler Handler + listenerOpts xnet.ListenerOptions + tlsOpts TLSOptions + certPool *x509.CertPool + tlsConfigCache *tls.Config + certsTTLTicker *time.Ticker addConnectionFn addConnectionFn removeConnectionFn removeConnectionFn @@ -107,6 +109,11 @@ func NewServer(address string, handler Handler, opts Options) Server { instrumentOpts := opts.InstrumentOptions() scope := instrumentOpts.MetricsScope() + var certsTTLTicker *time.Ticker + if opts.TLSOptions().CertificatesTTL() > 0 { + certsTTLTicker = time.NewTicker(opts.TLSOptions().CertificatesTTL()) + } + s := &server{ address: address, log: instrumentOpts.Logger(), @@ -120,6 +127,7 @@ func NewServer(address string, handler Handler, opts Options) Server { listenerOpts: opts.ListenerOptions(), tlsOpts: opts.TLSOptions(), certPool: x509.NewCertPool(), + certsTTLTicker: certsTTLTicker, } // Set up the connection functions. @@ -148,7 +156,24 @@ func (s *server) Serve(l net.Listener) error { return nil } +func (s *server) isTLSConfigCacheValid() bool { + if s.tlsConfigCache == nil || s.certsTTLTicker == nil { + return false + } + select { + case <-s.certsTTLTicker.C: + return false + default: + return true + } +} + func (s *server) getTLSConfig() (*tls.Config, error) { + s.Lock() + defer s.Unlock() + if s.isTLSConfigCacheValid() { + return s.tlsConfigCache, nil + } if s.tlsOpts.ClientCAFile() != "" { certs, err := os.ReadFile(s.tlsOpts.ClientCAFile()) if err != nil { @@ -162,17 +187,16 @@ func (s *server) getTLSConfig() (*tls.Config, error) { if s.tlsOpts.MutualTLSEnabled() { clientAuthType = tls.RequireAndVerifyClientCert } + cert, err := tls.LoadX509KeyPair(s.tlsOpts.CertFile(), s.tlsOpts.KeyFile()) + if err != nil { + return nil, fmt.Errorf("load x509 key pair error: %w", err) + } tlsConfig := &tls.Config{ - ClientCAs: s.certPool, - GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { - cert, err := tls.LoadX509KeyPair(s.tlsOpts.CertFile(), s.tlsOpts.KeyFile()) - if err != nil { - return nil, fmt.Errorf("load x509 key pair error: %w", err) - } - return &cert, nil - }, - ClientAuth: clientAuthType, + ClientCAs: s.certPool, + Certificates: []tls.Certificate{cert}, + ClientAuth: clientAuthType, } + s.tlsConfigCache = tlsConfig return tlsConfig, nil } diff --git a/src/x/server/tls_options.go b/src/x/server/tls_options.go index c8aeb0743c..c295769fc0 100644 --- a/src/x/server/tls_options.go +++ b/src/x/server/tls_options.go @@ -20,6 +20,8 @@ package server +import "time" + // TLSOptions provide a set of TLS options type TLSOptions interface { // SetMode sets the tls mode @@ -46,14 +48,20 @@ type TLSOptions interface { SetClientCAFile(value string) TLSOptions // ClientCAFile returns the CA file path ClientCAFile() string + + // SetCertificatesTTL sets the certificates TTL + SetCertificatesTTL(value time.Duration) TLSOptions + // CertificatesTTL returns the certificates TTL + CertificatesTTL() time.Duration } type tlsOptions struct { - mode TLSMode - mTLSEnabled bool - certFile string - keyFile string - clientCAFile string + mode TLSMode + mTLSEnabled bool + certFile string + keyFile string + clientCAFile string + certificatesTTL time.Duration } // NewTLSOptions creates a new set of tls options @@ -113,3 +121,13 @@ func (o *tlsOptions) SetClientCAFile(value string) TLSOptions { func (o *tlsOptions) ClientCAFile() string { return o.clientCAFile } + +func (o *tlsOptions) SetCertificatesTTL(value time.Duration) TLSOptions { + opts := *o + opts.certificatesTTL = value + return &opts +} + +func (o *tlsOptions) CertificatesTTL() time.Duration { + return o.certificatesTTL +} From 414711b16d4a06eb61ba8ff32f1fd79eeaea5bff Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 25 Jun 2024 11:34:22 +0000 Subject: [PATCH 17/47] Client TLS config cache --- src/aggregator/client/config.go | 13 ++++--- src/aggregator/client/conn.go | 57 +++++++++++++++++++++------- src/aggregator/client/conn_test.go | 3 +- src/aggregator/client/tls_options.go | 19 ++++++++++ 4 files changed, 72 insertions(+), 20 deletions(-) diff --git a/src/aggregator/client/config.go b/src/aggregator/client/config.go index 26a0d6c835..86661b6609 100644 --- a/src/aggregator/client/config.go +++ b/src/aggregator/client/config.go @@ -209,12 +209,13 @@ func (c *Configuration) NewClientOptions( // TLSConfiguration contains the TLS configuration type TLSConfiguration struct { - Enabled bool `yaml:"enabled"` - InsecureSkipVerify bool `yaml:"insecureSkipVerify"` - ServerName string `yaml:"serverName"` - CAFile string `yaml:"caFile"` - CertFile string `yaml:"certFile"` - KeyFile string `yaml:"keyFile"` + Enabled bool `yaml:"enabled"` + InsecureSkipVerify bool `yaml:"insecureSkipVerify"` + ServerName string `yaml:"serverName"` + CAFile string `yaml:"caFile"` + CertFile string `yaml:"certFile"` + KeyFile string `yaml:"keyFile"` + CertificatesTTL time.Duration `yaml:"certificatesTTL"` } // NewTLSOptions creates new TLS options diff --git a/src/aggregator/client/conn.go b/src/aggregator/client/conn.go index 45eb5322a4..74f69a2d94 100644 --- a/src/aggregator/client/conn.go +++ b/src/aggregator/client/conn.go @@ -83,10 +83,17 @@ type connection struct { dialer xnet.ContextDialerFn tls TLSOptions certPool *x509.CertPool + tlsConfigCache *tls.Config + certsTTLTicker *time.Ticker } // newConnection creates a new connection. func newConnection(addr string, opts ConnectionOptions) *connection { + var certsTTLTicker *time.Ticker + if opts.TLSOptions().CertificatesTTL() > 0 { + certsTTLTicker = time.NewTicker(opts.TLSOptions().CertificatesTTL()) + } + c := &connection{ addr: addr, connTimeout: opts.ConnectionTimeout(), @@ -106,9 +113,10 @@ func newConnection(addr string, opts ConnectionOptions) *connection { uninitWriter, xio.ResettableWriterOptions{WriteBufferSize: 0}, ), - metrics: newConnectionMetrics(opts.InstrumentOptions().MetricsScope()), - tls: opts.TLSOptions(), - certPool: x509.NewCertPool(), + metrics: newConnectionMetrics(opts.InstrumentOptions().MetricsScope()), + tls: opts.TLSOptions(), + certPool: x509.NewCertPool(), + certsTTLTicker: certsTTLTicker, } c.connectWithLockFn = c.connectWithLock c.writeWithLockFn = c.writeWithLock @@ -160,14 +168,29 @@ func (c *connection) Close() { c.mtx.Unlock() } -func (c *connection) upgradeToTLS(conn net.Conn) (net.Conn, error) { +func (c *connection) isTLSConfigCacheValid() bool { + if c.tlsConfigCache == nil || c.certsTTLTicker == nil { + return false + } + select { + case <-c.certsTTLTicker.C: + return false + default: + return true + } +} + +func (c *connection) getTLSConfig() (*tls.Config, error) { + if c.isTLSConfigCacheValid() { + return c.tlsConfigCache, nil + } if c.tls.CAFile() != "" { certs, err := os.ReadFile(c.tls.CAFile()) if err != nil { - return conn, fmt.Errorf("read bundle error: %w", err) + return nil, fmt.Errorf("read bundle error: %w", err) } if ok := c.certPool.AppendCertsFromPEM(certs); !ok { - return conn, fmt.Errorf("cannot append cert to cert pool") + return nil, fmt.Errorf("cannot append cert to cert pool") } } tlsConfig := &tls.Config{ @@ -176,13 +199,20 @@ func (c *connection) upgradeToTLS(conn net.Conn) (net.Conn, error) { ServerName: c.tls.ServerName(), } if c.tls.CertFile() != "" && c.tls.KeyFile() != "" { - tlsConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { - cert, err := tls.LoadX509KeyPair(c.tls.CertFile(), c.tls.KeyFile()) - if err != nil { - return nil, fmt.Errorf("load x509 key pair error: %w", err) - } - return &cert, nil + cert, err := tls.LoadX509KeyPair(c.tls.CertFile(), c.tls.KeyFile()) + if err != nil { + return nil, fmt.Errorf("load x509 key pair error: %w", err) } + tlsConfig.Certificates = []tls.Certificate{cert} + } + c.tlsConfigCache = tlsConfig + return tlsConfig, nil +} + +func (c *connection) upgradeToTLS(conn net.Conn) (net.Conn, error) { + tlsConfig, err := c.getTLSConfig() + if err != nil { + return nil, err } return tls.Client(conn, tlsConfig), nil } @@ -228,12 +258,13 @@ func (c *connection) connectWithLock() error { } if c.tls.Enabled() { - conn, err = c.upgradeToTLS(conn) + securedConn, err := c.upgradeToTLS(conn) if err != nil { c.metrics.connectError.Inc(1) conn.Close() return err } + conn = securedConn } if c.conn != nil { diff --git a/src/aggregator/client/conn_test.go b/src/aggregator/client/conn_test.go index 99bef41724..c45e778abd 100644 --- a/src/aggregator/client/conn_test.go +++ b/src/aggregator/client/conn_test.go @@ -495,7 +495,8 @@ func testTLSConnectionOptions() ConnectionOptions { SetInsecureSkipVerify(true). SetCAFile("./testdata/rootCA.crt"). SetCertFile("./testdata/client.crt"). - SetKeyFile("./testdata/client.key") + SetKeyFile("./testdata/client.key"). + SetCertificatesTTL(time.Second) return testConnectionOptions().SetTLSOptions(tlsOptions) } diff --git a/src/aggregator/client/tls_options.go b/src/aggregator/client/tls_options.go index 1133e1d0bb..b29528345c 100644 --- a/src/aggregator/client/tls_options.go +++ b/src/aggregator/client/tls_options.go @@ -20,6 +20,8 @@ package client +import "time" + type TLSOptions interface { // SetEnabled sets the TLS enabled option SetEnabled(value bool) TLSOptions @@ -56,6 +58,12 @@ type TLSOptions interface { // KeyFile returns the key file path KeyFile() string + + // SetCertificatesTTL sets the certificates TTL + SetCertificatesTTL(value time.Duration) TLSOptions + + // CertificatesTTL returns the certificates TTL + CertificatesTTL() time.Duration } type tlsOptions struct { @@ -65,6 +73,7 @@ type tlsOptions struct { caFile string certFile string keyFile string + certificatesTTL time.Duration } // NewTLSOptions creates new TLS options @@ -134,3 +143,13 @@ func (o *tlsOptions) SetKeyFile(value string) TLSOptions { func (o *tlsOptions) KeyFile() string { return o.keyFile } + +func (o *tlsOptions) SetCertificatesTTL(value time.Duration) TLSOptions { + opts := *o + opts.certificatesTTL = value + return &opts +} + +func (o *tlsOptions) CertificatesTTL() time.Duration { + return o.certificatesTTL +} From 1ee36def8198ca20551add2cbf91b9d5f377cad4 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 2 Jul 2024 11:46:48 +0000 Subject: [PATCH 18/47] TLS Config manager factored out into a separate package --- src/aggregator/client/config.go | 7 +- src/aggregator/client/config_test.go | 2 +- src/aggregator/client/conn.go | 65 +----- src/aggregator/client/conn_options.go | 13 +- src/aggregator/client/conn_test.go | 5 +- src/aggregator/client/options.go | 3 +- src/x/server/config.go | 11 +- src/x/server/config_test.go | 5 +- src/x/server/options.go | 13 +- src/x/server/server.go | 84 ++----- src/x/server/server_test.go | 17 +- src/x/server/tls_options.go | 133 ----------- src/x/tls/config_manager.go | 162 +++++++++++++ src/x/tls/config_manager_test.go | 219 ++++++++++++++++++ src/x/{server/tls_mode.go => tls/mode.go} | 27 ++- .../tls_mode_test.go => tls/mode_test.go} | 11 +- .../tls_options.go => x/tls/options.go} | 126 ++++++---- src/x/tls/testdata/1.crt | 20 ++ src/x/tls/testdata/1.key | 27 +++ src/x/tls/testdata/2.crt | 20 ++ src/x/tls/testdata/3.crt | 1 + 21 files changed, 606 insertions(+), 365 deletions(-) delete mode 100644 src/x/server/tls_options.go create mode 100644 src/x/tls/config_manager.go create mode 100644 src/x/tls/config_manager_test.go rename src/x/{server/tls_mode.go => tls/mode.go} (74%) rename src/x/{server/tls_mode_test.go => tls/mode_test.go} (87%) rename src/{aggregator/client/tls_options.go => x/tls/options.go} (51%) create mode 100644 src/x/tls/testdata/1.crt create mode 100644 src/x/tls/testdata/1.key create mode 100644 src/x/tls/testdata/2.crt create mode 100644 src/x/tls/testdata/3.crt diff --git a/src/aggregator/client/config.go b/src/aggregator/client/config.go index 86661b6609..55acf915a1 100644 --- a/src/aggregator/client/config.go +++ b/src/aggregator/client/config.go @@ -36,6 +36,7 @@ import ( xio "github.com/m3db/m3/src/x/io" "github.com/m3db/m3/src/x/pool" "github.com/m3db/m3/src/x/retry" + xtls "github.com/m3db/m3/src/x/tls" "github.com/uber-go/tally" ) @@ -219,9 +220,9 @@ type TLSConfiguration struct { } // NewTLSOptions creates new TLS options -func (c *TLSConfiguration) NewTLSOptions() TLSOptions { - return NewTLSOptions(). - SetEnabled(c.Enabled). +func (c *TLSConfiguration) NewTLSOptions() xtls.Options { + return xtls.NewOptions(). + SetClientEnabled(c.Enabled). SetInsecureSkipVerify(c.InsecureSkipVerify). SetServerName(c.ServerName). SetCAFile(c.CAFile). diff --git a/src/aggregator/client/config_test.go b/src/aggregator/client/config_test.go index 7425c3925b..edde9e5055 100644 --- a/src/aggregator/client/config_test.go +++ b/src/aggregator/client/config_test.go @@ -184,7 +184,7 @@ func TestNewClientOptions(t *testing.T) { require.Equal(t, 2, opts.ConnectionOptions().WriteRetryOptions().MaxRetries()) require.Equal(t, true, opts.ConnectionOptions().WriteRetryOptions().Jitter()) require.Equal(t, false, opts.ConnectionOptions().WriteRetryOptions().Forever()) - require.True(t, opts.ConnectionOptions().TLSOptions().Enabled()) + require.True(t, opts.ConnectionOptions().TLSOptions().ClientEnabled()) require.True(t, opts.ConnectionOptions().TLSOptions().InsecureSkipVerify()) require.Equal(t, "TestServer", opts.ConnectionOptions().TLSOptions().ServerName()) require.Equal(t, "/tmp/ca", opts.ConnectionOptions().TLSOptions().CAFile()) diff --git a/src/aggregator/client/conn.go b/src/aggregator/client/conn.go index 74f69a2d94..6873fcab13 100644 --- a/src/aggregator/client/conn.go +++ b/src/aggregator/client/conn.go @@ -23,12 +23,9 @@ package client import ( "context" "crypto/tls" - "crypto/x509" "errors" - "fmt" "math/rand" "net" - "os" "sync" "time" @@ -36,6 +33,7 @@ import ( xio "github.com/m3db/m3/src/x/io" xnet "github.com/m3db/m3/src/x/net" "github.com/m3db/m3/src/x/retry" + xtls "github.com/m3db/m3/src/x/tls" "github.com/uber-go/tally" ) @@ -81,19 +79,11 @@ type connection struct { mtx sync.Mutex keepAlive bool dialer xnet.ContextDialerFn - tls TLSOptions - certPool *x509.CertPool - tlsConfigCache *tls.Config - certsTTLTicker *time.Ticker + tlsConfigManager xtls.ConfigManager } // newConnection creates a new connection. func newConnection(addr string, opts ConnectionOptions) *connection { - var certsTTLTicker *time.Ticker - if opts.TLSOptions().CertificatesTTL() > 0 { - certsTTLTicker = time.NewTicker(opts.TLSOptions().CertificatesTTL()) - } - c := &connection{ addr: addr, connTimeout: opts.ConnectionTimeout(), @@ -113,10 +103,8 @@ func newConnection(addr string, opts ConnectionOptions) *connection { uninitWriter, xio.ResettableWriterOptions{WriteBufferSize: 0}, ), - metrics: newConnectionMetrics(opts.InstrumentOptions().MetricsScope()), - tls: opts.TLSOptions(), - certPool: x509.NewCertPool(), - certsTTLTicker: certsTTLTicker, + metrics: newConnectionMetrics(opts.InstrumentOptions().MetricsScope()), + tlsConfigManager: xtls.NewConfigManager(opts.TLSOptions(), opts.InstrumentOptions()), } c.connectWithLockFn = c.connectWithLock c.writeWithLockFn = c.writeWithLock @@ -168,49 +156,8 @@ func (c *connection) Close() { c.mtx.Unlock() } -func (c *connection) isTLSConfigCacheValid() bool { - if c.tlsConfigCache == nil || c.certsTTLTicker == nil { - return false - } - select { - case <-c.certsTTLTicker.C: - return false - default: - return true - } -} - -func (c *connection) getTLSConfig() (*tls.Config, error) { - if c.isTLSConfigCacheValid() { - return c.tlsConfigCache, nil - } - if c.tls.CAFile() != "" { - certs, err := os.ReadFile(c.tls.CAFile()) - if err != nil { - return nil, fmt.Errorf("read bundle error: %w", err) - } - if ok := c.certPool.AppendCertsFromPEM(certs); !ok { - return nil, fmt.Errorf("cannot append cert to cert pool") - } - } - tlsConfig := &tls.Config{ - RootCAs: c.certPool, - InsecureSkipVerify: c.tls.InsecureSkipVerify(), - ServerName: c.tls.ServerName(), - } - if c.tls.CertFile() != "" && c.tls.KeyFile() != "" { - cert, err := tls.LoadX509KeyPair(c.tls.CertFile(), c.tls.KeyFile()) - if err != nil { - return nil, fmt.Errorf("load x509 key pair error: %w", err) - } - tlsConfig.Certificates = []tls.Certificate{cert} - } - c.tlsConfigCache = tlsConfig - return tlsConfig, nil -} - func (c *connection) upgradeToTLS(conn net.Conn) (net.Conn, error) { - tlsConfig, err := c.getTLSConfig() + tlsConfig, err := c.tlsConfigManager.TLSConfig() if err != nil { return nil, err } @@ -257,7 +204,7 @@ func (c *connection) connectWithLock() error { } } - if c.tls.Enabled() { + if c.tlsConfigManager.ClientEnabled() { securedConn, err := c.upgradeToTLS(conn) if err != nil { c.metrics.connectError.Inc(1) diff --git a/src/aggregator/client/conn_options.go b/src/aggregator/client/conn_options.go index 5c3c823826..db6f1fc807 100644 --- a/src/aggregator/client/conn_options.go +++ b/src/aggregator/client/conn_options.go @@ -28,6 +28,7 @@ import ( xio "github.com/m3db/m3/src/x/io" xnet "github.com/m3db/m3/src/x/net" "github.com/m3db/m3/src/x/retry" + xtls "github.com/m3db/m3/src/x/tls" ) const ( @@ -114,10 +115,10 @@ type ConnectionOptions interface { RWOptions() xio.Options // SetTLSOptions sets TLS options - SetTLSOptions(value TLSOptions) ConnectionOptions + SetTLSOptions(value xtls.Options) ConnectionOptions // TLSOptions returns the TLS options - TLSOptions() TLSOptions + TLSOptions() xtls.Options // ContextDialer allows customizing the way an aggregator client the aggregator, at the TCP layer. // By default, this is: @@ -143,7 +144,7 @@ type connectionOptions struct { maxThreshold int multiplier int connKeepAlive bool - tlsOptions TLSOptions + tlsOptions xtls.Options dialer xnet.ContextDialerFn } @@ -166,7 +167,7 @@ func NewConnectionOptions() ConnectionOptions { multiplier: defaultReconnectThresholdMultiplier, maxDuration: defaultMaxReconnectDuration, writeRetryOpts: defaultWriteRetryOpts, - tlsOptions: NewTLSOptions(), + tlsOptions: xtls.NewOptions(), rwOpts: xio.NewOptions(), dialer: nil, // Will default to net.Dialer{}.DialContext } @@ -282,13 +283,13 @@ func (o *connectionOptions) RWOptions() xio.Options { return o.rwOpts } -func (o *connectionOptions) SetTLSOptions(value TLSOptions) ConnectionOptions { +func (o *connectionOptions) SetTLSOptions(value xtls.Options) ConnectionOptions { opts := *o opts.tlsOptions = value return &opts } -func (o *connectionOptions) TLSOptions() TLSOptions { +func (o *connectionOptions) TLSOptions() xtls.Options { return o.tlsOptions } diff --git a/src/aggregator/client/conn_test.go b/src/aggregator/client/conn_test.go index c45e778abd..a9bfcff2db 100644 --- a/src/aggregator/client/conn_test.go +++ b/src/aggregator/client/conn_test.go @@ -34,6 +34,7 @@ import ( "time" "github.com/m3db/m3/src/x/clock" + xtls "github.com/m3db/m3/src/x/tls" "github.com/golang/mock/gomock" "github.com/leanovate/gopter" @@ -490,8 +491,8 @@ func testConnectionOptions() ConnectionOptions { } func testTLSConnectionOptions() ConnectionOptions { - tlsOptions := NewTLSOptions(). - SetEnabled(true). + tlsOptions := xtls.NewOptions(). + SetClientEnabled(true). SetInsecureSkipVerify(true). SetCAFile("./testdata/rootCA.crt"). SetCertFile("./testdata/client.crt"). diff --git a/src/aggregator/client/options.go b/src/aggregator/client/options.go index 0d9bef6686..5279564a28 100644 --- a/src/aggregator/client/options.go +++ b/src/aggregator/client/options.go @@ -31,6 +31,7 @@ import ( "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/instrument" xio "github.com/m3db/m3/src/x/io" + xtls "github.com/m3db/m3/src/x/tls" ) // AggregatorClientType determines the aggregator client type. @@ -251,7 +252,7 @@ type options struct { maxBatchSize int flushWorkerCount int aggregatorClientType AggregatorClientType - tlsOptions TLSOptions + tlsOptions xtls.Options } // NewOptions creates a new set of client options. diff --git a/src/x/server/config.go b/src/x/server/config.go index c491fe76f7..a5333cd562 100644 --- a/src/x/server/config.go +++ b/src/x/server/config.go @@ -25,6 +25,7 @@ import ( "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/retry" + xtls "github.com/m3db/m3/src/x/tls" ) // Configuration configs a server. @@ -92,16 +93,16 @@ type TLSConfiguration struct { } // NewOptions creates TLS options -func (c TLSConfiguration) NewOptions() TLSOptions { - opts := NewTLSOptions(). +func (c TLSConfiguration) NewOptions() xtls.Options { + opts := xtls.NewOptions(). SetMutualTLSEnabled(c.MutualTLSEnabled). SetCertFile(c.CertFile). SetKeyFile(c.KeyFile). - SetClientCAFile(c.ClientCAFile). + SetCAFile(c.ClientCAFile). SetCertificatesTTL(c.CertificatesTTL) - var tlsMode TLSMode + var tlsMode xtls.ServerMode if err := tlsMode.UnmarshalText([]byte(c.Mode)); err == nil { - opts = opts.SetMode(tlsMode) + opts = opts.SetServerMode(tlsMode) } return opts } diff --git a/src/x/server/config_test.go b/src/x/server/config_test.go index 1c9a046d3d..817ae17a3f 100644 --- a/src/x/server/config_test.go +++ b/src/x/server/config_test.go @@ -25,6 +25,7 @@ import ( "time" "github.com/m3db/m3/src/x/instrument" + xtls "github.com/m3db/m3/src/x/tls" "github.com/stretchr/testify/require" yaml "gopkg.in/yaml.v2" @@ -61,11 +62,11 @@ tls: require.Equal(t, 5*time.Second, opts.TCPConnectionKeepAlivePeriod()) require.True(t, opts.TCPConnectionKeepAlive()) - require.Equal(t, TLSEnforced, opts.TLSOptions().Mode()) + require.Equal(t, xtls.Enforced, opts.TLSOptions().ServerMode()) require.True(t, opts.TLSOptions().MutualTLSEnabled()) require.Equal(t, "/tmp/cert", opts.TLSOptions().CertFile()) require.Equal(t, "/tmp/key", opts.TLSOptions().KeyFile()) - require.Equal(t, "/tmp/ca", opts.TLSOptions().ClientCAFile()) + require.Equal(t, "/tmp/ca", opts.TLSOptions().CAFile()) require.Equal(t, 10*time.Minute, opts.TLSOptions().CertificatesTTL()) require.NotNil(t, cfg.NewServer(nil, instrument.NewOptions())) diff --git a/src/x/server/options.go b/src/x/server/options.go index fb5a9df405..3cab5ec435 100644 --- a/src/x/server/options.go +++ b/src/x/server/options.go @@ -26,6 +26,7 @@ import ( "github.com/m3db/m3/src/x/instrument" xnet "github.com/m3db/m3/src/x/net" "github.com/m3db/m3/src/x/retry" + xtls "github.com/m3db/m3/src/x/tls" ) const ( @@ -73,10 +74,10 @@ type Options interface { ListenerOptions() xnet.ListenerOptions // SetTLSOptions sets the tls options for the server - SetTLSOptions(value TLSOptions) Options + SetTLSOptions(value xtls.Options) Options // TLSOptions returns the tls options for the server - TLSOptions() TLSOptions + TLSOptions() xtls.Options } type options struct { @@ -85,7 +86,7 @@ type options struct { tcpConnectionKeepAlive bool tcpConnectionKeepAlivePeriod time.Duration listenerOpts xnet.ListenerOptions - tlsOptions TLSOptions + tlsOptions xtls.Options } // NewOptions creates a new set of server options @@ -96,7 +97,7 @@ func NewOptions() Options { tcpConnectionKeepAlive: defaultTCPConnectionKeepAlive, tcpConnectionKeepAlivePeriod: defaultTCPConnectionKeepAlivePeriod, listenerOpts: xnet.NewListenerOptions(), - tlsOptions: NewTLSOptions(), + tlsOptions: xtls.NewOptions(), } } @@ -150,12 +151,12 @@ func (o *options) ListenerOptions() xnet.ListenerOptions { return o.listenerOpts } -func (o *options) SetTLSOptions(value TLSOptions) Options { +func (o *options) SetTLSOptions(value xtls.Options) Options { opts := *o opts.tlsOptions = value return &opts } -func (o *options) TLSOptions() TLSOptions { +func (o *options) TLSOptions() xtls.Options { return o.tlsOptions } diff --git a/src/x/server/server.go b/src/x/server/server.go index 8f97637e0e..602e70c4d5 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -22,17 +22,15 @@ package server import ( - "crypto/tls" - "crypto/x509" "fmt" "net" - "os" "sync" "sync/atomic" "time" xnet "github.com/m3db/m3/src/x/net" "github.com/m3db/m3/src/x/retry" + xtls "github.com/m3db/m3/src/x/tls" "github.com/uber-go/tally" "go.uber.org/zap" @@ -87,18 +85,15 @@ type server struct { tcpConnectionKeepAlive bool tcpConnectionKeepAlivePeriod time.Duration - closed bool - closedChan chan struct{} - numConns int32 - conns []net.Conn - wgConns sync.WaitGroup - metrics serverMetrics - handler Handler - listenerOpts xnet.ListenerOptions - tlsOpts TLSOptions - certPool *x509.CertPool - tlsConfigCache *tls.Config - certsTTLTicker *time.Ticker + closed bool + closedChan chan struct{} + numConns int32 + conns []net.Conn + wgConns sync.WaitGroup + metrics serverMetrics + handler Handler + listenerOpts xnet.ListenerOptions + tlsConfigManager xtls.ConfigManager addConnectionFn addConnectionFn removeConnectionFn removeConnectionFn @@ -109,11 +104,6 @@ func NewServer(address string, handler Handler, opts Options) Server { instrumentOpts := opts.InstrumentOptions() scope := instrumentOpts.MetricsScope() - var certsTTLTicker *time.Ticker - if opts.TLSOptions().CertificatesTTL() > 0 { - certsTTLTicker = time.NewTicker(opts.TLSOptions().CertificatesTTL()) - } - s := &server{ address: address, log: instrumentOpts.Logger(), @@ -125,9 +115,7 @@ func NewServer(address string, handler Handler, opts Options) Server { metrics: newServerMetrics(scope), handler: handler, listenerOpts: opts.ListenerOptions(), - tlsOpts: opts.TLSOptions(), - certPool: x509.NewCertPool(), - certsTTLTicker: certsTTLTicker, + tlsConfigManager: xtls.NewConfigManager(opts.TLSOptions(), instrumentOpts), } // Set up the connection functions. @@ -156,52 +144,8 @@ func (s *server) Serve(l net.Listener) error { return nil } -func (s *server) isTLSConfigCacheValid() bool { - if s.tlsConfigCache == nil || s.certsTTLTicker == nil { - return false - } - select { - case <-s.certsTTLTicker.C: - return false - default: - return true - } -} - -func (s *server) getTLSConfig() (*tls.Config, error) { - s.Lock() - defer s.Unlock() - if s.isTLSConfigCacheValid() { - return s.tlsConfigCache, nil - } - if s.tlsOpts.ClientCAFile() != "" { - certs, err := os.ReadFile(s.tlsOpts.ClientCAFile()) - if err != nil { - return nil, fmt.Errorf("read bundle error: %w", err) - } - if ok := s.certPool.AppendCertsFromPEM(certs); !ok { - return nil, fmt.Errorf("cannot append cert to cert pool") - } - } - clientAuthType := tls.NoClientCert - if s.tlsOpts.MutualTLSEnabled() { - clientAuthType = tls.RequireAndVerifyClientCert - } - cert, err := tls.LoadX509KeyPair(s.tlsOpts.CertFile(), s.tlsOpts.KeyFile()) - if err != nil { - return nil, fmt.Errorf("load x509 key pair error: %w", err) - } - tlsConfig := &tls.Config{ - ClientCAs: s.certPool, - Certificates: []tls.Certificate{cert}, - ClientAuth: clientAuthType, - } - s.tlsConfigCache = tlsConfig - return tlsConfig, nil -} - func (s *server) maybeUpgradeToTLS(conn SecuredConn) (SecuredConn, error) { - if s.tlsOpts.Mode() == TLSDisabled { + if s.tlsConfigManager.ServerMode() == xtls.Disabled { return conn, nil } isTLSConnection, err := conn.IsTLS() @@ -209,7 +153,7 @@ func (s *server) maybeUpgradeToTLS(conn SecuredConn) (SecuredConn, error) { return nil, err } if isTLSConnection { - tlsConfig, err := s.getTLSConfig() + tlsConfig, err := s.tlsConfigManager.TLSConfig() if err != nil { return nil, err } @@ -217,7 +161,7 @@ func (s *server) maybeUpgradeToTLS(conn SecuredConn) (SecuredConn, error) { if err != nil { return nil, err } - } else if s.tlsOpts.Mode() == TLSEnforced { + } else if s.tlsConfigManager.ServerMode() == xtls.Enforced { return nil, fmt.Errorf("not a tls connection") } return conn, nil diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index 2f156796c3..88536c7959 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -31,6 +31,7 @@ import ( "time" "github.com/m3db/m3/src/x/retry" + xtls "github.com/m3db/m3/src/x/tls" "github.com/stretchr/testify/require" ) @@ -40,20 +41,20 @@ const ( ) func testPlainTCPServer(addr string) (*server, *mockHandler, *int32, *int32) { - return testServer(addr, TLSDisabled, false) + return testServer(addr, xtls.Disabled, false) } // nolint: unparam -func testServer(addr string, tlsMode TLSMode, mTLSEnabled bool) (*server, *mockHandler, *int32, *int32) { +func testServer(addr string, tlsMode xtls.ServerMode, mTLSEnabled bool) (*server, *mockHandler, *int32, *int32) { var ( numAdded int32 numRemoved int32 ) - tlsOpts := NewTLSOptions(). - SetMode(tlsMode). + tlsOpts := xtls.NewOptions(). + SetServerMode(tlsMode). SetMutualTLSEnabled(mTLSEnabled). - SetClientCAFile("./testdata/rootCA.crt"). + SetCAFile("./testdata/rootCA.crt"). SetCertFile("./testdata/server.crt"). SetKeyFile("./testdata/server.key") @@ -130,7 +131,7 @@ func TestServe(t *testing.T) { } func TestTLSPermissiveServerListenAndClose(t *testing.T) { - s, h, numAdded, numRemoved := testServer(testListenAddress, TLSPermissive, false) + s, h, numAdded, numRemoved := testServer(testListenAddress, xtls.Permissive, false) var ( numClients = 9 @@ -176,7 +177,7 @@ func TestTLSPermissiveServerListenAndClose(t *testing.T) { } func TestTLSEnforcedServerListenAndClose(t *testing.T) { - s, h, numAdded, numRemoved := testServer(testListenAddress, TLSEnforced, false) + s, h, numAdded, numRemoved := testServer(testListenAddress, xtls.Enforced, false) var ( numClients = 10 @@ -222,7 +223,7 @@ func TestTLSEnforcedServerListenAndClose(t *testing.T) { } func TestMutualTLSServerListenAndClose(t *testing.T) { - s, h, numAdded, numRemoved := testServer(testListenAddress, TLSEnforced, true) + s, h, numAdded, numRemoved := testServer(testListenAddress, xtls.Enforced, true) var ( numClients = 9 diff --git a/src/x/server/tls_options.go b/src/x/server/tls_options.go deleted file mode 100644 index c295769fc0..0000000000 --- a/src/x/server/tls_options.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package server - -import "time" - -// TLSOptions provide a set of TLS options -type TLSOptions interface { - // SetMode sets the tls mode - SetMode(value TLSMode) TLSOptions - // Mode returns the tls mode - Mode() TLSMode - - // SetMutualTLSEnabled sets the mutual tls enabled option - SetMutualTLSEnabled(value bool) TLSOptions - // MutualTLSEnabled returns the mutual tls enabled option - MutualTLSEnabled() bool - - // SetCertFile sets the certificate file path - SetCertFile(value string) TLSOptions - // CertFile returns the certificate file path - CertFile() string - - // SetKeyFile sets the private key file path - SetKeyFile(value string) TLSOptions - // KeyFile returns the private key file path - KeyFile() string - - // SetClientCAFile sets the CA file path - SetClientCAFile(value string) TLSOptions - // ClientCAFile returns the CA file path - ClientCAFile() string - - // SetCertificatesTTL sets the certificates TTL - SetCertificatesTTL(value time.Duration) TLSOptions - // CertificatesTTL returns the certificates TTL - CertificatesTTL() time.Duration -} - -type tlsOptions struct { - mode TLSMode - mTLSEnabled bool - certFile string - keyFile string - clientCAFile string - certificatesTTL time.Duration -} - -// NewTLSOptions creates a new set of tls options -func NewTLSOptions() TLSOptions { - return &tlsOptions{ - mode: TLSDisabled, - mTLSEnabled: false, - } -} - -func (o *tlsOptions) SetMode(value TLSMode) TLSOptions { - opts := *o - opts.mode = value - return &opts -} - -func (o *tlsOptions) Mode() TLSMode { - return o.mode -} - -func (o *tlsOptions) SetMutualTLSEnabled(value bool) TLSOptions { - opts := *o - opts.mTLSEnabled = value - return &opts -} - -func (o *tlsOptions) MutualTLSEnabled() bool { - return o.mTLSEnabled -} - -func (o *tlsOptions) SetCertFile(value string) TLSOptions { - opts := *o - opts.certFile = value - return &opts -} - -func (o *tlsOptions) CertFile() string { - return o.certFile -} - -func (o *tlsOptions) SetKeyFile(value string) TLSOptions { - opts := *o - opts.keyFile = value - return &opts -} - -func (o *tlsOptions) KeyFile() string { - return o.keyFile -} - -func (o *tlsOptions) SetClientCAFile(value string) TLSOptions { - opts := *o - opts.clientCAFile = value - return &opts -} - -func (o *tlsOptions) ClientCAFile() string { - return o.clientCAFile -} - -func (o *tlsOptions) SetCertificatesTTL(value time.Duration) TLSOptions { - opts := *o - opts.certificatesTTL = value - return &opts -} - -func (o *tlsOptions) CertificatesTTL() time.Duration { - return o.certificatesTTL -} diff --git a/src/x/tls/config_manager.go b/src/x/tls/config_manager.go new file mode 100644 index 0000000000..d6d809511d --- /dev/null +++ b/src/x/tls/config_manager.go @@ -0,0 +1,162 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package tls + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "sync" + "time" + + "github.com/m3db/m3/src/x/instrument" + "github.com/uber-go/tally" + "go.uber.org/zap" +) + +const getConfigMetricName = "get-tls-config" + +var sleepFn = time.Sleep + +type ConfigManager interface { + TLSConfig() (*tls.Config, error) + ServerMode() ServerMode + ClientEnabled() bool +} + +type configManager struct { + sync.RWMutex + + log *zap.Logger + metrics configManagerMetrics + options Options + certPool *x509.CertPool + tlsConfig *tls.Config +} + +type configManagerMetrics struct { + getTLSConfigSuccess tally.Counter + getTLSConfigErrors tally.Counter +} + +func newConfigManagerScope(scope tally.Scope) configManagerMetrics { + return configManagerMetrics{ + getTLSConfigSuccess: scope.Tagged(map[string]string{"success": "true"}).Counter(getConfigMetricName), + getTLSConfigErrors: scope.Tagged(map[string]string{"success": "false"}).Counter(getConfigMetricName), + } +} + +func NewConfigManager(opts Options, instrumentOpts instrument.Options) ConfigManager { + scope := instrumentOpts.MetricsScope() + c := &configManager{ + log: instrumentOpts.Logger(), + metrics: newConfigManagerScope(scope), + options: opts, + certPool: x509.NewCertPool(), + } + go c.updateCertificates() + return c +} + +func (c *configManager) updateCertificates() { + if c.options.CertificatesTTL() == 0 { + return + } + for { + tlsConfig, err := c.loadTLSConfig() + if err != nil { + c.metrics.getTLSConfigErrors.Inc(1) + c.log.Error("get tls config error", zap.Error(err)) + sleepFn(c.options.CertificatesTTL()) + continue + } + c.Lock() + c.tlsConfig = tlsConfig + c.Unlock() + c.metrics.getTLSConfigSuccess.Inc(1) + sleepFn(c.options.CertificatesTTL()) + } +} + +func (c *configManager) loadCertPool() (*x509.CertPool, error) { + if c.options.CAFile() != "" { + certs, err := os.ReadFile(c.options.CAFile()) + if err != nil { + return nil, fmt.Errorf("read bundle error: %w", err) + } + if ok := c.certPool.AppendCertsFromPEM(certs); !ok { + return nil, fmt.Errorf("cannot append cert to cert pool") + } + } + return c.certPool, nil +} + +func (c *configManager) loadX509KeyPair() ([]tls.Certificate, error) { + if c.options.CertFile() == "" || c.options.KeyFile() == "" { + return []tls.Certificate{}, nil + } + cert, err := tls.LoadX509KeyPair(c.options.CertFile(), c.options.KeyFile()) + if err != nil { + return nil, fmt.Errorf("load x509 key pair error: %w", err) + } + return []tls.Certificate{cert}, nil +} + +func (c *configManager) loadTLSConfig() (*tls.Config, error) { + certPool, err := c.loadCertPool() + if err != nil { + return nil, fmt.Errorf("load cert pool failed: %w", err) + } + certificate, err := c.loadX509KeyPair() + if err != nil { + return nil, fmt.Errorf("load x509 key pair failed: %w", err) + } + clientAuthType := tls.NoClientCert + if c.options.MutualTLSEnabled() { + clientAuthType = tls.RequireAndVerifyClientCert + } + return &tls.Config{ + RootCAs: certPool, + ClientCAs: certPool, + Certificates: certificate, + ClientAuth: clientAuthType, + InsecureSkipVerify: c.options.InsecureSkipVerify(), + ServerName: c.options.ServerName(), + }, nil +} + +func (c *configManager) TLSConfig() (*tls.Config, error) { + if c.options.CertificatesTTL() == 0 || c.tlsConfig == nil { + return c.loadTLSConfig() + } + c.RLock() + defer c.RUnlock() + return c.tlsConfig, nil +} + +func (c *configManager) ServerMode() ServerMode { + return c.options.ServerMode() +} + +func (c *configManager) ClientEnabled() bool { + return c.options.ClientEnabled() +} diff --git a/src/x/tls/config_manager_test.go b/src/x/tls/config_manager_test.go new file mode 100644 index 0000000000..0971aab5ba --- /dev/null +++ b/src/x/tls/config_manager_test.go @@ -0,0 +1,219 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package tls + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "testing" + "time" + + "github.com/m3db/m3/src/x/instrument" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +type counterMock struct { + incCallback func() +} + +func (c *counterMock) Inc(_ int64) { + c.incCallback() +} + +func appendCA(filename string, certPool *x509.CertPool) error { + certs, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("read bundle error: %w", err) + } + if ok := certPool.AppendCertsFromPEM(certs); !ok { + return fmt.Errorf("cannot append cert to cert pool") + } + return nil +} + +func TestLoadCertPool(t *testing.T) { + opts := NewOptions() + cm := &configManager{ + options: opts, + certPool: x509.NewCertPool(), + } + expectedCertPool := x509.NewCertPool() + + opts = opts.SetCAFile("") + cm.options = opts + certPool, err := cm.loadCertPool() + require.NoError(t, err) + require.True(t, expectedCertPool.Equal(certPool)) + + opts = opts.SetCAFile("testdata/1.crt") + cm.options = opts + certPool, err = cm.loadCertPool() + require.NoError(t, err) + err = appendCA("testdata/1.crt", expectedCertPool) + require.NoError(t, err) + require.True(t, expectedCertPool.Equal(certPool)) + require.True(t, expectedCertPool.Equal(cm.certPool)) + + opts = opts.SetCAFile("testdata/2.crt") + cm.options = opts + certPool, err = cm.loadCertPool() + require.NoError(t, err) + err = appendCA("testdata/2.crt", expectedCertPool) + require.NoError(t, err) + require.True(t, expectedCertPool.Equal(certPool)) + require.True(t, expectedCertPool.Equal(cm.certPool)) + + opts = opts.SetCAFile("testdata/3.crt") + cm.options = opts + certPool, err = cm.loadCertPool() + require.Error(t, err) + require.True(t, expectedCertPool.Equal(cm.certPool)) + + opts = opts.SetCAFile("wrong/path") + cm.options = opts + certPool, err = cm.loadCertPool() + require.Error(t, err) +} + +func TestLoadX509KeyPair(t *testing.T) { + opts := NewOptions() + cm := &configManager{ + options: opts, + } + + opts = opts.SetCertFile("").SetKeyFile("not empty") + cm.options = opts + certificates, err := cm.loadX509KeyPair() + require.NoError(t, err) + require.Len(t, certificates, 0) + + opts = opts.SetCertFile("not empty").SetKeyFile("") + cm.options = opts + certificates, err = cm.loadX509KeyPair() + require.NoError(t, err) + require.Len(t, certificates, 0) + + opts = opts.SetCertFile("wrong/path").SetKeyFile("wrong/path") + cm.options = opts + certificates, err = cm.loadX509KeyPair() + require.Error(t, err) + + opts = opts.SetCertFile("testdata/1.crt").SetKeyFile("testdata/1.key") + cm.options = opts + certificates, err = cm.loadX509KeyPair() + require.NoError(t, err) + require.Len(t, certificates, 1) +} + +func TestLoadTLSConfig(t *testing.T) { + opts := NewOptions() + cm := &configManager{ + options: opts, + certPool: x509.NewCertPool(), + log: zap.NewNop(), + } + + opts = opts.SetCAFile("wrong/path") + cm.options = opts + tlsConfig, err := cm.loadTLSConfig() + require.Error(t, err) + + opts = opts.SetCAFile("testdata/1.crt") + opts = opts.SetCertFile("wrong/path").SetKeyFile("wrong/path") + cm.options = opts + tlsConfig, err = cm.loadTLSConfig() + require.Error(t, err) + + opts = opts. + SetCertFile("testdata/1.crt"). + SetKeyFile("testdata/1.key"). + SetMutualTLSEnabled(true). + SetInsecureSkipVerify(true). + SetServerName("server name") + cm.options = opts + tlsConfig, err = cm.loadTLSConfig() + require.NoError(t, err) + require.NotNil(t, tlsConfig.RootCAs) + require.NotNil(t, tlsConfig.ClientCAs) + require.Len(t, tlsConfig.Certificates, 1) + require.Equal(t, tls.RequireAndVerifyClientCert, tlsConfig.ClientAuth) + require.True(t, tlsConfig.InsecureSkipVerify) + require.Equal(t, "server name", tlsConfig.ServerName) +} + +func TestUpdateCertificate(t *testing.T) { + var waitCh = make(chan bool) + var waitCalledCh = make(chan bool) + var readConfigSuccess = 0 + successCounterMock := &counterMock{ + incCallback: func() { readConfigSuccess++ }, + } + var readConfigErrors = 0 + errorCounterMock := &counterMock{ + incCallback: func() { readConfigErrors++ }, + } + cmm := configManagerMetrics{ + getTLSConfigSuccess: successCounterMock, + getTLSConfigErrors: errorCounterMock, + } + originalSleepFn := sleepFn + sleepFn = func(d time.Duration) { + waitCalledCh <- true + <-waitCh + } + defer func() { sleepFn = originalSleepFn }() + + instrumentOpts := instrument.NewOptions().SetLogger(zap.NewNop()) + opts := NewOptions(). + SetCertificatesTTL(0) + cmInterface := NewConfigManager(opts, instrumentOpts) + cm := cmInterface.(*configManager) + cm.metrics = cmm + require.Nil(t, cm.tlsConfig) + require.Equal(t, 0, readConfigSuccess) + require.Equal(t, 0, readConfigErrors) + + opts = opts. + SetCertificatesTTL(time.Second). + SetCAFile("testdata/1.crt"). + SetCertFile("testdata/1.crt"). + SetKeyFile("testdata/1.key"). + SetMutualTLSEnabled(true). + SetInsecureSkipVerify(true). + SetServerName("server name") + cmInterface = NewConfigManager(opts, instrumentOpts) + cm = cmInterface.(*configManager) + cm.metrics = cmm + <-waitCalledCh + require.NotNil(t, cm.tlsConfig) + require.Equal(t, 1, readConfigSuccess) + require.Equal(t, 0, readConfigErrors) + + opts = opts.SetCAFile("wrong/path") + cm.options = opts + waitCh <- true + <-waitCalledCh + require.Equal(t, 1, readConfigSuccess) + require.Equal(t, 1, readConfigErrors) +} diff --git a/src/x/server/tls_mode.go b/src/x/tls/mode.go similarity index 74% rename from src/x/server/tls_mode.go rename to src/x/tls/mode.go index 04e8f5a1c3..b77597dd7e 100644 --- a/src/x/server/tls_mode.go +++ b/src/x/tls/mode.go @@ -18,31 +18,30 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -// Package server implements a network server. -package server +package tls import "fmt" const ( - // TLSDisabled allows plaintext connections only - TLSDisabled TLSMode = iota - // TLSPermissive allows both TLS and plaintext connections - TLSPermissive - // TLSEnforced allows TLS connections only - TLSEnforced + // Disabled allows plaintext connections only + Disabled ServerMode = iota + // Permissive allows both TLS and plaintext connections + Permissive + // Enforced allows TLS connections only + Enforced ) -// TLSMode represents the TLS mode -type TLSMode uint16 +// ServerMode represents the TLS mode +type ServerMode uint16 -func (t *TLSMode) UnmarshalText(mode []byte) error { +func (t *ServerMode) UnmarshalText(mode []byte) error { switch string(mode) { case "disabled": - *t = TLSDisabled + *t = Disabled case "permissive": - *t = TLSPermissive + *t = Permissive case "enforced": - *t = TLSEnforced + *t = Enforced default: return fmt.Errorf("unknown tls mode: %s", mode) } diff --git a/src/x/server/tls_mode_test.go b/src/x/tls/mode_test.go similarity index 87% rename from src/x/server/tls_mode_test.go rename to src/x/tls/mode_test.go index 590566a487..620ef3b279 100644 --- a/src/x/server/tls_mode_test.go +++ b/src/x/tls/mode_test.go @@ -18,8 +18,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -// Package server implements a network server. -package server +package tls import ( "testing" @@ -28,16 +27,16 @@ import ( ) func TestTLSModeUnmarshal(t *testing.T) { - var tlsMode TLSMode + var tlsMode ServerMode require.NoError(t, tlsMode.UnmarshalText([]byte("disabled"))) - require.Equal(t, TLSDisabled, tlsMode) + require.Equal(t, Disabled, tlsMode) require.NoError(t, tlsMode.UnmarshalText([]byte("permissive"))) - require.Equal(t, TLSPermissive, tlsMode) + require.Equal(t, Permissive, tlsMode) require.NoError(t, tlsMode.UnmarshalText([]byte("enforced"))) - require.Equal(t, TLSEnforced, tlsMode) + require.Equal(t, Enforced, tlsMode) require.Error(t, tlsMode.UnmarshalText([]byte("unknown"))) } diff --git a/src/aggregator/client/tls_options.go b/src/x/tls/options.go similarity index 51% rename from src/aggregator/client/tls_options.go rename to src/x/tls/options.go index b29528345c..fa261b378d 100644 --- a/src/aggregator/client/tls_options.go +++ b/src/x/tls/options.go @@ -1,4 +1,4 @@ -// Copyright (c) 2018 Uber Technologies, Inc. +// Copyright (c) 2017 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -18,138 +18,166 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -package client +package tls import "time" -type TLSOptions interface { - // SetEnabled sets the TLS enabled option - SetEnabled(value bool) TLSOptions - - // Enabled returns the TLS enabled option - Enabled() bool +// Options provide a set of TLS options +type Options interface { + // SetClientEnabled sets the TLS enabled option + SetClientEnabled(value bool) Options + // ClientEnabled returns the TLS enabled option + ClientEnabled() bool // SetInsecureSkipVerify sets the insecure skip verify option - SetInsecureSkipVerify(value bool) TLSOptions - + SetInsecureSkipVerify(value bool) Options // InsecureSkipVerify returns the insecure skip verify option InsecureSkipVerify() bool // SetServerName sets the server name option - SetServerName(value string) TLSOptions - + SetServerName(value string) Options // ServerName returns the server name option ServerName() string - // SetCAFile sets the CA file path - SetCAFile(value string) TLSOptions + // SetServerMode sets the tls mode + SetServerMode(value ServerMode) Options + // Mode returns the tls mode + ServerMode() ServerMode - // CAFile returns the CA file path - CAFile() string + // SetMutualTLSEnabled sets the mutual tls enabled option + SetMutualTLSEnabled(value bool) Options + // MutualTLSEnabled returns the mutual tls enabled option + MutualTLSEnabled() bool // SetCertFile sets the certificate file path - SetCertFile(value string) TLSOptions - + SetCertFile(value string) Options // CertFile returns the certificate file path CertFile() string - // SetKeyFile sets the key file path - SetKeyFile(value string) TLSOptions - - // KeyFile returns the key file path + // SetKeyFile sets the private key file path + SetKeyFile(value string) Options + // KeyFile returns the private key file path KeyFile() string - // SetCertificatesTTL sets the certificates TTL - SetCertificatesTTL(value time.Duration) TLSOptions + // SetCAFile sets the CA file path + SetCAFile(value string) Options + // CAFile returns the CA file path + CAFile() string + // SetCertificatesTTL sets the certificates TTL + SetCertificatesTTL(value time.Duration) Options // CertificatesTTL returns the certificates TTL CertificatesTTL() time.Duration } -type tlsOptions struct { - enabled bool +type options struct { + clientEnabled bool insecureSkipVerify bool serverName string - caFile string + serverMode ServerMode + mTLSEnabled bool certFile string keyFile string + caFile string certificatesTTL time.Duration } -// NewTLSOptions creates new TLS options -func NewTLSOptions() TLSOptions { - return &tlsOptions{ - enabled: false, +// NewOptions creates a new set of tls options +func NewOptions() Options { + return &options{ + clientEnabled: false, insecureSkipVerify: true, + serverMode: Disabled, + mTLSEnabled: false, } } -func (o *tlsOptions) SetEnabled(value bool) TLSOptions { +func (o *options) SetClientEnabled(value bool) Options { opts := *o - opts.enabled = value + opts.clientEnabled = value return &opts } -func (o *tlsOptions) Enabled() bool { - return o.enabled +func (o *options) ClientEnabled() bool { + return o.clientEnabled } -func (o *tlsOptions) SetInsecureSkipVerify(value bool) TLSOptions { +func (o *options) SetInsecureSkipVerify(value bool) Options { opts := *o opts.insecureSkipVerify = value return &opts } -func (o *tlsOptions) InsecureSkipVerify() bool { +func (o *options) InsecureSkipVerify() bool { return o.insecureSkipVerify } -func (o *tlsOptions) SetServerName(value string) TLSOptions { +func (o *options) SetServerName(value string) Options { opts := *o opts.serverName = value return &opts } -func (o *tlsOptions) ServerName() string { +func (o *options) ServerName() string { return o.serverName } -func (o *tlsOptions) SetCAFile(value string) TLSOptions { +func (o *options) SetServerMode(value ServerMode) Options { opts := *o - opts.caFile = value + opts.serverMode = value return &opts } -func (o *tlsOptions) CAFile() string { - return o.caFile +func (o *options) ServerMode() ServerMode { + return o.serverMode +} + +func (o *options) SetMutualTLSEnabled(value bool) Options { + opts := *o + opts.mTLSEnabled = value + return &opts +} + +func (o *options) MutualTLSEnabled() bool { + return o.mTLSEnabled } -func (o *tlsOptions) SetCertFile(value string) TLSOptions { +func (o *options) SetCertFile(value string) Options { opts := *o opts.certFile = value return &opts } -func (o *tlsOptions) CertFile() string { +func (o *options) CertFile() string { return o.certFile } -func (o *tlsOptions) SetKeyFile(value string) TLSOptions { +func (o *options) SetKeyFile(value string) Options { opts := *o opts.keyFile = value return &opts } -func (o *tlsOptions) KeyFile() string { +func (o *options) KeyFile() string { return o.keyFile } -func (o *tlsOptions) SetCertificatesTTL(value time.Duration) TLSOptions { +func (o *options) SetCAFile(value string) Options { + opts := *o + opts.caFile = value + return &opts +} + +func (o *options) CAFile() string { + return o.caFile +} + +func (o *options) SetCertificatesTTL(value time.Duration) Options { opts := *o opts.certificatesTTL = value return &opts } -func (o *tlsOptions) CertificatesTTL() time.Duration { +func (o *options) CertificatesTTL() time.Duration { return o.certificatesTTL } diff --git a/src/x/tls/testdata/1.crt b/src/x/tls/testdata/1.crt new file mode 100644 index 0000000000..3618a40052 --- /dev/null +++ b/src/x/tls/testdata/1.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPzCCAicCFFJVDW2T6lamcRZYStsBCQTKicKvMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAk5MMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwHhcNMjQw +NDA0MTE0ODIwWhcNMzEwNzA3MTE0ODIwWjBeMQswCQYDVQQGEwJOTDETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRcwFQYDVQQDDA5UZXN0U2VydmVyQ2VydDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAOoGyqgrsBCufd9UNt6wr9UJzPaVqdrr/Zble2UPMVZOynAT69W5 +gMdNvrIDU6l1EnRkLyxmP7+beYZvWyU+kN9LyP0pdcgSQdtyhbr1CkqMxhM1MA2B +OFvcEFN2rybaqcLDlq2WUQkmyuVQt7/weLU9jC596PtxUL1e0+wqIItISWpcmemf +cwH9TVlD8Yh6emf87v27afxc0bZCauPsrEPvOC00oU4epXnA7TeDs4RtfqVAuVHX +oEG3oNZ3NHXIIgSht3rNAEfOxBX+7KGvW4hW9ci6W758ECGzHFJThhzO3+w+0YPP +29m+v4CVRaXAGxkVRPiAUDpU8RSPteywTyMCAwEAATANBgkqhkiG9w0BAQsFAAOC +AQEAuQME+WSPw1KKK1jgh1wuiyM5hou1BeagfwOipM3CEblTiHfoERsd2f7Oi6e2 +zPACrvKjVP6bHKjZPEztFHGXDuLdBqK97Vd/o4IFl6p/8EuBeWV85o4Rk1PMhqBI +0jSxsXXhBLgSTkUBgSllwgAbAaxm+L3wBJiQOhh/wEdfNSJ/GeL5477J56Q12/hA +rRCghYRAVVpr44NfG/PbuuMklQHK2l+wkJFBnx+zAAtZF/nb5yCVSy8z/p2TT72a +STfegXYJtE3p4X2+78xajrI7x/PWhNlzJRn3txLW6461vYJIB3ZDVUA2LZtmi3nd +9BUuJVSvsoVwlvrUBF0S1P0vWg== +-----END CERTIFICATE----- diff --git a/src/x/tls/testdata/1.key b/src/x/tls/testdata/1.key new file mode 100644 index 0000000000..6b6e1cd4cc --- /dev/null +++ b/src/x/tls/testdata/1.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA6gbKqCuwEK5931Q23rCv1QnM9pWp2uv9luV7ZQ8xVk7KcBPr +1bmAx02+sgNTqXUSdGQvLGY/v5t5hm9bJT6Q30vI/Sl1yBJB23KFuvUKSozGEzUw +DYE4W9wQU3avJtqpwsOWrZZRCSbK5VC3v/B4tT2MLn3o+3FQvV7T7Cogi0hJalyZ +6Z9zAf1NWUPxiHp6Z/zu/btp/FzRtkJq4+ysQ+84LTShTh6lecDtN4OzhG1+pUC5 +UdegQbeg1nc0dcgiBKG3es0AR87EFf7soa9biFb1yLpbvnwQIbMcUlOGHM7f7D7R +g8/b2b6/gJVFpcAbGRVE+IBQOlTxFI+17LBPIwIDAQABAoIBAG7rZSYr/rkJWxN8 +wLzvlYctq/27ldPzQjBA+Ck/+CxmD8DrUD3uPEE6cgXBWefZWzPbQBiVJeXU9NZo +9RREATo0m0CSZAa90IUIVKX1+ji3XDJCZVIhSusO9JX6jK2nugpb1ZDUy/fAcS4b +AdxxR2TvlP98Ie77GD+JMHygwugAJPQ4AFZQjO2F/2k3crVt9Frt1VSvf72nkM/C +sDJy4kv+lDZsbeveNPzJo1honiKdCDWpg4duLiLkkG4YvftWHGzpXHoOWnuwnrT5 +jrQTaG10d1jMpnxOwDrxRsFYWtz4kbJm2VtRsgWRIy4A4Vv8VKfFJ0cSu0CChjTE +7yVrExECgYEA9orYMAuuu/c0HGVcXOsA9x6XHtBXr5wK1fI973sGUrtkshTlBgjG +Bc7H+AhVBVdqr2ipm9pFZYSV1OYB2+3AgMQjhoZANR9jr5bmRjJgWuBi5umrFKa4 +n0IQbkwSzjxKvkhpJK25BY2kpjh7/128Q80PMkKnm0LR2gyUWvLFuUUCgYEA8wEJ +PyxHBu93NnbwTP0SUEOYFa654ctiVLbicdcUsbo/Qo4gl/0AIpGmMdeKqx3FWn9I +kCH1xlvb7hp7FNFhDZIiw6CT8V2RQHXdEO9/S0jjOOBg29hxARIgoU+PjjaQG/H+ +SqlDR/ijXUghlllwtP6jCJqy29+Ukpx8WnIkiUcCgYEA4DNXcinNlyWw//m1StJz +KEdrShxRAWO3/uOyKywHjgjYA5I4y4oC+pkXBJofJyJd5HIbe5286hruXpNkMWo+ +W1sCq30+upB3COEDO+bn9PyFNFRevgL3deyQIJ2RpHbzyaLz6uUIkuJCTt6i9JF7 +eXTrUtuGrAzR5ab26tZHtLkCgYBZtW42diAafVjjMHFx2pNiMjH4/zH8G1tHruRV +5nLrUbh8139M3Mep+18BU3MLzMfIquic3Qygl9TPR9NcUhekLOjMk4FKP6KZy2k7 +EU9K+bN4kczwHC1LmeeV5Ys3kwkMZ//ajDYcYbvdef18RbSH/OEvEf0pkpaEsWRu +92hpjQKBgH6Y3dW0bQIhXikjboD00vPn0zPuh35I9OfrPHMqDJiz+xom7Sz9tlGk +NfFll5PTf0gZtU7160uzhCp/OqXT3IdVgVt/G5ey13oYP7pCb8MWiuyNW0yUgtTp +GM5GUnEW5zJ1t2qJOxwoq/FA7D5C7EGRM/QpoMD0tsJoJKYyWkJO +-----END RSA PRIVATE KEY----- diff --git a/src/x/tls/testdata/2.crt b/src/x/tls/testdata/2.crt new file mode 100644 index 0000000000..366d36e8b8 --- /dev/null +++ b/src/x/tls/testdata/2.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDPzCCAicCFFJVDW2T6lamcRZYStsBCQTKicKwMA0GCSqGSIb3DQEBCwUAMFox +CzAJBgNVBAYTAk5MMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl +cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMClRlc3RSb290Q0EwHhcNMjQw +NDE1MTQ0NjEyWhcNMzQwNDEzMTQ0NjEyWjBeMQswCQYDVQQGEwJOTDETMBEGA1UE +CAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk +MRcwFQYDVQQDDA5UZXN0Q2xpZW50Q2VydDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAMXgv5tgZD5iBW3LfTC27jPHTLlsAw8ZlBFLmd7potdS3Hy512Xa +kKOaKRKH/0Yla4BKGNh0ZP+xege15CpTFqwtzDPCRkaRvfSxRW19MukeergRDqqI +tEwURHQ5gfub7XB9e5wuNmW2GGs/xgiWsxxm6UMidXkhA5a2//OmDSm2JdcoxJbk +1m1OUzXUuLKLzCIRAbJFFIM1RpvjWSonxpJ2lsNCVQ+LdItxE9mAI+Y+PaDonyzg +ehfAqkl+wxGRevaDFFJr1defxmnJhaYv40AcEpXOWeetZFlspEtrPF7b/3GifRdx +Mlwq27wn3mk4wF1wa12kRNKiz+HHjZrB7UECAwEAATANBgkqhkiG9w0BAQsFAAOC +AQEAKblZ+C4Pdv5grKiCQcfFt53SbIys8uavxmzJNn9nq1QVgBmR/hX8yG8ULeh5 +kKrccwe2WXCBWsIj9cvGCzR+MPMvuIXd3d58yLybmIUBDfJSBC0v5TaNWP2ZdJDG +fH38DfQTeE9XcjpRtCGYtYwReBptSlNiYQlvwkukTNz3mV+0BpvQu/9/R0BNCJAF +9VeLgUcG7T7HC+foCeU7h83Y0xH0OALe33ntcrMPHUclXqImCiC7dl8pb3H7o/xJ +jS3KZxfv6ioxpirvpEbX/EeF+H1HQZUZ8y3+J3K37jMgzNamd/xsaJxEfCq4SE+n +HmgeXgqoNjrM6J0jYAfx1MjbIA== +-----END CERTIFICATE----- diff --git a/src/x/tls/testdata/3.crt b/src/x/tls/testdata/3.crt new file mode 100644 index 0000000000..dd3d9a25e0 --- /dev/null +++ b/src/x/tls/testdata/3.crt @@ -0,0 +1 @@ +wrong data From 7997a0144f0011f2aa0f45659c5200771cec7257 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Mon, 8 Jul 2024 12:24:41 +0000 Subject: [PATCH 19/47] Metric added for upgrade to tls errors --- src/x/server/server.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/x/server/server.go b/src/x/server/server.go index 602e70c4d5..0832a84dab 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -62,12 +62,14 @@ type Handler interface { } type serverMetrics struct { - openConnections tally.Gauge + openConnections tally.Gauge + upgradeToTLSErrors tally.Counter } func newServerMetrics(scope tally.Scope) serverMetrics { return serverMetrics{ - openConnections: scope.Gauge("open-connections"), + openConnections: scope.Gauge("open-connections"), + upgradeToTLSErrors: scope.Counter("upgrade-to-tls-errors"), } } @@ -188,6 +190,8 @@ func (s *server) serve() { securedConn, err := s.maybeUpgradeToTLS(conn) if err != nil { + s.metrics.upgradeToTLSErrors.Inc(1) + s.log.Error("unable to upgrade connection to TLS", zap.Error(err)) return } s.handler.Handle(securedConn) From 8fd3c43b8ec9446b2d20c2cfedee0547c0768524 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 9 Jul 2024 09:14:10 +0000 Subject: [PATCH 20/47] newConfigManagerScope renamed to newConfigManagerMetrics --- src/x/tls/config_manager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/x/tls/config_manager.go b/src/x/tls/config_manager.go index d6d809511d..1e5d2810a8 100644 --- a/src/x/tls/config_manager.go +++ b/src/x/tls/config_manager.go @@ -58,7 +58,7 @@ type configManagerMetrics struct { getTLSConfigErrors tally.Counter } -func newConfigManagerScope(scope tally.Scope) configManagerMetrics { +func newConfigManagerMetrics(scope tally.Scope) configManagerMetrics { return configManagerMetrics{ getTLSConfigSuccess: scope.Tagged(map[string]string{"success": "true"}).Counter(getConfigMetricName), getTLSConfigErrors: scope.Tagged(map[string]string{"success": "false"}).Counter(getConfigMetricName), @@ -69,7 +69,7 @@ func NewConfigManager(opts Options, instrumentOpts instrument.Options) ConfigMan scope := instrumentOpts.MetricsScope() c := &configManager{ log: instrumentOpts.Logger(), - metrics: newConfigManagerScope(scope), + metrics: newConfigManagerMetrics(scope), options: opts, certPool: x509.NewCertPool(), } From 2c78899b495833a99b27aeea371eaa87eba5930f Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 9 Jul 2024 09:16:53 +0000 Subject: [PATCH 21/47] Config manager mutex moved to a field --- src/x/tls/config_manager.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/x/tls/config_manager.go b/src/x/tls/config_manager.go index 1e5d2810a8..88a431d62b 100644 --- a/src/x/tls/config_manager.go +++ b/src/x/tls/config_manager.go @@ -44,7 +44,7 @@ type ConfigManager interface { } type configManager struct { - sync.RWMutex + mu sync.RWMutex log *zap.Logger metrics configManagerMetrics @@ -89,9 +89,9 @@ func (c *configManager) updateCertificates() { sleepFn(c.options.CertificatesTTL()) continue } - c.Lock() + c.mu.Lock() c.tlsConfig = tlsConfig - c.Unlock() + c.mu.Unlock() c.metrics.getTLSConfigSuccess.Inc(1) sleepFn(c.options.CertificatesTTL()) } @@ -148,8 +148,8 @@ func (c *configManager) TLSConfig() (*tls.Config, error) { if c.options.CertificatesTTL() == 0 || c.tlsConfig == nil { return c.loadTLSConfig() } - c.RLock() - defer c.RUnlock() + c.mu.RLock() + defer c.mu.RUnlock() return c.tlsConfig, nil } From 19e99e0a789b7e0c6b047528fa434d9b78e67db4 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 9 Jul 2024 09:25:03 +0000 Subject: [PATCH 22/47] Redundant error check removed --- src/x/server/server.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/x/server/server.go b/src/x/server/server.go index 0832a84dab..de2521da92 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -160,9 +160,6 @@ func (s *server) maybeUpgradeToTLS(conn SecuredConn) (SecuredConn, error) { return nil, err } conn = conn.UpgradeToTLS(tlsConfig) - if err != nil { - return nil, err - } } else if s.tlsConfigManager.ServerMode() == xtls.Enforced { return nil, fmt.Errorf("not a tls connection") } From 3068241c7a2b118121ab9a614676bb57a3458630 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 9 Jul 2024 12:06:21 +0000 Subject: [PATCH 23/47] Write/read data tests added to secured connection --- src/x/server/secured_conn_test.go | 57 +++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/src/x/server/secured_conn_test.go b/src/x/server/secured_conn_test.go index 70d89952be..9c1109703d 100644 --- a/src/x/server/secured_conn_test.go +++ b/src/x/server/secured_conn_test.go @@ -29,24 +29,41 @@ import ( "github.com/stretchr/testify/require" ) -func testTCPServer(connCh chan net.Conn, errCh chan error) (net.Listener, error) { +func testTCPServer(connCh chan SecuredConn, errCh chan error) (net.Listener, error) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { return nil, err } - go func(net.Listener, chan net.Conn, chan error) { + go func(net.Listener, chan SecuredConn, chan error) { conn, err := listener.Accept() if err != nil { errCh <- err } else { - connCh <- conn + securedConn := newSecuredConn(conn) + isTLS, err := securedConn.IsTLS() + if err != nil { + errCh <- err + return + } + if isTLS { + certs, err := tls.LoadX509KeyPair("testdata/server.crt", "testdata/server.key") + if err != nil { + errCh <- err + return + } + tlsConfig := tls.Config{Certificates: []tls.Certificate{certs}} + securedConn = securedConn.UpgradeToTLS(&tlsConfig) + tlsConn := securedConn.GetConn().(*tls.Conn) + tlsConn.Handshake() + } + connCh <- securedConn } }(listener, connCh, errCh) return listener, nil } func TestPlainTCPConnection(t *testing.T) { - connCh := make(chan net.Conn) + connCh := make(chan SecuredConn) errCh := make(chan error) listener, err := testTCPServer(connCh, errCh) require.NoError(t, err) @@ -54,7 +71,8 @@ func TestPlainTCPConnection(t *testing.T) { clientConn, err := net.Dial("tcp", listener.Addr().String()) require.NoError(t, err) - _, err = clientConn.Write([]byte("not a tls connection")) + data := []byte("not a tls connection") + _, err = clientConn.Write(data) require.NoError(t, err) var conn SecuredConn @@ -70,32 +88,43 @@ func TestPlainTCPConnection(t *testing.T) { isTLS, err := conn.IsTLS() require.NoError(t, err) require.False(t, isTLS) + result := make([]byte, len(data)) + _, err = conn.Read(result) + require.NoError(t, err) + require.Equal(t, data, result) } func TestTLSConnection(t *testing.T) { - connCh := make(chan net.Conn) + connCh := make(chan SecuredConn) errCh := make(chan error) listener, err := testTCPServer(connCh, errCh) require.NoError(t, err) defer listener.Close() - tcpConn, err := net.Dial("tcp", listener.Addr().String()) + clientConn, err := tls.Dial("tcp", listener.Addr().String(), &tls.Config{InsecureSkipVerify: true}) require.NoError(t, err) - tlsConn := tls.Client(tcpConn, &tls.Config{InsecureSkipVerify: true}) - defer tlsConn.Close() - go tlsConn.Handshake() + defer clientConn.Close() - var conn SecuredConn + data := []byte("tls connection") + _, err = clientConn.Write(data) + require.NoError(t, err) + + var serverConn SecuredConn select { case newConn := <-connCh: - conn = newSecuredConn(newConn) + serverConn = newConn case newErr := <-errCh: err = newErr } require.NoError(t, err) - defer conn.Close() + defer serverConn.Close() - isTLS, err := conn.IsTLS() + isTLS, err := serverConn.IsTLS() require.NoError(t, err) require.True(t, isTLS) + + result := make([]byte, len(data)) + _, err = serverConn.Read(result) + require.NoError(t, err) + require.Equal(t, data, result) } From 1f1444d04f215695664540133ac557f11f2f1e1c Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 9 Jul 2024 13:53:11 +0000 Subject: [PATCH 24/47] Use tally.TestScope for metrics testing --- src/x/tls/config_manager_test.go | 37 +++++++++++--------------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/src/x/tls/config_manager_test.go b/src/x/tls/config_manager_test.go index 0971aab5ba..10c9015151 100644 --- a/src/x/tls/config_manager_test.go +++ b/src/x/tls/config_manager_test.go @@ -29,18 +29,12 @@ import ( "time" "github.com/m3db/m3/src/x/instrument" + "github.com/m3db/m3/src/x/tallytest" "github.com/stretchr/testify/require" + "github.com/uber-go/tally" "go.uber.org/zap" ) -type counterMock struct { - incCallback func() -} - -func (c *counterMock) Inc(_ int64) { - c.incCallback() -} - func appendCA(filename string, certPool *x509.CertPool) error { certs, err := os.ReadFile(filename) if err != nil { @@ -165,17 +159,12 @@ func TestLoadTLSConfig(t *testing.T) { func TestUpdateCertificate(t *testing.T) { var waitCh = make(chan bool) var waitCalledCh = make(chan bool) - var readConfigSuccess = 0 - successCounterMock := &counterMock{ - incCallback: func() { readConfigSuccess++ }, - } - var readConfigErrors = 0 - errorCounterMock := &counterMock{ - incCallback: func() { readConfigErrors++ }, - } + const successMetricName = "success" + const errorMetricName = "error" + testScope := tally.NewTestScope("", map[string]string{}) cmm := configManagerMetrics{ - getTLSConfigSuccess: successCounterMock, - getTLSConfigErrors: errorCounterMock, + getTLSConfigSuccess: testScope.Counter(successMetricName), + getTLSConfigErrors: testScope.Counter(errorMetricName), } originalSleepFn := sleepFn sleepFn = func(d time.Duration) { @@ -191,8 +180,8 @@ func TestUpdateCertificate(t *testing.T) { cm := cmInterface.(*configManager) cm.metrics = cmm require.Nil(t, cm.tlsConfig) - require.Equal(t, 0, readConfigSuccess) - require.Equal(t, 0, readConfigErrors) + tallytest.AssertCounterValue(t, 0, testScope.Snapshot(), successMetricName, map[string]string{}) + tallytest.AssertCounterValue(t, 0, testScope.Snapshot(), errorMetricName, map[string]string{}) opts = opts. SetCertificatesTTL(time.Second). @@ -207,13 +196,13 @@ func TestUpdateCertificate(t *testing.T) { cm.metrics = cmm <-waitCalledCh require.NotNil(t, cm.tlsConfig) - require.Equal(t, 1, readConfigSuccess) - require.Equal(t, 0, readConfigErrors) + tallytest.AssertCounterValue(t, 1, testScope.Snapshot(), successMetricName, map[string]string{}) + tallytest.AssertCounterValue(t, 0, testScope.Snapshot(), errorMetricName, map[string]string{}) opts = opts.SetCAFile("wrong/path") cm.options = opts waitCh <- true <-waitCalledCh - require.Equal(t, 1, readConfigSuccess) - require.Equal(t, 1, readConfigErrors) + tallytest.AssertCounterValue(t, 1, testScope.Snapshot(), successMetricName, map[string]string{}) + tallytest.AssertCounterValue(t, 1, testScope.Snapshot(), errorMetricName, map[string]string{}) } From 4e059482b4960b5151549416c399fa942766294a Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 9 Jul 2024 14:02:53 +0000 Subject: [PATCH 25/47] Use t.Cleanup to close the tls server --- src/aggregator/client/conn_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/aggregator/client/conn_test.go b/src/aggregator/client/conn_test.go index a9bfcff2db..8f18b078ea 100644 --- a/src/aggregator/client/conn_test.go +++ b/src/aggregator/client/conn_test.go @@ -427,6 +427,7 @@ func TestTLSConnectWriteToServer(t *testing.T) { ClientCAs: certPool, ClientAuth: tls.RequireAndVerifyClientCert, }) + t.Cleanup(func() { l.Close() }) require.NoError(t, err) serverAddr := l.Addr().String() @@ -470,8 +471,6 @@ func TestTLSConnectWriteToServer(t *testing.T) { require.Equal(t, 0, conn.numFailures) require.NotNil(t, conn.conn) - // Stop the server. - l.Close() // nolint: errcheck wg.Wait() // Close the connection From ec56885eca7cd5a47406668053740a1c650a6355 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 9 Jul 2024 14:15:23 +0000 Subject: [PATCH 26/47] Waiting for handler calls refactored --- src/x/server/server_test.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index 88536c7959..191781db75 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -40,6 +40,14 @@ const ( testListenAddress = "127.0.0.1:0" ) +func waitForHandler(h *mockHandler, numCalls int, numChecks int, waitTime time.Duration) { + checks := 0 + for h.called() < numCalls && checks < numChecks { + time.Sleep(waitTime) + checks++ + } +} + func testPlainTCPServer(addr string) (*server, *mockHandler, *int32, *int32) { return testServer(addr, xtls.Disabled, false) } @@ -104,6 +112,7 @@ func TestServerListenAndClose(t *testing.T) { for h.called() < numClients { time.Sleep(100 * time.Millisecond) } + waitForHandler(h, numClients, 5, 100*time.Millisecond) require.False(t, h.isClosed()) @@ -160,10 +169,7 @@ func TestTLSPermissiveServerListenAndClose(t *testing.T) { require.NoError(t, err) } - numChecks := 0 - for h.called() < numClients && numChecks < 5 { - time.Sleep(100 * time.Millisecond) - } + waitForHandler(h, numClients, 5, 100*time.Millisecond) require.False(t, h.isClosed()) @@ -205,11 +211,7 @@ func TestTLSEnforcedServerListenAndClose(t *testing.T) { require.NoError(t, err) } - numChecks := 0 - for h.called() < numClients/2 && numChecks < 5 { - time.Sleep(100 * time.Millisecond) - numChecks++ - } + waitForHandler(h, numClients/2, 5, 100*time.Millisecond) require.False(t, h.isClosed()) @@ -249,11 +251,7 @@ func TestMutualTLSServerListenAndClose(t *testing.T) { require.NoError(t, err) } - numChecks := 0 - for h.called() < numClients && numChecks < 5 { - time.Sleep(100 * time.Millisecond) - numChecks++ - } + waitForHandler(h, numClients, 5, 100*time.Millisecond) require.False(t, h.isClosed()) From f2594b20870cc80a0ced64dd88d02245fdeecfc8 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 9 Jul 2024 14:22:24 +0000 Subject: [PATCH 27/47] Early returns refactoring --- src/x/server/server.go | 15 ++++++++------- src/x/tls/config_manager.go | 17 +++++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/x/server/server.go b/src/x/server/server.go index de2521da92..ffb3aa6811 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -154,15 +154,16 @@ func (s *server) maybeUpgradeToTLS(conn SecuredConn) (SecuredConn, error) { if err != nil { return nil, err } - if isTLSConnection { - tlsConfig, err := s.tlsConfigManager.TLSConfig() - if err != nil { - return nil, err - } - conn = conn.UpgradeToTLS(tlsConfig) - } else if s.tlsConfigManager.ServerMode() == xtls.Enforced { + if !isTLSConnection && s.tlsConfigManager.ServerMode() == xtls.Enforced { return nil, fmt.Errorf("not a tls connection") + } else if !isTLSConnection { + return conn, nil + } + tlsConfig, err := s.tlsConfigManager.TLSConfig() + if err != nil { + return nil, err } + conn = conn.UpgradeToTLS(tlsConfig) return conn, nil } diff --git a/src/x/tls/config_manager.go b/src/x/tls/config_manager.go index 88a431d62b..023c9794ea 100644 --- a/src/x/tls/config_manager.go +++ b/src/x/tls/config_manager.go @@ -98,14 +98,15 @@ func (c *configManager) updateCertificates() { } func (c *configManager) loadCertPool() (*x509.CertPool, error) { - if c.options.CAFile() != "" { - certs, err := os.ReadFile(c.options.CAFile()) - if err != nil { - return nil, fmt.Errorf("read bundle error: %w", err) - } - if ok := c.certPool.AppendCertsFromPEM(certs); !ok { - return nil, fmt.Errorf("cannot append cert to cert pool") - } + if c.options.CAFile() == "" { + return c.certPool, nil + } + certs, err := os.ReadFile(c.options.CAFile()) + if err != nil { + return nil, fmt.Errorf("read bundle error: %w", err) + } + if ok := c.certPool.AppendCertsFromPEM(certs); !ok { + return nil, fmt.Errorf("cannot append cert to cert pool") } return c.certPool, nil } From d53144b430a5011ed250c59424581929dd07e9b2 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 9 Jul 2024 14:58:07 +0000 Subject: [PATCH 28/47] Server tests refactored into a table test --- src/x/server/server_test.go | 209 +++++++++++++++--------------------- 1 file changed, 88 insertions(+), 121 deletions(-) diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index 191781db75..6d76abe5d4 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -139,129 +139,96 @@ func TestServe(t *testing.T) { s.Close() } -func TestTLSPermissiveServerListenAndClose(t *testing.T) { - s, h, numAdded, numRemoved := testServer(testListenAddress, xtls.Permissive, false) - - var ( - numClients = 9 - expectedRes []string - ) - - err := s.ListenAndServe() - require.NoError(t, err) - listenAddr := s.listener.Addr().String() - - for i := 0; i < numClients; i++ { - var conn net.Conn - var err error - if i%2 == 0 { - conn, err = net.Dial("tcp", listenAddr) - } else { - conn, err = tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true}) - - } - require.NoError(t, err) - - msg := fmt.Sprintf("msg%d", i) - expectedRes = append(expectedRes, msg) - - _, err = conn.Write([]byte(msg)) - require.NoError(t, err) - } - - waitForHandler(h, numClients, 5, 100*time.Millisecond) - - require.False(t, h.isClosed()) - - s.Close() - - require.True(t, h.isClosed()) - require.Equal(t, int32(numClients), atomic.LoadInt32(numAdded)) - require.Equal(t, int32(numClients), atomic.LoadInt32(numRemoved)) - require.Equal(t, numClients, h.called()) - require.Equal(t, expectedRes, h.res()) -} - -func TestTLSEnforcedServerListenAndClose(t *testing.T) { - s, h, numAdded, numRemoved := testServer(testListenAddress, xtls.Enforced, false) - - var ( - numClients = 10 - expectedRes []string - ) - - err := s.ListenAndServe() - require.NoError(t, err) - listenAddr := s.listener.Addr().String() - - for i := 0; i < numClients; i++ { - var conn net.Conn - var err error - msg := fmt.Sprintf("msg%d", i) - - if i%2 == 0 { - conn, err = net.Dial("tcp", listenAddr) - } else { - conn, err = tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true}) - expectedRes = append(expectedRes, msg) - } - require.NoError(t, err) - - _, err = conn.Write([]byte(msg)) - require.NoError(t, err) +func TestTLS(t *testing.T) { + tests := []struct { + name string + tlsMode xtls.ServerMode + numClients int + expectedServerCalls int + dialFn func(i int, listenAddr string) (net.Conn, error) + appendExpectedResultFn func(expecteResult []string, i int, msg string) []string + }{ + { + name: "TLS permissive mode", + tlsMode: xtls.Permissive, + numClients: 9, + expectedServerCalls: 9, + dialFn: func(i int, listenAddr string) (net.Conn, error) { + if i%2 == 0 { + return net.Dial("tcp", listenAddr) + } else { + return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true}) + } + }, + appendExpectedResultFn: func(expectedResult []string, i int, msg string) []string { + return append(expectedResult, msg) + }, + }, + { + name: "TLS enforced mode", + tlsMode: xtls.Enforced, + numClients: 10, + expectedServerCalls: 5, + dialFn: func(i int, listenAddr string) (net.Conn, error) { + if i%2 == 0 { + return net.Dial("tcp", listenAddr) + } else { + return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true}) + } + }, + appendExpectedResultFn: func(expectedResult []string, i int, msg string) []string { + if i%2 == 1 { + return append(expectedResult, msg) + } else { + return expectedResult + } + }, + }, + { + name: "Mutual TLS", + tlsMode: xtls.Enforced, + numClients: 9, + expectedServerCalls: 9, + dialFn: func(i int, listenAddr string) (net.Conn, error) { + cert, err := tls.LoadX509KeyPair("./testdata/client.crt", "./testdata/client.key") + require.NoError(t, err) + return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true, Certificates: []tls.Certificate{cert}}) + }, + appendExpectedResultFn: func(expecteResult []string, i int, msg string) []string { + return append(expecteResult, msg) + }, + }, } - - waitForHandler(h, numClients/2, 5, 100*time.Millisecond) - - require.False(t, h.isClosed()) - - s.Close() - - require.True(t, h.isClosed()) - require.Equal(t, int32(numClients), atomic.LoadInt32(numAdded)) - require.Equal(t, int32(numClients), atomic.LoadInt32(numRemoved)) - require.Equal(t, numClients/2, h.called()) - require.Equal(t, expectedRes, h.res()) -} - -func TestMutualTLSServerListenAndClose(t *testing.T) { - s, h, numAdded, numRemoved := testServer(testListenAddress, xtls.Enforced, true) - - var ( - numClients = 9 - expectedRes []string - ) - - err := s.ListenAndServe() - require.NoError(t, err) - listenAddr := s.listener.Addr().String() - cert, err := tls.LoadX509KeyPair("./testdata/client.crt", "./testdata/client.key") - require.NoError(t, err) - - for i := 0; i < numClients; i++ { - var conn net.Conn - var err error - conn, err = tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true, Certificates: []tls.Certificate{cert}}) - require.NoError(t, err) - - msg := fmt.Sprintf("msg%d", i) - expectedRes = append(expectedRes, msg) - - _, err = conn.Write([]byte(msg)) - require.NoError(t, err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, h, numAdded, numRemoved := testServer(testListenAddress, tt.tlsMode, false) + var expectedRes []string + err := s.ListenAndServe() + require.NoError(t, err) + listenAddr := s.listener.Addr().String() + for i := 0; i < tt.numClients; i++ { + conn, err := tt.dialFn(i, listenAddr) + require.NoError(t, err) + + msg := fmt.Sprintf("msg%d", i) + expectedRes = tt.appendExpectedResultFn(expectedRes, i, msg) + + _, err = conn.Write([]byte(msg)) + require.NoError(t, err) + } + waitForHandler(h, tt.expectedServerCalls, 5, 100*time.Millisecond) + + require.False(t, h.isClosed()) + + s.Close() + + require.True(t, h.isClosed()) + require.Equal(t, int32(tt.numClients), atomic.LoadInt32(numAdded)) + require.Equal(t, int32(tt.numClients), atomic.LoadInt32(numRemoved)) + require.Equal(t, tt.expectedServerCalls, h.called()) + require.Equal(t, expectedRes, h.res()) + }) } - - waitForHandler(h, numClients, 5, 100*time.Millisecond) - - require.False(t, h.isClosed()) - - s.Close() - - require.True(t, h.isClosed()) - require.Equal(t, int32(numClients), atomic.LoadInt32(numAdded)) - require.Equal(t, int32(numClients), atomic.LoadInt32(numRemoved)) - require.Equal(t, numClients, h.called()) - require.Equal(t, expectedRes, h.res()) } type mockHandler struct { From 858208142ab47cc30548504cbc2d534816f70b4d Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Thu, 11 Jul 2024 11:51:02 +0000 Subject: [PATCH 29/47] bufio.Reader replaced with a peek function --- src/x/server/secured_conn.go | 72 ++++++++++++++++++------------- src/x/server/secured_conn_test.go | 16 +++---- src/x/server/server.go | 4 +- 3 files changed, 51 insertions(+), 41 deletions(-) diff --git a/src/x/server/secured_conn.go b/src/x/server/secured_conn.go index 81e2b4c92b..c7ff5e22ea 100644 --- a/src/x/server/secured_conn.go +++ b/src/x/server/secured_conn.go @@ -22,77 +22,87 @@ package server import ( - "bufio" "crypto/tls" "net" + "sync" ) // TLSHandshakeFirstByte is the first byte of a tls connection handshake const TLSHandshakeFirstByte = 0x16 -func newSecuredConn(conn net.Conn) SecuredConn { +func newSecuredConn(conn net.Conn) *securedConn { return &securedConn{ - r: bufio.NewReader(conn), - Conn: conn, - isTLS: nil, + Conn: conn, } } -// SecuredConn represents the secured connection -type SecuredConn interface { - net.Conn - IsTLS() (bool, error) - GetConn() net.Conn - UpgradeToTLS(*tls.Config) SecuredConn -} - type securedConn struct { net.Conn - r *bufio.Reader - isTLS *bool + mu sync.Mutex + isTLS *bool + peekedByte *byte } // IsTLS returns is the connection is TLS or not. // It peeks at the first byte and checks // if it is equal to the TLS handshake first byte // https://www.rfc-editor.org/rfc/rfc5246#appendix-A.1 -func (b *securedConn) IsTLS() (bool, error) { - if b.isTLS != nil { - return *b.isTLS, nil +func (s *securedConn) IsTLS() (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.isTLS != nil { + return *s.isTLS, nil } - connBytes, err := b.r.Peek(1) + firstByte, err := s.peek() if err != nil { return false, err } - isTLS := len(connBytes) > 0 && connBytes[0] == TLSHandshakeFirstByte - b.isTLS = &isTLS + isTLS := firstByte == TLSHandshakeFirstByte + s.isTLS = &isTLS + return isTLS, nil } -func (b *securedConn) UpgradeToTLS(tlsConfig *tls.Config) SecuredConn { - tlsConn := tls.Server(b, tlsConfig) +func (s *securedConn) UpgradeToTLS(tlsConfig *tls.Config) *securedConn { + tlsConn := tls.Server(s, tlsConfig) t := true return &securedConn{ - r: bufio.NewReader(tlsConn), Conn: tlsConn, isTLS: &t, } } // Read reads n bytes -func (b *securedConn) Read(n []byte) (int, error) { +func (s *securedConn) Read(n []byte) (int, error) { // Before reading we need to ensure if we know the type of the connection. // After reading data it will be impossible to determine // if the connection has the TLS layer or not. - if b.isTLS == nil { - if _, err := b.IsTLS(); err != nil { + if s.isTLS == nil { + if _, err := s.IsTLS(); err != nil { return 0, err } } - return b.r.Read(n) + s.mu.Lock() + defer s.mu.Unlock() + if s.peekedByte != nil { + n[0] = *s.peekedByte + s.peekedByte = nil + n, err := s.Conn.Read(n[1:]) + return n + 1, err + } + return s.Conn.Read(n) } -// GetConn returns net.Conn connection -func (b *securedConn) GetConn() net.Conn { - return b.Conn +func (s *securedConn) peek() (byte, error) { + if s.peekedByte != nil { + return *s.peekedByte, nil + } + + var buf [1]byte + _, err := s.Conn.Read(buf[:]) + if err != nil { + return 0, err + } + s.peekedByte = &buf[0] + return buf[0], nil } diff --git a/src/x/server/secured_conn_test.go b/src/x/server/secured_conn_test.go index 9c1109703d..1ace37b30d 100644 --- a/src/x/server/secured_conn_test.go +++ b/src/x/server/secured_conn_test.go @@ -29,12 +29,12 @@ import ( "github.com/stretchr/testify/require" ) -func testTCPServer(connCh chan SecuredConn, errCh chan error) (net.Listener, error) { +func testTCPServer(connCh chan *securedConn, errCh chan error) (net.Listener, error) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { return nil, err } - go func(net.Listener, chan SecuredConn, chan error) { + go func(net.Listener, chan *securedConn, chan error) { conn, err := listener.Accept() if err != nil { errCh <- err @@ -53,7 +53,7 @@ func testTCPServer(connCh chan SecuredConn, errCh chan error) (net.Listener, err } tlsConfig := tls.Config{Certificates: []tls.Certificate{certs}} securedConn = securedConn.UpgradeToTLS(&tlsConfig) - tlsConn := securedConn.GetConn().(*tls.Conn) + tlsConn := securedConn.Conn.(*tls.Conn) tlsConn.Handshake() } connCh <- securedConn @@ -63,7 +63,7 @@ func testTCPServer(connCh chan SecuredConn, errCh chan error) (net.Listener, err } func TestPlainTCPConnection(t *testing.T) { - connCh := make(chan SecuredConn) + connCh := make(chan *securedConn) errCh := make(chan error) listener, err := testTCPServer(connCh, errCh) require.NoError(t, err) @@ -75,10 +75,10 @@ func TestPlainTCPConnection(t *testing.T) { _, err = clientConn.Write(data) require.NoError(t, err) - var conn SecuredConn + var conn *securedConn select { case newConn := <-connCh: - conn = newSecuredConn(newConn) + conn = newConn case newErr := <-errCh: err = newErr } @@ -95,7 +95,7 @@ func TestPlainTCPConnection(t *testing.T) { } func TestTLSConnection(t *testing.T) { - connCh := make(chan SecuredConn) + connCh := make(chan *securedConn) errCh := make(chan error) listener, err := testTCPServer(connCh, errCh) require.NoError(t, err) @@ -109,7 +109,7 @@ func TestTLSConnection(t *testing.T) { _, err = clientConn.Write(data) require.NoError(t, err) - var serverConn SecuredConn + var serverConn *securedConn select { case newConn := <-connCh: serverConn = newConn diff --git a/src/x/server/server.go b/src/x/server/server.go index ffb3aa6811..99f9c61b0b 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -146,7 +146,7 @@ func (s *server) Serve(l net.Listener) error { return nil } -func (s *server) maybeUpgradeToTLS(conn SecuredConn) (SecuredConn, error) { +func (s *server) maybeUpgradeToTLS(conn *securedConn) (*securedConn, error) { if s.tlsConfigManager.ServerMode() == xtls.Disabled { return conn, nil } @@ -171,7 +171,7 @@ func (s *server) serve() { connCh, errCh := xnet.StartForeverAcceptLoop(s.listener, s.retryOpts) for conn := range connCh { conn := newSecuredConn(conn) - if tcpConn, ok := conn.GetConn().(*net.TCPConn); ok { + if tcpConn, ok := conn.Conn.(*net.TCPConn); ok { tcpConn.SetKeepAlive(s.tcpConnectionKeepAlive) if s.tcpConnectionKeepAlivePeriod != 0 { tcpConn.SetKeepAlivePeriod(s.tcpConnectionKeepAlivePeriod) From 89e7e36ced46140de144b7c59473d43f7b0c73e2 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Fri, 12 Jul 2024 12:30:43 +0000 Subject: [PATCH 30/47] Server benchmark --- src/x/server/server_benchmark_test.go | 152 ++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 src/x/server/server_benchmark_test.go diff --git a/src/x/server/server_benchmark_test.go b/src/x/server/server_benchmark_test.go new file mode 100644 index 0000000000..91ba75fad7 --- /dev/null +++ b/src/x/server/server_benchmark_test.go @@ -0,0 +1,152 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package server + +import ( + "bufio" + "crypto/tls" + "fmt" + "net" + "testing" + "time" + + "github.com/m3db/m3/src/x/retry" + xtls "github.com/m3db/m3/src/x/tls" + "github.com/stretchr/testify/require" +) + +const ( + testBenchListenAddress = "127.0.0.1:0" +) + +func newMockKeepAliveHandler() *mockKeepAliveHandler { return &mockKeepAliveHandler{} } + +type mockKeepAliveHandler struct { + mockHandler +} + +func (h *mockKeepAliveHandler) Handle(conn net.Conn) { + defer conn.Close() + + reader := bufio.NewReader(conn) + + for { + data, err := reader.ReadString('\n') + if err != nil { + break + } + _, err = conn.Write([]byte(data)) + if err != nil { + break + } + } +} + +// nolint: unparam +func testBenchServer(addr string, h Handler, tlsMode xtls.ServerMode, mTLSEnabled bool) *server { + tlsOpts := xtls.NewOptions(). + SetServerMode(tlsMode). + SetMutualTLSEnabled(mTLSEnabled). + SetCAFile("./testdata/rootCA.crt"). + SetCertFile("./testdata/server.crt"). + SetKeyFile("./testdata/server.key") + + opts := NewOptions().SetRetryOptions(retry.NewOptions().SetMaxRetries(2)) + opts = opts.SetInstrumentOptions(opts.InstrumentOptions().SetReportInterval(time.Second)).SetTLSOptions(tlsOpts) + + s := NewServer(addr, h, opts).(*server) + + s.addConnectionFn = func(conn net.Conn) bool { + ret := s.addConnection(conn) + return ret + } + + s.removeConnectionFn = func(conn net.Conn) { + s.removeConnection(conn) + } + + return s +} + +func dial(listenAddr string, tlsMode xtls.ServerMode, certs []tls.Certificate, b *testing.B) (net.Conn, error) { + if tlsMode == xtls.Disabled { + return net.Dial("tcp", listenAddr) + } + return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true, Certificates: certs}) +} + +func benchmarkServer(tlsMode xtls.ServerMode, mTLSEnabled bool, b *testing.B) { + handler := newMockHandler() + server := testBenchServer(testBenchListenAddress, handler, tlsMode, mTLSEnabled) + b.Cleanup(func() { server.Close(); handler.Close() }) + err := server.ListenAndServe() + require.NoError(b, err) + cert, err := tls.LoadX509KeyPair("./testdata/client.crt", "./testdata/client.key") + require.NoError(b, err) + for n := 0; n < b.N; n++ { + conn, err := dial(server.listener.Addr().String(), tlsMode, []tls.Certificate{cert}, b) + require.NoError(b, err) + msg := fmt.Sprintf("msg%d", n) + conn.Write([]byte(msg)) + } + waitForHandler(handler, b.N, 5, 100*time.Millisecond) + require.Equal(b, b.N, handler.called()) +} + +func BenchmarkPlainTCPServer(b *testing.B) { + benchmarkServer(xtls.Disabled, false, b) +} + +func BenchmarkTLSServer(b *testing.B) { + benchmarkServer(xtls.Enforced, false, b) +} + +func BenchmarkMTLSServer(b *testing.B) { + benchmarkServer(xtls.Enforced, true, b) +} + +func benchmarkKeepAliveServer(tlsMode xtls.ServerMode, mTLSEnabled bool, b *testing.B) { + handler := newMockKeepAliveHandler() + server := testBenchServer(testBenchListenAddress, handler, tlsMode, mTLSEnabled) + b.Cleanup(func() { server.Close(); handler.Close() }) + err := server.ListenAndServe() + require.NoError(b, err) + cert, err := tls.LoadX509KeyPair("./testdata/client.crt", "./testdata/client.key") + require.NoError(b, err) + conn, err := dial(server.listener.Addr().String(), tlsMode, []tls.Certificate{cert}, b) + require.NoError(b, err) + for n := 0; n < b.N; n++ { + msg := fmt.Sprintf("msg%d", n) + conn.Write([]byte(msg)) + } +} + +func BenchmarkKeepAlivePlainTCPServer(b *testing.B) { + benchmarkKeepAliveServer(xtls.Disabled, false, b) +} + +func BenchmarkKeepAliveTLSServer(b *testing.B) { + benchmarkKeepAliveServer(xtls.Enforced, false, b) +} + +func BenchmarkKeepAliveMTLSServer(b *testing.B) { + benchmarkKeepAliveServer(xtls.Enforced, true, b) +} From 161a54782e261381ccd9f2345aac143ac09ae9cd Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Fri, 12 Jul 2024 14:39:39 +0000 Subject: [PATCH 31/47] Tests for KeepAlive --- src/x/server/server_benchmark_test.go | 2 +- src/x/server/server_test.go | 41 +++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/x/server/server_benchmark_test.go b/src/x/server/server_benchmark_test.go index 91ba75fad7..4a1d8273b6 100644 --- a/src/x/server/server_benchmark_test.go +++ b/src/x/server/server_benchmark_test.go @@ -107,7 +107,7 @@ func benchmarkServer(tlsMode xtls.ServerMode, mTLSEnabled bool, b *testing.B) { msg := fmt.Sprintf("msg%d", n) conn.Write([]byte(msg)) } - waitForHandler(handler, b.N, 5, 100*time.Millisecond) + waitFor(func() bool { return handler.called() == b.N }, 5, 100*time.Millisecond) require.Equal(b, b.N, handler.called()) } diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index 6d76abe5d4..79fa37a4fb 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -27,6 +27,7 @@ import ( "sort" "sync" "sync/atomic" + "syscall" "testing" "time" @@ -40,9 +41,32 @@ const ( testListenAddress = "127.0.0.1:0" ) -func waitForHandler(h *mockHandler, numCalls int, numChecks int, waitTime time.Duration) { +func isKeepAlive(conn net.Conn) (bool, error) { + tcpConn, ok := conn.(*net.TCPConn) + if !ok { + return false, nil + } + file, err := tcpConn.File() + if err != nil { + return false, err + } + defer file.Close() + + fd := int(file.Fd()) + + value, err := syscall.GetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_KEEPALIVE) + if err != nil { + return false, err + } + if value == 1 { + return true, nil + } + return false, nil +} + +func waitFor(checkFn func() bool, numChecks int, waitTime time.Duration) { checks := 0 - for h.called() < numCalls && checks < numChecks { + for !checkFn() && checks < numChecks { time.Sleep(waitTime) checks++ } @@ -66,7 +90,10 @@ func testServer(addr string, tlsMode xtls.ServerMode, mTLSEnabled bool) (*server SetCertFile("./testdata/server.crt"). SetKeyFile("./testdata/server.key") - opts := NewOptions().SetRetryOptions(retry.NewOptions().SetMaxRetries(2)) + opts := NewOptions(). + SetRetryOptions(retry.NewOptions().SetMaxRetries(2)). + SetTCPConnectionKeepAlive(true). + SetTCPConnectionKeepAlivePeriod(time.Hour) opts = opts.SetInstrumentOptions(opts.InstrumentOptions().SetReportInterval(time.Second)).SetTLSOptions(tlsOpts) h := newMockHandler() @@ -112,7 +139,7 @@ func TestServerListenAndClose(t *testing.T) { for h.called() < numClients { time.Sleep(100 * time.Millisecond) } - waitForHandler(h, numClients, 5, 100*time.Millisecond) + waitFor(func() bool { return h.called() == numClients }, 5, 100*time.Millisecond) require.False(t, h.isClosed()) @@ -209,6 +236,9 @@ func TestTLS(t *testing.T) { for i := 0; i < tt.numClients; i++ { conn, err := tt.dialFn(i, listenAddr) require.NoError(t, err) + waitFor(func() bool { return len(s.conns) == 1 }, 5, 100*time.Millisecond) + keepAlive, err := isKeepAlive(s.conns[0].(*securedConn).Conn) + require.True(t, keepAlive) msg := fmt.Sprintf("msg%d", i) expectedRes = tt.appendExpectedResultFn(expectedRes, i, msg) @@ -216,8 +246,7 @@ func TestTLS(t *testing.T) { _, err = conn.Write([]byte(msg)) require.NoError(t, err) } - waitForHandler(h, tt.expectedServerCalls, 5, 100*time.Millisecond) - + waitFor(func() bool { return h.called() == tt.expectedServerCalls }, 5, 100*time.Millisecond) require.False(t, h.isClosed()) s.Close() From c4fb2d2e41e911ae32c5e2eb37bceb639ed71789 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Fri, 12 Jul 2024 15:53:29 +0000 Subject: [PATCH 32/47] peekedByte pointer replaced with byte and peekedByteIsSet. Mutex removed from securedconn --- src/x/server/secured_conn.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/x/server/secured_conn.go b/src/x/server/secured_conn.go index c7ff5e22ea..a975a59a78 100644 --- a/src/x/server/secured_conn.go +++ b/src/x/server/secured_conn.go @@ -24,7 +24,6 @@ package server import ( "crypto/tls" "net" - "sync" ) // TLSHandshakeFirstByte is the first byte of a tls connection handshake @@ -38,9 +37,9 @@ func newSecuredConn(conn net.Conn) *securedConn { type securedConn struct { net.Conn - mu sync.Mutex - isTLS *bool - peekedByte *byte + isTLS *bool + peekedByte byte + peekedByteIsSet bool } // IsTLS returns is the connection is TLS or not. @@ -48,8 +47,6 @@ type securedConn struct { // if it is equal to the TLS handshake first byte // https://www.rfc-editor.org/rfc/rfc5246#appendix-A.1 func (s *securedConn) IsTLS() (bool, error) { - s.mu.Lock() - defer s.mu.Unlock() if s.isTLS != nil { return *s.isTLS, nil } @@ -82,11 +79,9 @@ func (s *securedConn) Read(n []byte) (int, error) { return 0, err } } - s.mu.Lock() - defer s.mu.Unlock() - if s.peekedByte != nil { - n[0] = *s.peekedByte - s.peekedByte = nil + if s.peekedByteIsSet && len(n) > 0 { + n[0] = s.peekedByte + s.peekedByteIsSet = false n, err := s.Conn.Read(n[1:]) return n + 1, err } @@ -94,8 +89,8 @@ func (s *securedConn) Read(n []byte) (int, error) { } func (s *securedConn) peek() (byte, error) { - if s.peekedByte != nil { - return *s.peekedByte, nil + if s.peekedByteIsSet { + return s.peekedByte, nil } var buf [1]byte @@ -103,6 +98,7 @@ func (s *securedConn) peek() (byte, error) { if err != nil { return 0, err } - s.peekedByte = &buf[0] + s.peekedByte = buf[0] + s.peekedByteIsSet = true return buf[0], nil } From 96aa72c2f705835ad741434b02586dc9df4a9a27 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Thu, 25 Jul 2024 10:51:28 +0000 Subject: [PATCH 33/47] ServerMode methods generated with enumer --- src/x/tls/mode.go | 20 +------- src/x/tls/servermode_enumer.go | 94 ++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 18 deletions(-) create mode 100644 src/x/tls/servermode_enumer.go diff --git a/src/x/tls/mode.go b/src/x/tls/mode.go index b77597dd7e..9cf7d113a7 100644 --- a/src/x/tls/mode.go +++ b/src/x/tls/mode.go @@ -20,7 +20,8 @@ package tls -import "fmt" +// ServerMode represents the TLS mode +type ServerMode uint16 const ( // Disabled allows plaintext connections only @@ -30,20 +31,3 @@ const ( // Enforced allows TLS connections only Enforced ) - -// ServerMode represents the TLS mode -type ServerMode uint16 - -func (t *ServerMode) UnmarshalText(mode []byte) error { - switch string(mode) { - case "disabled": - *t = Disabled - case "permissive": - *t = Permissive - case "enforced": - *t = Enforced - default: - return fmt.Errorf("unknown tls mode: %s", mode) - } - return nil -} diff --git a/src/x/tls/servermode_enumer.go b/src/x/tls/servermode_enumer.go new file mode 100644 index 0000000000..64ac5918b5 --- /dev/null +++ b/src/x/tls/servermode_enumer.go @@ -0,0 +1,94 @@ +// Code generated by "enumer -type ServerMode -text"; DO NOT EDIT. + +package tls + +import ( + "fmt" + "strings" +) + +const _ServerModeName = "DisabledPermissiveEnforced" + +var _ServerModeIndex = [...]uint8{0, 8, 18, 26} + +const _ServerModeLowerName = "disabledpermissiveenforced" + +func (i ServerMode) String() string { + if i >= ServerMode(len(_ServerModeIndex)-1) { + return fmt.Sprintf("ServerMode(%d)", i) + } + return _ServerModeName[_ServerModeIndex[i]:_ServerModeIndex[i+1]] +} + +// An "invalid array index" compiler error signifies that the constant values have changed. +// Re-run the stringer command to generate them again. +func _ServerModeNoOp() { + var x [1]struct{} + _ = x[Disabled-(0)] + _ = x[Permissive-(1)] + _ = x[Enforced-(2)] +} + +var _ServerModeValues = []ServerMode{Disabled, Permissive, Enforced} + +var _ServerModeNameToValueMap = map[string]ServerMode{ + _ServerModeName[0:8]: Disabled, + _ServerModeLowerName[0:8]: Disabled, + _ServerModeName[8:18]: Permissive, + _ServerModeLowerName[8:18]: Permissive, + _ServerModeName[18:26]: Enforced, + _ServerModeLowerName[18:26]: Enforced, +} + +var _ServerModeNames = []string{ + _ServerModeName[0:8], + _ServerModeName[8:18], + _ServerModeName[18:26], +} + +// ServerModeString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func ServerModeString(s string) (ServerMode, error) { + if val, ok := _ServerModeNameToValueMap[s]; ok { + return val, nil + } + + if val, ok := _ServerModeNameToValueMap[strings.ToLower(s)]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to ServerMode values", s) +} + +// ServerModeValues returns all values of the enum +func ServerModeValues() []ServerMode { + return _ServerModeValues +} + +// ServerModeStrings returns a slice of all String values of the enum +func ServerModeStrings() []string { + strs := make([]string, len(_ServerModeNames)) + copy(strs, _ServerModeNames) + return strs +} + +// IsAServerMode returns "true" if the value is listed in the enum definition. "false" otherwise +func (i ServerMode) IsAServerMode() bool { + for _, v := range _ServerModeValues { + if i == v { + return true + } + } + return false +} + +// MarshalText implements the encoding.TextMarshaler interface for ServerMode +func (i ServerMode) MarshalText() ([]byte, error) { + return []byte(i.String()), nil +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface for ServerMode +func (i *ServerMode) UnmarshalText(text []byte) error { + var err error + *i, err = ServerModeString(string(text)) + return err +} From 6a44f2e4abca03ba90620556a4f948e2de037371 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 30 Jul 2024 12:29:24 +0000 Subject: [PATCH 34/47] Linter fixed --- src/aggregator/client/conn.go | 2 +- src/aggregator/client/conn_test.go | 7 +++++-- src/x/server/secured_conn_test.go | 18 ++++++++++-------- src/x/server/server.go | 2 +- src/x/server/server_benchmark_test.go | 10 ++++++---- src/x/server/server_test.go | 14 ++++++-------- src/x/tls/config_manager_test.go | 12 ++++++------ 7 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/aggregator/client/conn.go b/src/aggregator/client/conn.go index 6873fcab13..1018aef546 100644 --- a/src/aggregator/client/conn.go +++ b/src/aggregator/client/conn.go @@ -208,7 +208,7 @@ func (c *connection) connectWithLock() error { securedConn, err := c.upgradeToTLS(conn) if err != nil { c.metrics.connectError.Inc(1) - conn.Close() + conn.Close() // nolint: errcheck return err } conn = securedConn diff --git a/src/aggregator/client/conn_test.go b/src/aggregator/client/conn_test.go index 8f18b078ea..a18debc588 100644 --- a/src/aggregator/client/conn_test.go +++ b/src/aggregator/client/conn_test.go @@ -426,8 +426,9 @@ func TestTLSConnectWriteToServer(t *testing.T) { Certificates: []tls.Certificate{serverCert}, ClientCAs: certPool, ClientAuth: tls.RequireAndVerifyClientCert, + MinVersion: tls.VersionTLS13, }) - t.Cleanup(func() { l.Close() }) + t.Cleanup(func() { l.Close() }) // nolint: errcheck require.NoError(t, err) serverAddr := l.Addr().String() @@ -436,9 +437,10 @@ func TestTLSConnectWriteToServer(t *testing.T) { // Ignore the first testing connection. conn, err := l.Accept() + require.NoError(t, err) tlsConn, ok := conn.(*tls.Conn) require.True(t, ok) - tlsConn.Handshake() + err = tlsConn.Handshake() require.NoError(t, err) require.NoError(t, conn.Close()) @@ -456,6 +458,7 @@ func TestTLSConnectWriteToServer(t *testing.T) { require.NoError(t, err) // Wait until the server starts up. dialer := net.Dialer{Timeout: time.Minute} + // #nosec G402 testConn, err := tls.DialWithDialer(&dialer, tcpProtocol, serverAddr, &tls.Config{ InsecureSkipVerify: true, Certificates: []tls.Certificate{clientCert}, diff --git a/src/x/server/secured_conn_test.go b/src/x/server/secured_conn_test.go index 1ace37b30d..c4e0fd5dd4 100644 --- a/src/x/server/secured_conn_test.go +++ b/src/x/server/secured_conn_test.go @@ -51,10 +51,12 @@ func testTCPServer(connCh chan *securedConn, errCh chan error) (net.Listener, er errCh <- err return } - tlsConfig := tls.Config{Certificates: []tls.Certificate{certs}} + tlsConfig := tls.Config{Certificates: []tls.Certificate{certs}, MinVersion: tls.VersionTLS13} securedConn = securedConn.UpgradeToTLS(&tlsConfig) tlsConn := securedConn.Conn.(*tls.Conn) - tlsConn.Handshake() + if err = tlsConn.Handshake(); err != nil { + errCh <- err + } } connCh <- securedConn } @@ -67,7 +69,7 @@ func TestPlainTCPConnection(t *testing.T) { errCh := make(chan error) listener, err := testTCPServer(connCh, errCh) require.NoError(t, err) - defer listener.Close() + defer listener.Close() // nolint: errcheck clientConn, err := net.Dial("tcp", listener.Addr().String()) require.NoError(t, err) @@ -83,7 +85,7 @@ func TestPlainTCPConnection(t *testing.T) { err = newErr } require.NoError(t, err) - defer conn.Close() + defer conn.Close() // nolint: errcheck isTLS, err := conn.IsTLS() require.NoError(t, err) @@ -99,11 +101,11 @@ func TestTLSConnection(t *testing.T) { errCh := make(chan error) listener, err := testTCPServer(connCh, errCh) require.NoError(t, err) - defer listener.Close() + defer listener.Close() // nolint: errcheck - clientConn, err := tls.Dial("tcp", listener.Addr().String(), &tls.Config{InsecureSkipVerify: true}) + clientConn, err := tls.Dial("tcp", listener.Addr().String(), &tls.Config{InsecureSkipVerify: true}) // #nosec G402 require.NoError(t, err) - defer clientConn.Close() + defer clientConn.Close() // nolint: errcheck data := []byte("tls connection") _, err = clientConn.Write(data) @@ -117,7 +119,7 @@ func TestTLSConnection(t *testing.T) { err = newErr } require.NoError(t, err) - defer serverConn.Close() + defer serverConn.Close() // nolint: errcheck isTLS, err := serverConn.IsTLS() require.NoError(t, err) diff --git a/src/x/server/server.go b/src/x/server/server.go index 99f9c61b0b..6e61104cab 100644 --- a/src/x/server/server.go +++ b/src/x/server/server.go @@ -182,7 +182,7 @@ func (s *server) serve() { } else { s.wgConns.Add(1) go func() { - defer conn.Close() + defer conn.Close() // nolint: errcheck defer s.removeConnectionFn(conn) defer s.wgConns.Done() diff --git a/src/x/server/server_benchmark_test.go b/src/x/server/server_benchmark_test.go index 4a1d8273b6..d8342ebfb9 100644 --- a/src/x/server/server_benchmark_test.go +++ b/src/x/server/server_benchmark_test.go @@ -44,7 +44,7 @@ type mockKeepAliveHandler struct { } func (h *mockKeepAliveHandler) Handle(conn net.Conn) { - defer conn.Close() + defer conn.Close() // nolint: errcheck reader := bufio.NewReader(conn) @@ -90,7 +90,7 @@ func dial(listenAddr string, tlsMode xtls.ServerMode, certs []tls.Certificate, b if tlsMode == xtls.Disabled { return net.Dial("tcp", listenAddr) } - return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true, Certificates: certs}) + return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true, Certificates: certs}) // #nosec G402 } func benchmarkServer(tlsMode xtls.ServerMode, mTLSEnabled bool, b *testing.B) { @@ -105,7 +105,8 @@ func benchmarkServer(tlsMode xtls.ServerMode, mTLSEnabled bool, b *testing.B) { conn, err := dial(server.listener.Addr().String(), tlsMode, []tls.Certificate{cert}, b) require.NoError(b, err) msg := fmt.Sprintf("msg%d", n) - conn.Write([]byte(msg)) + _, err = conn.Write([]byte(msg)) + require.NoError(b, err) } waitFor(func() bool { return handler.called() == b.N }, 5, 100*time.Millisecond) require.Equal(b, b.N, handler.called()) @@ -135,7 +136,8 @@ func benchmarkKeepAliveServer(tlsMode xtls.ServerMode, mTLSEnabled bool, b *test require.NoError(b, err) for n := 0; n < b.N; n++ { msg := fmt.Sprintf("msg%d", n) - conn.Write([]byte(msg)) + _, err = conn.Write([]byte(msg)) + require.NoError(b, err) } } diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index 79fa37a4fb..1211b8646e 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -50,7 +50,7 @@ func isKeepAlive(conn net.Conn) (bool, error) { if err != nil { return false, err } - defer file.Close() + defer file.Close() // nolint: errcheck fd := int(file.Fd()) @@ -183,9 +183,8 @@ func TestTLS(t *testing.T) { dialFn: func(i int, listenAddr string) (net.Conn, error) { if i%2 == 0 { return net.Dial("tcp", listenAddr) - } else { - return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true}) } + return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true}) // #nosec G402 }, appendExpectedResultFn: func(expectedResult []string, i int, msg string) []string { return append(expectedResult, msg) @@ -199,16 +198,14 @@ func TestTLS(t *testing.T) { dialFn: func(i int, listenAddr string) (net.Conn, error) { if i%2 == 0 { return net.Dial("tcp", listenAddr) - } else { - return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true}) } + return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true}) // #nosec G402 }, appendExpectedResultFn: func(expectedResult []string, i int, msg string) []string { if i%2 == 1 { return append(expectedResult, msg) - } else { - return expectedResult } + return expectedResult }, }, { @@ -219,7 +216,7 @@ func TestTLS(t *testing.T) { dialFn: func(i int, listenAddr string) (net.Conn, error) { cert, err := tls.LoadX509KeyPair("./testdata/client.crt", "./testdata/client.key") require.NoError(t, err) - return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true, Certificates: []tls.Certificate{cert}}) + return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true, Certificates: []tls.Certificate{cert}}) // #nosec G402 }, appendExpectedResultFn: func(expecteResult []string, i int, msg string) []string { return append(expecteResult, msg) @@ -239,6 +236,7 @@ func TestTLS(t *testing.T) { waitFor(func() bool { return len(s.conns) == 1 }, 5, 100*time.Millisecond) keepAlive, err := isKeepAlive(s.conns[0].(*securedConn).Conn) require.True(t, keepAlive) + require.NoError(t, err) msg := fmt.Sprintf("msg%d", i) expectedRes = tt.appendExpectedResultFn(expectedRes, i, msg) diff --git a/src/x/tls/config_manager_test.go b/src/x/tls/config_manager_test.go index 10c9015151..e0ca268a5b 100644 --- a/src/x/tls/config_manager_test.go +++ b/src/x/tls/config_manager_test.go @@ -58,7 +58,7 @@ func TestLoadCertPool(t *testing.T) { cm.options = opts certPool, err := cm.loadCertPool() require.NoError(t, err) - require.True(t, expectedCertPool.Equal(certPool)) + require.Equal(t, expectedCertPool.Subjects(), certPool.Subjects()) opts = opts.SetCAFile("testdata/1.crt") cm.options = opts @@ -66,8 +66,8 @@ func TestLoadCertPool(t *testing.T) { require.NoError(t, err) err = appendCA("testdata/1.crt", expectedCertPool) require.NoError(t, err) - require.True(t, expectedCertPool.Equal(certPool)) - require.True(t, expectedCertPool.Equal(cm.certPool)) + require.Equal(t, expectedCertPool.Subjects(), certPool.Subjects()) + require.Equal(t, expectedCertPool.Subjects(), cm.certPool.Subjects()) opts = opts.SetCAFile("testdata/2.crt") cm.options = opts @@ -75,14 +75,14 @@ func TestLoadCertPool(t *testing.T) { require.NoError(t, err) err = appendCA("testdata/2.crt", expectedCertPool) require.NoError(t, err) - require.True(t, expectedCertPool.Equal(certPool)) - require.True(t, expectedCertPool.Equal(cm.certPool)) + require.Equal(t, expectedCertPool.Subjects(), certPool.Subjects()) + require.Equal(t, expectedCertPool.Subjects(), cm.certPool.Subjects()) opts = opts.SetCAFile("testdata/3.crt") cm.options = opts certPool, err = cm.loadCertPool() require.Error(t, err) - require.True(t, expectedCertPool.Equal(cm.certPool)) + require.Equal(t, expectedCertPool.Subjects(), cm.certPool.Subjects()) opts = opts.SetCAFile("wrong/path") cm.options = opts From e848c9783dee42efa7614abd8da060bfc4dfba4a Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 30 Jul 2024 12:54:01 +0000 Subject: [PATCH 35/47] Linter fixed --- src/x/server/server_benchmark_test.go | 1 + src/x/server/server_test.go | 9 +++++++-- src/x/tls/config_manager.go | 4 ++++ src/x/tls/config_manager_test.go | 12 +++++++----- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/x/server/server_benchmark_test.go b/src/x/server/server_benchmark_test.go index d8342ebfb9..212b17c0aa 100644 --- a/src/x/server/server_benchmark_test.go +++ b/src/x/server/server_benchmark_test.go @@ -30,6 +30,7 @@ import ( "github.com/m3db/m3/src/x/retry" xtls "github.com/m3db/m3/src/x/tls" + "github.com/stretchr/testify/require" ) diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index 1211b8646e..b8b873170f 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -50,7 +50,7 @@ func isKeepAlive(conn net.Conn) (bool, error) { if err != nil { return false, err } - defer file.Close() // nolint: errcheck + defer file.Close() // nolint: errcheck,gosec fd := int(file.Fd()) @@ -216,7 +216,12 @@ func TestTLS(t *testing.T) { dialFn: func(i int, listenAddr string) (net.Conn, error) { cert, err := tls.LoadX509KeyPair("./testdata/client.crt", "./testdata/client.key") require.NoError(t, err) - return tls.Dial("tcp", listenAddr, &tls.Config{InsecureSkipVerify: true, Certificates: []tls.Certificate{cert}}) // #nosec G402 + // #nosec G402 + return tls.Dial( + "tcp", + listenAddr, + &tls.Config{InsecureSkipVerify: true, Certificates: []tls.Certificate{cert}}, + ) }, appendExpectedResultFn: func(expecteResult []string, i int, msg string) []string { return append(expecteResult, msg) diff --git a/src/x/tls/config_manager.go b/src/x/tls/config_manager.go index 023c9794ea..d6048fe60c 100644 --- a/src/x/tls/config_manager.go +++ b/src/x/tls/config_manager.go @@ -29,6 +29,7 @@ import ( "time" "github.com/m3db/m3/src/x/instrument" + "github.com/uber-go/tally" "go.uber.org/zap" ) @@ -37,6 +38,7 @@ const getConfigMetricName = "get-tls-config" var sleepFn = time.Sleep +// ConfigManager is a tls config manager capable of certificate rotation type ConfigManager interface { TLSConfig() (*tls.Config, error) ServerMode() ServerMode @@ -65,6 +67,7 @@ func newConfigManagerMetrics(scope tally.Scope) configManagerMetrics { } } +// NewConfigManager creates a new config manager func NewConfigManager(opts Options, instrumentOpts instrument.Options) ConfigManager { scope := instrumentOpts.MetricsScope() c := &configManager{ @@ -135,6 +138,7 @@ func (c *configManager) loadTLSConfig() (*tls.Config, error) { if c.options.MutualTLSEnabled() { clientAuthType = tls.RequireAndVerifyClientCert } + // #nosec G402 return &tls.Config{ RootCAs: certPool, ClientCAs: certPool, diff --git a/src/x/tls/config_manager_test.go b/src/x/tls/config_manager_test.go index e0ca268a5b..1a882a4adc 100644 --- a/src/x/tls/config_manager_test.go +++ b/src/x/tls/config_manager_test.go @@ -30,13 +30,14 @@ import ( "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/tallytest" - "github.com/stretchr/testify/require" "github.com/uber-go/tally" "go.uber.org/zap" + + "github.com/stretchr/testify/require" ) func appendCA(filename string, certPool *x509.CertPool) error { - certs, err := os.ReadFile(filename) + certs, err := os.ReadFile(filename) // #nosec G304 if err != nil { return fmt.Errorf("read bundle error: %w", err) } @@ -112,6 +113,7 @@ func TestLoadX509KeyPair(t *testing.T) { cm.options = opts certificates, err = cm.loadX509KeyPair() require.Error(t, err) + require.Len(t, certificates, 0) opts = opts.SetCertFile("testdata/1.crt").SetKeyFile("testdata/1.key") cm.options = opts @@ -130,13 +132,13 @@ func TestLoadTLSConfig(t *testing.T) { opts = opts.SetCAFile("wrong/path") cm.options = opts - tlsConfig, err := cm.loadTLSConfig() + _, err := cm.loadTLSConfig() require.Error(t, err) opts = opts.SetCAFile("testdata/1.crt") opts = opts.SetCertFile("wrong/path").SetKeyFile("wrong/path") cm.options = opts - tlsConfig, err = cm.loadTLSConfig() + _, err = cm.loadTLSConfig() require.Error(t, err) opts = opts. @@ -146,7 +148,7 @@ func TestLoadTLSConfig(t *testing.T) { SetInsecureSkipVerify(true). SetServerName("server name") cm.options = opts - tlsConfig, err = cm.loadTLSConfig() + tlsConfig, err := cm.loadTLSConfig() require.NoError(t, err) require.NotNil(t, tlsConfig.RootCAs) require.NotNil(t, tlsConfig.ClientCAs) From a07157d97ff30203edf7853826442961b31a0cf0 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 30 Jul 2024 13:11:24 +0000 Subject: [PATCH 36/47] Linter fixed --- src/x/serialize/encode_decode_test.go | 4 ++-- src/x/tls/config_manager_test.go | 1 + src/x/tls/options.go | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/x/serialize/encode_decode_test.go b/src/x/serialize/encode_decode_test.go index f5963a3942..652aa0fdd8 100644 --- a/src/x/serialize/encode_decode_test.go +++ b/src/x/serialize/encode_decode_test.go @@ -24,9 +24,9 @@ import ( "strings" "testing" - "github.com/stretchr/testify/require" - "github.com/m3db/m3/src/x/ident" + + "github.com/stretchr/testify/require" ) func TestRoundTripLiteralsOfMaximumLength(t *testing.T) { diff --git a/src/x/tls/config_manager_test.go b/src/x/tls/config_manager_test.go index 1a882a4adc..284b7b64b5 100644 --- a/src/x/tls/config_manager_test.go +++ b/src/x/tls/config_manager_test.go @@ -30,6 +30,7 @@ import ( "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/tallytest" + "github.com/uber-go/tally" "go.uber.org/zap" diff --git a/src/x/tls/options.go b/src/x/tls/options.go index fa261b378d..9d30ec61a0 100644 --- a/src/x/tls/options.go +++ b/src/x/tls/options.go @@ -71,15 +71,15 @@ type Options interface { } type options struct { - clientEnabled bool - insecureSkipVerify bool serverName string - serverMode ServerMode - mTLSEnabled bool certFile string keyFile string caFile string certificatesTTL time.Duration + serverMode ServerMode + clientEnabled bool + insecureSkipVerify bool + mTLSEnabled bool } // NewOptions creates a new set of tls options From 001ccb3b779b0cb775e50103221d0ecad0063949 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 30 Jul 2024 13:31:25 +0000 Subject: [PATCH 37/47] Imports fixed --- src/x/tls/config_manager_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/x/tls/config_manager_test.go b/src/x/tls/config_manager_test.go index 284b7b64b5..fd94124aa0 100644 --- a/src/x/tls/config_manager_test.go +++ b/src/x/tls/config_manager_test.go @@ -31,10 +31,10 @@ import ( "github.com/m3db/m3/src/x/instrument" "github.com/m3db/m3/src/x/tallytest" + "github.com/stretchr/testify/require" + "github.com/uber-go/tally" "go.uber.org/zap" - - "github.com/stretchr/testify/require" ) func appendCA(filename string, certPool *x509.CertPool) error { From 2e39e94d59a324c951bbc9c234cda08f6c3f05cd Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 30 Jul 2024 13:44:28 +0000 Subject: [PATCH 38/47] Imports fixed --- src/x/tls/config_manager_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/x/tls/config_manager_test.go b/src/x/tls/config_manager_test.go index fd94124aa0..1b691ac39c 100644 --- a/src/x/tls/config_manager_test.go +++ b/src/x/tls/config_manager_test.go @@ -32,7 +32,6 @@ import ( "github.com/m3db/m3/src/x/tallytest" "github.com/stretchr/testify/require" - "github.com/uber-go/tally" "go.uber.org/zap" ) From ed12ea45d5014f1348e5052236495bfd56164654 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 30 Jul 2024 14:25:29 +0000 Subject: [PATCH 39/47] Tests fixed --- src/x/server/server_test.go | 5 ++++- src/x/tls/config_manager_test.go | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index b8b873170f..32fa31d2bc 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -239,7 +239,10 @@ func TestTLS(t *testing.T) { conn, err := tt.dialFn(i, listenAddr) require.NoError(t, err) waitFor(func() bool { return len(s.conns) == 1 }, 5, 100*time.Millisecond) - keepAlive, err := isKeepAlive(s.conns[0].(*securedConn).Conn) + s.Lock() + c := s.conns[0] + s.Unlock() + keepAlive, err := isKeepAlive(c.(*securedConn).Conn) require.True(t, keepAlive) require.NoError(t, err) diff --git a/src/x/tls/config_manager_test.go b/src/x/tls/config_manager_test.go index 1b691ac39c..552db50d30 100644 --- a/src/x/tls/config_manager_test.go +++ b/src/x/tls/config_manager_test.go @@ -195,7 +195,9 @@ func TestUpdateCertificate(t *testing.T) { SetServerName("server name") cmInterface = NewConfigManager(opts, instrumentOpts) cm = cmInterface.(*configManager) + cm.mu.Lock() cm.metrics = cmm + cm.mu.Unlock() <-waitCalledCh require.NotNil(t, cm.tlsConfig) tallytest.AssertCounterValue(t, 1, testScope.Snapshot(), successMetricName, map[string]string{}) From 2a2a17edaa777b8812ecda6cc3a971458c8cb665 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 30 Jul 2024 14:56:06 +0000 Subject: [PATCH 40/47] TLS tests fixed --- src/x/server/server_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index 32fa31d2bc..13479bbe68 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -238,7 +238,11 @@ func TestTLS(t *testing.T) { for i := 0; i < tt.numClients; i++ { conn, err := tt.dialFn(i, listenAddr) require.NoError(t, err) - waitFor(func() bool { return len(s.conns) == 1 }, 5, 100*time.Millisecond) + waitFor(func() bool { + s.Lock() + defer s.Unlock() + return len(s.conns) == 1 + }, 5, 100*time.Millisecond) s.Lock() c := s.conns[0] s.Unlock() From 1b3ec54166655f2b8dcbb85f01eefd7a4f61472c Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Tue, 30 Jul 2024 15:02:56 +0000 Subject: [PATCH 41/47] loadCertPool function fixed --- src/x/tls/config_manager.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/x/tls/config_manager.go b/src/x/tls/config_manager.go index d6048fe60c..071afcf840 100644 --- a/src/x/tls/config_manager.go +++ b/src/x/tls/config_manager.go @@ -108,6 +108,8 @@ func (c *configManager) loadCertPool() (*x509.CertPool, error) { if err != nil { return nil, fmt.Errorf("read bundle error: %w", err) } + c.mu.Lock() + defer c.mu.Unlock() if ok := c.certPool.AppendCertsFromPEM(certs); !ok { return nil, fmt.Errorf("cannot append cert to cert pool") } From 5a81da4b8aabd44070e13d880e8b8c608f10dbd3 Mon Sep 17 00:00:00 2001 From: pranithraparthi <125410054+pranithraparthi@users.noreply.github.com> Date: Tue, 30 Jul 2024 10:59:57 -0400 Subject: [PATCH 42/47] Add metrics for time taken to connected to m3db cluster during startup (#4267) --- src/dbnode/client/session.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/dbnode/client/session.go b/src/dbnode/client/session.go index a6a528cc98..bb2ac91561 100644 --- a/src/dbnode/client/session.go +++ b/src/dbnode/client/session.go @@ -199,6 +199,7 @@ type sessionMetrics struct { topologyUpdatedSuccess tally.Counter topologyUpdatedError tally.Counter streamFromPeersMetrics map[shardMetricsKey]streamFromPeersMetrics + clusterConnectLatency tally.Histogram } func newSessionMetrics(scope tally.Scope) sessionMetrics { @@ -225,6 +226,7 @@ func newSessionMetrics(scope tally.Scope) sessionMetrics { topologyUpdatedSuccess: scope.Counter("topology.updated-success"), topologyUpdatedError: scope.Counter("topology.updated-error"), streamFromPeersMetrics: make(map[shardMetricsKey]streamFromPeersMetrics), + clusterConnectLatency: histogramWithDurationBuckets(scope, "cluster-connect.latency"), } } @@ -895,6 +897,7 @@ func (s *session) hostQueues( } }() + startConnectAttempt := s.nowFn() for { if now := s.nowFn(); now.Sub(start) >= s.opts.ClusterConnectTimeout() { switch firstConnectConsistencyLevel { @@ -941,6 +944,7 @@ func (s *session) hostQueues( } connected = true + s.metrics.clusterConnectLatency.RecordDuration(s.nowFn().Sub(startConnectAttempt)) return queues, replicas, majority, nil } From b582bc5b67d0d378f1db713a13e5442846073b56 Mon Sep 17 00:00:00 2001 From: kentzeng12 <114431467+kentzeng12@users.noreply.github.com> Date: Tue, 30 Jul 2024 19:39:28 -0700 Subject: [PATCH 43/47] [buildkite] Fix integration dbnode lru and dbnode recently read tests in buildkite pipeline (#4284) * uncomment test * comment out TestGraphiteFindSequential * do not skip TestGraphiteFindParallel * skip TestGraphiteFindSequential * uncomment dbnode read test --- .buildkite/pipeline.yml | 48 +++++++++++++------- src/dbnode/integration/graphite_find_test.go | 1 + 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 605e114cfe..79d2a19bbc 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -200,22 +200,38 @@ steps: # gopath-checkout#v1.0.1: # import: github.com/m3db/m3 # <<: *common -# - name: "Integration (dbnode Recently Read) %n" -# parallelism: 2 -# command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read -# plugins: -# docker-compose#v2.5.1: -# run: app -# workdir: /go/src/github.com/m3db/m3 -# <<: *common -# - name: "Integration (dbnode LRU) %n" -# parallelism: 2 -# command: make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru -# plugins: -# docker-compose#v2.5.1: -# run: app -# workdir: /go/src/github.com/m3db/m3 -# <<: *common + - name: "Integration (dbnode Recently Read) %n" + parallelism: 2 + plugins: + docker-compose#v2.5.1: + run: app + workdir: /go/src/github.com/m3db/m3 + kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=recently_read + <<: *common + - name: "Integration (dbnode LRU) %n" + parallelism: 2 + plugins: + docker-compose#v2.5.1: + run: app + workdir: /go/src/github.com/m3db/m3 + kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + make clean install-vendor-m3 test-ci-integration-dbnode cache_policy=lru + <<: *common - name: "Integration (aggregator TCP client) %n" parallelism: 1 plugins: diff --git a/src/dbnode/integration/graphite_find_test.go b/src/dbnode/integration/graphite_find_test.go index f708bdde30..a3e5dc66ea 100644 --- a/src/dbnode/integration/graphite_find_test.go +++ b/src/dbnode/integration/graphite_find_test.go @@ -74,6 +74,7 @@ func TestGraphiteFindSequential(t *testing.T) { // NB(rob): We need to investigate why using high concurrency (and hence // need to use small dataset size since otherwise verification takes // forever) encounters errors running on CI. + t.SkipNow() testGraphiteFind(t, testGraphiteFindOptions{ checkConcurrency: 1, datasetSize: mediumDatasetSize, From 8ba3134e6b31c95759cc245e6614ccb990bfe331 Mon Sep 17 00:00:00 2001 From: kentzeng12 <114431467+kentzeng12@users.noreply.github.com> Date: Wed, 31 Jul 2024 12:41:17 -0700 Subject: [PATCH 44/47] [buildkite] Fix Docker and Doc build test in buildkite pipeline (#4282) * uncomment last test * uncomment second command * add common step * add another command * switch order of commands * fix typo * test1 * uncomment rest of passing pipeline tests --- .buildkite/pipeline.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 79d2a19bbc..03fe8be3d3 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -298,14 +298,16 @@ steps: # env: # - FOSSA_API_KEY # <<: *common -# - name: "Check for docker and docs builds :docker: :books:" -# commands: -# - ".ci/docker/check_do_docker.sh" -# - ".buildkite/scripts/check_do_docs.sh" -# agents: -# queue: init -# timeout_in_minutes: 10 -# retry: -# automatic: -# limit: 1 -# manual: true + - name: "Check for docker and docs builds :docker: :books:" + plugins: + kubernetes: + <<: *kubernetes + podSpec: + <<: *podSpec + containers: + - <<: *commandContainer + command: + - |- + .buildkite/scripts/check_do_docs.sh + .ci/docker/check_do_docker.sh + <<: *common From f35027cd77558d18678754fee59688244d6f7bd5 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Thu, 15 Aug 2024 15:25:58 +0000 Subject: [PATCH 45/47] Linter errors fixed --- src/aggregator/client/options.go | 2 -- src/x/serialize/encode_decode_test.go | 4 ++-- src/x/server/server_benchmark_test.go | 10 +++++----- src/x/server/server_test.go | 9 +++++---- src/x/tls/config_manager.go | 4 ++-- src/x/tls/config_manager_test.go | 10 +++++----- 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/aggregator/client/options.go b/src/aggregator/client/options.go index 5279564a28..f42dc69411 100644 --- a/src/aggregator/client/options.go +++ b/src/aggregator/client/options.go @@ -31,7 +31,6 @@ import ( "github.com/m3db/m3/src/x/clock" "github.com/m3db/m3/src/x/instrument" xio "github.com/m3db/m3/src/x/io" - xtls "github.com/m3db/m3/src/x/tls" ) // AggregatorClientType determines the aggregator client type. @@ -252,7 +251,6 @@ type options struct { maxBatchSize int flushWorkerCount int aggregatorClientType AggregatorClientType - tlsOptions xtls.Options } // NewOptions creates a new set of client options. diff --git a/src/x/serialize/encode_decode_test.go b/src/x/serialize/encode_decode_test.go index 652aa0fdd8..f5963a3942 100644 --- a/src/x/serialize/encode_decode_test.go +++ b/src/x/serialize/encode_decode_test.go @@ -24,9 +24,9 @@ import ( "strings" "testing" - "github.com/m3db/m3/src/x/ident" - "github.com/stretchr/testify/require" + + "github.com/m3db/m3/src/x/ident" ) func TestRoundTripLiteralsOfMaximumLength(t *testing.T) { diff --git a/src/x/server/server_benchmark_test.go b/src/x/server/server_benchmark_test.go index 212b17c0aa..2adbfed57a 100644 --- a/src/x/server/server_benchmark_test.go +++ b/src/x/server/server_benchmark_test.go @@ -28,10 +28,10 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/m3db/m3/src/x/retry" xtls "github.com/m3db/m3/src/x/tls" - - "github.com/stretchr/testify/require" ) const ( @@ -87,7 +87,7 @@ func testBenchServer(addr string, h Handler, tlsMode xtls.ServerMode, mTLSEnable return s } -func dial(listenAddr string, tlsMode xtls.ServerMode, certs []tls.Certificate, b *testing.B) (net.Conn, error) { +func dial(listenAddr string, tlsMode xtls.ServerMode, certs []tls.Certificate) (net.Conn, error) { if tlsMode == xtls.Disabled { return net.Dial("tcp", listenAddr) } @@ -103,7 +103,7 @@ func benchmarkServer(tlsMode xtls.ServerMode, mTLSEnabled bool, b *testing.B) { cert, err := tls.LoadX509KeyPair("./testdata/client.crt", "./testdata/client.key") require.NoError(b, err) for n := 0; n < b.N; n++ { - conn, err := dial(server.listener.Addr().String(), tlsMode, []tls.Certificate{cert}, b) + conn, err := dial(server.listener.Addr().String(), tlsMode, []tls.Certificate{cert}) require.NoError(b, err) msg := fmt.Sprintf("msg%d", n) _, err = conn.Write([]byte(msg)) @@ -133,7 +133,7 @@ func benchmarkKeepAliveServer(tlsMode xtls.ServerMode, mTLSEnabled bool, b *test require.NoError(b, err) cert, err := tls.LoadX509KeyPair("./testdata/client.crt", "./testdata/client.key") require.NoError(b, err) - conn, err := dial(server.listener.Addr().String(), tlsMode, []tls.Certificate{cert}, b) + conn, err := dial(server.listener.Addr().String(), tlsMode, []tls.Certificate{cert}) require.NoError(b, err) for n := 0; n < b.N; n++ { msg := fmt.Sprintf("msg%d", n) diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index 15d6a5340a..64f0fc2bb3 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -64,7 +64,8 @@ func isKeepAlive(conn net.Conn) (bool, error) { return false, nil } -func waitFor(checkFn func() bool, numChecks int, waitTime time.Duration) { +func waitFor(checkFn func() bool, waitTime time.Duration) { + numChecks := 5 checks := 0 for !checkFn() && checks < numChecks { time.Sleep(waitTime) @@ -139,7 +140,7 @@ func TestServerListenAndClose(t *testing.T) { for h.called() < numClients { time.Sleep(100 * time.Millisecond) } - waitFor(func() bool { return h.called() == numClients }, 5, 100*time.Millisecond) + waitFor(func() bool { return h.called() == numClients }, 100*time.Millisecond) require.False(t, h.isClosed()) @@ -242,7 +243,7 @@ func TestTLS(t *testing.T) { s.Lock() defer s.Unlock() return len(s.conns) == 1 - }, 5, 100*time.Millisecond) + }, 100*time.Millisecond) s.Lock() c := s.conns[0] s.Unlock() @@ -256,7 +257,7 @@ func TestTLS(t *testing.T) { _, err = conn.Write([]byte(msg)) require.NoError(t, err) } - waitFor(func() bool { return h.called() == tt.expectedServerCalls }, 5, 100*time.Millisecond) + waitFor(func() bool { return h.called() == tt.expectedServerCalls }, 100*time.Millisecond) require.False(t, h.isClosed()) s.Close() diff --git a/src/x/tls/config_manager.go b/src/x/tls/config_manager.go index 071afcf840..b9d318748f 100644 --- a/src/x/tls/config_manager.go +++ b/src/x/tls/config_manager.go @@ -28,10 +28,10 @@ import ( "sync" "time" - "github.com/m3db/m3/src/x/instrument" - "github.com/uber-go/tally" "go.uber.org/zap" + + "github.com/m3db/m3/src/x/instrument" ) const getConfigMetricName = "get-tls-config" diff --git a/src/x/tls/config_manager_test.go b/src/x/tls/config_manager_test.go index 552db50d30..795916de78 100644 --- a/src/x/tls/config_manager_test.go +++ b/src/x/tls/config_manager_test.go @@ -28,12 +28,12 @@ import ( "testing" "time" - "github.com/m3db/m3/src/x/instrument" - "github.com/m3db/m3/src/x/tallytest" - "github.com/stretchr/testify/require" "github.com/uber-go/tally" "go.uber.org/zap" + + "github.com/m3db/m3/src/x/instrument" + "github.com/m3db/m3/src/x/tallytest" ) func appendCA(filename string, certPool *x509.CertPool) error { @@ -81,13 +81,13 @@ func TestLoadCertPool(t *testing.T) { opts = opts.SetCAFile("testdata/3.crt") cm.options = opts - certPool, err = cm.loadCertPool() + _, err = cm.loadCertPool() require.Error(t, err) require.Equal(t, expectedCertPool.Subjects(), cm.certPool.Subjects()) opts = opts.SetCAFile("wrong/path") cm.options = opts - certPool, err = cm.loadCertPool() + _, err = cm.loadCertPool() require.Error(t, err) } From ace2a9569249b8fe798b70a1663fa0524114a4b0 Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Thu, 15 Aug 2024 15:39:48 +0000 Subject: [PATCH 46/47] Benchmark test fixed --- src/x/server/server_benchmark_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/x/server/server_benchmark_test.go b/src/x/server/server_benchmark_test.go index 2adbfed57a..6d11753e92 100644 --- a/src/x/server/server_benchmark_test.go +++ b/src/x/server/server_benchmark_test.go @@ -109,7 +109,7 @@ func benchmarkServer(tlsMode xtls.ServerMode, mTLSEnabled bool, b *testing.B) { _, err = conn.Write([]byte(msg)) require.NoError(b, err) } - waitFor(func() bool { return handler.called() == b.N }, 5, 100*time.Millisecond) + waitFor(func() bool { return handler.called() == b.N }, 100*time.Millisecond) require.Equal(b, b.N, handler.called()) } From 2e9ed48c711ff7f75aa58aa5721c3360fbdd8eae Mon Sep 17 00:00:00 2001 From: "roman.mazhut" Date: Thu, 15 Aug 2024 15:50:48 +0000 Subject: [PATCH 47/47] Linter errors fixed --- src/x/server/server_benchmark_test.go | 2 +- src/x/server/server_test.go | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/x/server/server_benchmark_test.go b/src/x/server/server_benchmark_test.go index 6d11753e92..afc83cff53 100644 --- a/src/x/server/server_benchmark_test.go +++ b/src/x/server/server_benchmark_test.go @@ -109,7 +109,7 @@ func benchmarkServer(tlsMode xtls.ServerMode, mTLSEnabled bool, b *testing.B) { _, err = conn.Write([]byte(msg)) require.NoError(b, err) } - waitFor(func() bool { return handler.called() == b.N }, 100*time.Millisecond) + waitFor(func() bool { return handler.called() == b.N }) require.Equal(b, b.N, handler.called()) } diff --git a/src/x/server/server_test.go b/src/x/server/server_test.go index 64f0fc2bb3..40b2deb162 100644 --- a/src/x/server/server_test.go +++ b/src/x/server/server_test.go @@ -64,8 +64,9 @@ func isKeepAlive(conn net.Conn) (bool, error) { return false, nil } -func waitFor(checkFn func() bool, waitTime time.Duration) { +func waitFor(checkFn func() bool) { numChecks := 5 + waitTime := 100 * time.Millisecond checks := 0 for !checkFn() && checks < numChecks { time.Sleep(waitTime) @@ -140,7 +141,7 @@ func TestServerListenAndClose(t *testing.T) { for h.called() < numClients { time.Sleep(100 * time.Millisecond) } - waitFor(func() bool { return h.called() == numClients }, 100*time.Millisecond) + waitFor(func() bool { return h.called() == numClients }) require.False(t, h.isClosed()) @@ -243,7 +244,7 @@ func TestTLS(t *testing.T) { s.Lock() defer s.Unlock() return len(s.conns) == 1 - }, 100*time.Millisecond) + }) s.Lock() c := s.conns[0] s.Unlock() @@ -257,7 +258,7 @@ func TestTLS(t *testing.T) { _, err = conn.Write([]byte(msg)) require.NoError(t, err) } - waitFor(func() bool { return h.called() == tt.expectedServerCalls }, 100*time.Millisecond) + waitFor(func() bool { return h.called() == tt.expectedServerCalls }) require.False(t, h.isClosed()) s.Close()