Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[6.1.2] Backport - Improve process connection error handling and logging (#6471) #6440

Merged
merged 2 commits into from
Apr 21, 2021
Merged

[6.1.2] Backport - Improve process connection error handling and logging (#6471) #6440

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
108e4a2
Refactor process.newClient for better error handling and logging.
Joerger Apr 15, 2021
68fe808
Merge branch 'branch/v6.1' into joerger/backport-6.1/fix/direct-conne…
Joerger Apr 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 55 additions & 50 deletions lib/service/connect.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@ limitations under the License.
package service

import (
"crypto/tls"
"net"
"path/filepath"
"strconv"
"time"

"golang.org/x/crypto/ssh"

"github.com/gravitational/roundtrip"
"github.com/gravitational/teleport"
apiclient "github.com/gravitational/teleport/api/client"
"github.com/gravitational/teleport/lib"
Expand Down Expand Up @@ -798,49 +800,49 @@ func (process *TeleportProcess) rotate(conn *Connector, localState auth.StateV2,
// The proxy address might be configured in process environment as defaults.TunnelPublicAddrEnvar
// in which case, no attempt at discovering the reverse tunnel address is made.
func (process *TeleportProcess) newClient(authServers []utils.NetAddr, identity *auth.Identity) (*auth.Client, error) {
directClient, err := process.newClientDirect(authServers, identity)
tlsConfig, err := identity.TLSConfig(process.Config.CipherSuites)
if err != nil {
return nil, trace.Wrap(err)
}

// Try and connect to the Auth Server. If the request fails, try and
// connect through a tunnel.
process.log.Debugf("Attempting to connect to Auth Server directly.")
if _, err = directClient.GetLocalClusterName(); err != nil {
process.log.WithError(err).Warn("Failed to connect to Auth Server directly.")
if err := directClient.Close(); err != nil {
process.log.WithError(err).Warn("Failed to close direct Auth Server client.")
}

// Don't attempt to connect through a tunnel as a proxy or auth server.
if identity.ID.Role == teleport.RoleAuth || identity.ID.Role == teleport.RoleProxy {
return nil, trace.Wrap(err)
}
directClient, directDialErr := process.newClientDirect(authServers, tlsConfig)
if directDialErr == nil {
process.log.Debug("Connected to Auth Server with direct connection.")
return directClient, nil
}
process.log.Debug("Failed to connect to Auth Server directly.")

var proxyAddr string
if process.Config.SSH.ProxyReverseTunnelFallbackAddr != nil {
proxyAddr = process.Config.SSH.ProxyReverseTunnelFallbackAddr.String()
} else {
// Discover address of SSH reverse tunnel server.
proxyAddr, err = process.findReverseTunnel(authServers)
if err != nil {
return nil, trace.Wrap(err)
}
}
// Don't attempt to connect through a tunnel as a proxy or auth server.
if identity.ID.Role == teleport.RoleAuth || identity.ID.Role == teleport.RoleProxy {
return nil, trace.Wrap(directDialErr)
}

logger := process.log.WithField("proxy-addr", proxyAddr)
logger.Debug("Attempting to connect to Auth Server through tunnel.")
tunnelClient, err := process.newClientThroughTunnel(proxyAddr, identity)
var proxyAddr string
if process.Config.SSH.ProxyReverseTunnelFallbackAddr != nil {
proxyAddr = process.Config.SSH.ProxyReverseTunnelFallbackAddr.String()
} else {
// Discover address of SSH reverse tunnel server.
var err error
proxyAddr, err = process.findReverseTunnel(authServers)
if err != nil {
return nil, trace.Wrap(err)
process.log.Debug("Failed to discover reverse tunnel address.")
return nil, trace.NewAggregate(directDialErr, err)
}
}

logger.Debug("Connected to Auth Server through tunnel.")
return tunnelClient, nil
logger := process.log.WithField("proxy-addr", proxyAddr)
logger.Debug("Attempting to connect to Auth Server through tunnel.")
tunnelClient, err := process.newClientThroughTunnel(proxyAddr, tlsConfig, identity.SSHClientConfig())
if err != nil {
logger.Debug("Failed to connect to Auth Server through tunnel.")
return nil, trace.NewAggregate(directDialErr, err)
}

process.log.Debug("Connected to Auth Server with direct connection.")
return directClient, nil
logger.Debug("Connected to Auth Server through tunnel.")
return tunnelClient, nil
}

// findReverseTunnel uses the web proxy to discover where the SSH reverse tunnel
Expand Down Expand Up @@ -902,16 +904,11 @@ func tunnelAddr(settings apiclient.ProxySettings) (string, error) {
return settings.SSH.TunnelListenAddr, nil
}

func (process *TeleportProcess) newClientThroughTunnel(proxyAddr string, identity *auth.Identity) (*auth.Client, error) {
tlsConfig, err := identity.TLSConfig(process.Config.CipherSuites)
if err != nil {
return nil, trace.Wrap(err)
}

func (process *TeleportProcess) newClientThroughTunnel(proxyAddr string, tlsConfig *tls.Config, sshConfig *ssh.ClientConfig) (*auth.Client, error) {
clt, err := auth.NewClient(apiclient.Config{
Dialer: &reversetunnel.TunnelAuthDialer{
ProxyAddr: proxyAddr,
ClientConfig: identity.SSHClientConfig(),
ClientConfig: sshConfig,
},
Credentials: []apiclient.Credentials{
apiclient.LoadTLS(tlsConfig),
Expand All @@ -925,29 +922,37 @@ func (process *TeleportProcess) newClientThroughTunnel(proxyAddr string, identit
// get the underlying error.
_, err = clt.GetLocalClusterName()
if err != nil {
if err2 := clt.Close(); err != nil {
process.log.WithError(err2).Warn("Failed to close Auth Server tunnel client.")
}
return nil, trace.Unwrap(err)
}

return clt, nil
}

func (process *TeleportProcess) newClientDirect(authServers []utils.NetAddr, identity *auth.Identity) (*auth.Client, error) {
tlsConfig, err := identity.TLSConfig(process.Config.CipherSuites)
if err != nil {
return nil, trace.Wrap(err)
}
func (process *TeleportProcess) newClientDirect(authServers []utils.NetAddr, tlsConfig *tls.Config) (*auth.Client, error) {
var cltParams []roundtrip.ClientParam
if process.Config.ClientTimeout != 0 {
return auth.NewClient(apiclient.Config{
Addrs: utils.NetAddrsToStrings(authServers),
Credentials: []apiclient.Credentials{
apiclient.LoadTLS(tlsConfig),
},
}, auth.ClientTimeout(process.Config.ClientTimeout))
}
return auth.NewClient(apiclient.Config{
cltParams = []roundtrip.ClientParam{auth.ClientTimeout(process.Config.ClientTimeout)}
}

clt, err := auth.NewClient(apiclient.Config{
Addrs: utils.NetAddrsToStrings(authServers),
Credentials: []apiclient.Credentials{
apiclient.LoadTLS(tlsConfig),
},
})
}, cltParams...)
if err != nil {
return nil, trace.Wrap(err)
}

if _, err := clt.GetLocalClusterName(); err != nil {
if err2 := clt.Close(); err2 != nil {
process.log.WithError(err2).Warn("Failed to close direct Auth Server client.")
}
return nil, trace.Wrap(err)
}

return clt, nil
}