Skip to content

Commit

Permalink
credentials: support google default creds (#2315)
Browse files Browse the repository at this point in the history
Google default creds is a combo of ALTS, TLS and OAuth2. The right set of creds will be picked to use based on environment.

This PR contains:
 - A new `creds.Bundle` type
   - changes to use it in ClientConn and transport
   - dial option to set the bundle for a ClientConn
   - balancer options and NewSubConnOption to set it for SubConn
 - Google default creds implementation by @cesarghali 
 - grpclb changes to use different creds mode for different servers
 - interop client changes for google default creds testing
  • Loading branch information
menghanl authored Sep 25, 2018
1 parent ebea9b5 commit 4dedfdc
Show file tree
Hide file tree
Showing 16 changed files with 559 additions and 43 deletions.
9 changes: 8 additions & 1 deletion balancer/balancer.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,12 @@ type SubConn interface {
}

// NewSubConnOptions contains options to create new SubConn.
type NewSubConnOptions struct{}
type NewSubConnOptions struct {
// CredsBundle is the credentials bundle that will be used in the created
// SubConn. If it's nil, the original creds from grpc DialOptions will be
// used.
CredsBundle credentials.Bundle
}

// ClientConn represents a gRPC ClientConn.
//
Expand Down Expand Up @@ -125,6 +130,8 @@ type BuildOptions struct {
// use to dial to a remote load balancer server. The Balancer implementations
// can ignore this if it does not need to talk to another party securely.
DialCreds credentials.TransportCredentials
// CredsBundle is the credentials bundle that the Balancer can use.
CredsBundle credentials.Bundle
// Dialer is the custom dialer the Balancer implementation can use to dial
// to a remote load balancer server. The Balancer implementations
// can ignore this if it doesn't need to talk to remote balancer.
Expand Down
34 changes: 29 additions & 5 deletions balancer/grpclb/grpclb.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,9 @@ import (
"google.golang.org/grpc/balancer"
lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1"
"google.golang.org/grpc/connectivity"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/grpclog"
"google.golang.org/grpc/internal"
"google.golang.org/grpc/internal/backoff"
"google.golang.org/grpc/resolver"
)
Expand Down Expand Up @@ -166,13 +168,35 @@ func (b *lbBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) bal
backoff: defaultBackoffConfig, // TODO: make backoff configurable.
}

var err error
if opt.CredsBundle != nil {
lb.grpclbClientConnCreds, err = opt.CredsBundle.NewWithMode(internal.CredsBundleModeBalancer)
if err != nil {
grpclog.Warningf("lbBalancer: client connection creds NewWithMode failed: %v", err)
}
lb.grpclbBackendCreds, err = opt.CredsBundle.NewWithMode(internal.CredsBundleModeBackendFromBalancer)
if err != nil {
grpclog.Warningf("lbBalancer: backend creds NewWithMode failed: %v", err)
}
}

return lb
}

type lbBalancer struct {
cc *lbCacheClientConn
target string
opt balancer.BuildOptions
cc *lbCacheClientConn
target string
opt balancer.BuildOptions

// grpclbClientConnCreds is the creds bundle to be used to connect to grpclb
// servers. If it's nil, use the TransportCredentials from BuildOptions
// instead.
grpclbClientConnCreds credentials.Bundle
// grpclbBackendCreds is the creds bundle to be used for addresses that are
// returned by grpclb server. If it's nil, don't set anything when creating
// SubConns.
grpclbBackendCreds credentials.Bundle

fallbackTimeout time.Duration
doneCh chan struct{}

Expand Down Expand Up @@ -302,7 +326,7 @@ func (lb *lbBalancer) fallbackToBackendsAfter(fallbackTimeout time.Duration) {
return
}
lb.fallbackTimerExpired = true
lb.refreshSubConns(lb.resolvedBackendAddrs)
lb.refreshSubConns(lb.resolvedBackendAddrs, false)
lb.mu.Unlock()
}

Expand Down Expand Up @@ -349,7 +373,7 @@ func (lb *lbBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) {
// This means we received a new list of resolved backends, and we are
// still in fallback mode. Need to update the list of backends we are
// using to the new list of backends.
lb.refreshSubConns(lb.resolvedBackendAddrs)
lb.refreshSubConns(lb.resolvedBackendAddrs, false)
}
lb.mu.Unlock()
}
Expand Down
22 changes: 16 additions & 6 deletions balancer/grpclb/grpclb_remote_balancer.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func (lb *lbBalancer) processServerList(l *lbpb.ServerList) {
}

// Call refreshSubConns to create/remove SubConns.
lb.refreshSubConns(backendAddrs)
lb.refreshSubConns(backendAddrs, true)
// Regenerate and update picker no matter if there's update on backends (if
// any SubConn will be newed/removed). Because since the full serverList was
// different, there might be updates in drops or pick weights(different
Expand All @@ -96,7 +96,12 @@ func (lb *lbBalancer) processServerList(l *lbpb.ServerList) {
// indicating whether the backendAddrs are different from the cached
// backendAddrs (whether any SubConn was newed/removed).
// Caller must hold lb.mu.
func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address) bool {
func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fromGRPCLBServer bool) bool {
opts := balancer.NewSubConnOptions{}
if fromGRPCLBServer {
opts.CredsBundle = lb.grpclbBackendCreds
}

lb.backendAddrs = nil
var backendsUpdated bool
// addrsSet is the set converted from backendAddrs, it's used to quick
Expand All @@ -113,7 +118,7 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address) bool {
backendsUpdated = true

// Use addrWithMD to create the SubConn.
sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{})
sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, opts)
if err != nil {
grpclog.Warningf("roundrobinBalancer: failed to create new SubConn: %v", err)
continue
Expand Down Expand Up @@ -266,6 +271,8 @@ func (lb *lbBalancer) dialRemoteLB(remoteLBName string) {
grpclog.Warningf("grpclb: failed to override the server name in the credentials: %v, using Insecure", err)
dopts = append(dopts, grpc.WithInsecure())
}
} else if bundle := lb.grpclbClientConnCreds; bundle != nil {
dopts = append(dopts, grpc.WithCredentialsBundle(bundle))
} else {
dopts = append(dopts, grpc.WithInsecure())
}
Expand All @@ -283,9 +290,12 @@ func (lb *lbBalancer) dialRemoteLB(remoteLBName string) {
dopts = append(dopts, grpc.WithChannelzParentID(lb.opt.ChannelzParentID))
}

// DialContext using manualResolver.Scheme, which is a random scheme generated
// when init grpclb. The target name is not important.
cc, err := grpc.DialContext(context.Background(), "grpclb:///grpclb.server", dopts...)
// DialContext using manualResolver.Scheme, which is a random scheme
// generated when init grpclb. The target scheme here is not important.
//
// The grpc dial target will be used by the creds (ALTS) as the authority,
// so it has to be set to remoteLBName that comes from resolver.
cc, err := grpc.DialContext(context.Background(), remoteLBName, dopts...)
if err != nil {
grpclog.Fatalf("failed to dial: %v", err)
}
Expand Down
5 changes: 3 additions & 2 deletions balancer_conn_wrappers.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer
if ccb.subConns == nil {
return nil, fmt.Errorf("grpc: ClientConn balancer wrapper was closed")
}
ac, err := ccb.cc.newAddrConn(addrs)
ac, err := ccb.cc.newAddrConn(addrs, opts)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -257,6 +257,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) {
}
if !acbw.ac.tryUpdateAddrs(addrs) {
cc := acbw.ac.cc
opts := acbw.ac.scopts
acbw.ac.mu.Lock()
// Set old ac.acbw to nil so the Shutdown state update will be ignored
// by balancer.
Expand All @@ -272,7 +273,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) {
return
}

ac, err := cc.newAddrConn(addrs)
ac, err := cc.newAddrConn(addrs, opts)
if err != nil {
grpclog.Warningf("acBalancerWrapper: UpdateAddresses: failed to newAddrConn: %v", err)
return
Expand Down
18 changes: 15 additions & 3 deletions clientconn.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,9 @@ var (
// being set for ClientConn. Users should either set one or explicitly
// call WithInsecure DialOption to disable security.
errNoTransportSecurity = errors.New("grpc: no transport security set (use grpc.WithInsecure() explicitly or set credentials)")
// errTransportCredsAndBundle indicates that creds bundle is used together
// with other individual Transport Credentials.
errTransportCredsAndBundle = errors.New("grpc: credentials.Bundle may not be used with individual TransportCredentials")
// errTransportCredentialsMissing indicates that users want to transmit security
// information (e.g., oauth2 token) which requires secure connection on an insecure
// connection.
Expand Down Expand Up @@ -156,11 +159,14 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
}

if !cc.dopts.insecure {
if cc.dopts.copts.TransportCredentials == nil {
if cc.dopts.copts.TransportCredentials == nil && cc.dopts.copts.CredsBundle == nil {
return nil, errNoTransportSecurity
}
if cc.dopts.copts.TransportCredentials != nil && cc.dopts.copts.CredsBundle != nil {
return nil, errTransportCredsAndBundle
}
} else {
if cc.dopts.copts.TransportCredentials != nil {
if cc.dopts.copts.TransportCredentials != nil || cc.dopts.copts.CredsBundle != nil {
return nil, errCredentialsConflict
}
for _, cd := range cc.dopts.copts.PerRPCCredentials {
Expand Down Expand Up @@ -273,6 +279,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn *
}
cc.balancerBuildOpts = balancer.BuildOptions{
DialCreds: credsClone,
CredsBundle: cc.dopts.copts.CredsBundle,
Dialer: cc.dopts.copts.Dialer,
ChannelzParentID: cc.channelzID,
}
Expand Down Expand Up @@ -560,10 +567,11 @@ func (cc *ClientConn) handleSubConnStateChange(sc balancer.SubConn, s connectivi
// newAddrConn creates an addrConn for addrs and adds it to cc.conns.
//
// Caller needs to make sure len(addrs) > 0.
func (cc *ClientConn) newAddrConn(addrs []resolver.Address) (*addrConn, error) {
func (cc *ClientConn) newAddrConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (*addrConn, error) {
ac := &addrConn{
cc: cc,
addrs: addrs,
scopts: opts,
dopts: cc.dopts,
czData: new(channelzData),
successfulHandshake: true, // make the first nextAddr() call _not_ move addrIdx up by 1
Expand Down Expand Up @@ -861,6 +869,7 @@ type addrConn struct {
dopts dialOptions
events trace.EventLog
acbw balancer.SubConn
scopts balancer.NewSubConnOptions

transport transport.ClientTransport // The current transport.

Expand Down Expand Up @@ -1004,6 +1013,9 @@ func (ac *addrConn) resetTransport(resolveNow bool) {

addr := ac.addrs[ac.addrIdx]
copts := ac.dopts.copts
if ac.scopts.CredsBundle != nil {
copts.CredsBundle = ac.scopts.CredsBundle
}
ac.mu.Unlock()

if channelz.IsOn() {
Expand Down
2 changes: 1 addition & 1 deletion credentials/alts/alts.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ var (
// ErrUntrustedPlatform is returned from ClientHandshake and
// ServerHandshake is running on a platform where the trustworthiness of
// the handshaker service is not guaranteed.
ErrUntrustedPlatform = errors.New("untrusted platform")
ErrUntrustedPlatform = errors.New("ALTS: untrusted platform. ALTS is only supported on GCP")
)

// AuthInfo exposes security information from the ALTS handshake to the
Expand Down
19 changes: 19 additions & 0 deletions credentials/credentials.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,25 @@ type TransportCredentials interface {
OverrideServerName(string) error
}

// Bundle is a combination of TransportCredentials and PerRPCCredentials.
//
// It also contains a mode switching method, so it can be used as a combination
// of different credential policies.
//
// Bundle cannot be used together with individual TransportCredentials.
// PerRPCCredentials from Bundle will be appended to other PerRPCCredentials.
//
// This API is experimental.
type Bundle interface {
TransportCredentials() TransportCredentials
PerRPCCredentials() PerRPCCredentials
// NewWithMode should make a copy of Bundle, and switch mode. Modifying the
// existing Bundle may cause races.
//
// NewWithMode returns nil if the requested mode is not supported.
NewWithMode(mode string) (Bundle, error)
}

// TLSInfo contains the auth information for a TLS authenticated connection.
// It implements the AuthInfo interface.
type TLSInfo struct {
Expand Down
100 changes: 100 additions & 0 deletions credentials/google/google.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
*
* Copyright 2018 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package google

import (
"fmt"
"time"

"golang.org/x/net/context"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/alts"
"google.golang.org/grpc/credentials/oauth"
"google.golang.org/grpc/grpclog"
"google.golang.org/grpc/internal"
)

const tokenRequestTimeout = 30 * time.Second

// NewDefaultCredentials returns a credentials bundle that is configured to work
// with google services.
//
// This API is experimental.
func NewDefaultCredentials() credentials.Bundle {
c := &creds{}
bundle, err := c.NewWithMode(internal.CredsBundleModeFallback)
if err != nil {
grpclog.Warningf("google default creds: failed to create new creds: %v", err)
}
return bundle
}

// creds implements credentials.Bundle.
type creds struct {
// Supported modes are defined in internal/internal.go.
mode string
// The transport credentials associated with this bundle.
transportCreds credentials.TransportCredentials
// The per RPC credentials associated with this bundle.
perRPCCreds credentials.PerRPCCredentials
}

func (c *creds) TransportCredentials() credentials.TransportCredentials {
return c.transportCreds
}

func (c *creds) PerRPCCredentials() credentials.PerRPCCredentials {
if c == nil {
return nil
}
return c.perRPCCreds
}

// NewWithMode should make a copy of Bundle, and switch mode. Modifying the
// existing Bundle may cause races.
func (c *creds) NewWithMode(mode string) (credentials.Bundle, error) {
newCreds := &creds{mode: mode}

// Create transport credentials.
switch mode {
case internal.CredsBundleModeFallback:
newCreds.transportCreds = credentials.NewTLS(nil)
case internal.CredsBundleModeBackendFromBalancer, internal.CredsBundleModeBalancer:
// Only the clients can use google default credentials, so we only need
// to create new ALTS client creds here.
newCreds.transportCreds = alts.NewClientCreds(alts.DefaultClientOptions())
default:
return nil, fmt.Errorf("google default creds: unsupported mode: %v", mode)
}

if mode == internal.CredsBundleModeFallback || mode == internal.CredsBundleModeBackendFromBalancer {
// Create per RPC credentials.
// For the time being, we required per RPC credentials for both TLS and
// ALTS. In the future, this will only be required for TLS.
ctx, cancel := context.WithTimeout(context.Background(), tokenRequestTimeout)
defer cancel()
var err error
newCreds.perRPCCreds, err = oauth.NewApplicationDefault(ctx)
if err != nil {
grpclog.Warningf("google default creds: failed to create application oauth: %v", err)
}
}

return newCreds, nil
}
Loading

0 comments on commit 4dedfdc

Please sign in to comment.