From 4dedfdc82cb6616457887f5ab38e2b0c0bb20875 Mon Sep 17 00:00:00 2001 From: Menghan Li Date: Tue, 25 Sep 2018 13:17:25 -0700 Subject: [PATCH] credentials: support google default creds (#2315) Google default creds is a combo of ALTS, TLS and OAuth2. The right set of creds will be picked to use based on environment. This PR contains: - A new `creds.Bundle` type - changes to use it in ClientConn and transport - dial option to set the bundle for a ClientConn - balancer options and NewSubConnOption to set it for SubConn - Google default creds implementation by @cesarghali - grpclb changes to use different creds mode for different servers - interop client changes for google default creds testing --- balancer/balancer.go | 9 +- balancer/grpclb/grpclb.go | 34 +++++- balancer/grpclb/grpclb_remote_balancer.go | 22 +++- balancer_conn_wrappers.go | 5 +- clientconn.go | 18 ++- credentials/alts/alts.go | 2 +- credentials/credentials.go | 19 +++ credentials/google/google.go | 100 ++++++++++++++++ dialoptions.go | 14 ++- internal/internal.go | 11 ++ internal/transport/http2_client.go | 23 +++- internal/transport/transport.go | 6 +- interop/client/client.go | 70 ++++++++--- test/balancer_test.go | 134 ++++++++++++++++++++++ test/creds_test.go | 131 +++++++++++++++++++++ test/end2end_test.go | 4 + 16 files changed, 559 insertions(+), 43 deletions(-) create mode 100644 credentials/google/google.go create mode 100644 test/balancer_test.go create mode 100644 test/creds_test.go diff --git a/balancer/balancer.go b/balancer/balancer.go index 069feb1e723a..eb2231a4cfb7 100644 --- a/balancer/balancer.go +++ b/balancer/balancer.go @@ -88,7 +88,12 @@ type SubConn interface { } // NewSubConnOptions contains options to create new SubConn. -type NewSubConnOptions struct{} +type NewSubConnOptions struct { + // CredsBundle is the credentials bundle that will be used in the created + // SubConn. If it's nil, the original creds from grpc DialOptions will be + // used. + CredsBundle credentials.Bundle +} // ClientConn represents a gRPC ClientConn. // @@ -125,6 +130,8 @@ type BuildOptions struct { // use to dial to a remote load balancer server. The Balancer implementations // can ignore this if it does not need to talk to another party securely. DialCreds credentials.TransportCredentials + // CredsBundle is the credentials bundle that the Balancer can use. + CredsBundle credentials.Bundle // Dialer is the custom dialer the Balancer implementation can use to dial // to a remote load balancer server. The Balancer implementations // can ignore this if it doesn't need to talk to remote balancer. diff --git a/balancer/grpclb/grpclb.go b/balancer/grpclb/grpclb.go index 0162ac7a01c8..2e1583641b65 100644 --- a/balancer/grpclb/grpclb.go +++ b/balancer/grpclb/grpclb.go @@ -37,7 +37,9 @@ import ( "google.golang.org/grpc/balancer" lbpb "google.golang.org/grpc/balancer/grpclb/grpc_lb_v1" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" "google.golang.org/grpc/internal/backoff" "google.golang.org/grpc/resolver" ) @@ -166,13 +168,35 @@ func (b *lbBuilder) Build(cc balancer.ClientConn, opt balancer.BuildOptions) bal backoff: defaultBackoffConfig, // TODO: make backoff configurable. } + var err error + if opt.CredsBundle != nil { + lb.grpclbClientConnCreds, err = opt.CredsBundle.NewWithMode(internal.CredsBundleModeBalancer) + if err != nil { + grpclog.Warningf("lbBalancer: client connection creds NewWithMode failed: %v", err) + } + lb.grpclbBackendCreds, err = opt.CredsBundle.NewWithMode(internal.CredsBundleModeBackendFromBalancer) + if err != nil { + grpclog.Warningf("lbBalancer: backend creds NewWithMode failed: %v", err) + } + } + return lb } type lbBalancer struct { - cc *lbCacheClientConn - target string - opt balancer.BuildOptions + cc *lbCacheClientConn + target string + opt balancer.BuildOptions + + // grpclbClientConnCreds is the creds bundle to be used to connect to grpclb + // servers. If it's nil, use the TransportCredentials from BuildOptions + // instead. + grpclbClientConnCreds credentials.Bundle + // grpclbBackendCreds is the creds bundle to be used for addresses that are + // returned by grpclb server. If it's nil, don't set anything when creating + // SubConns. + grpclbBackendCreds credentials.Bundle + fallbackTimeout time.Duration doneCh chan struct{} @@ -302,7 +326,7 @@ func (lb *lbBalancer) fallbackToBackendsAfter(fallbackTimeout time.Duration) { return } lb.fallbackTimerExpired = true - lb.refreshSubConns(lb.resolvedBackendAddrs) + lb.refreshSubConns(lb.resolvedBackendAddrs, false) lb.mu.Unlock() } @@ -349,7 +373,7 @@ func (lb *lbBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { // This means we received a new list of resolved backends, and we are // still in fallback mode. Need to update the list of backends we are // using to the new list of backends. - lb.refreshSubConns(lb.resolvedBackendAddrs) + lb.refreshSubConns(lb.resolvedBackendAddrs, false) } lb.mu.Unlock() } diff --git a/balancer/grpclb/grpclb_remote_balancer.go b/balancer/grpclb/grpclb_remote_balancer.go index d0ecd68be48d..7410329046c2 100644 --- a/balancer/grpclb/grpclb_remote_balancer.go +++ b/balancer/grpclb/grpclb_remote_balancer.go @@ -80,7 +80,7 @@ func (lb *lbBalancer) processServerList(l *lbpb.ServerList) { } // Call refreshSubConns to create/remove SubConns. - lb.refreshSubConns(backendAddrs) + lb.refreshSubConns(backendAddrs, true) // Regenerate and update picker no matter if there's update on backends (if // any SubConn will be newed/removed). Because since the full serverList was // different, there might be updates in drops or pick weights(different @@ -96,7 +96,12 @@ func (lb *lbBalancer) processServerList(l *lbpb.ServerList) { // indicating whether the backendAddrs are different from the cached // backendAddrs (whether any SubConn was newed/removed). // Caller must hold lb.mu. -func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address) bool { +func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address, fromGRPCLBServer bool) bool { + opts := balancer.NewSubConnOptions{} + if fromGRPCLBServer { + opts.CredsBundle = lb.grpclbBackendCreds + } + lb.backendAddrs = nil var backendsUpdated bool // addrsSet is the set converted from backendAddrs, it's used to quick @@ -113,7 +118,7 @@ func (lb *lbBalancer) refreshSubConns(backendAddrs []resolver.Address) bool { backendsUpdated = true // Use addrWithMD to create the SubConn. - sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, balancer.NewSubConnOptions{}) + sc, err := lb.cc.NewSubConn([]resolver.Address{addr}, opts) if err != nil { grpclog.Warningf("roundrobinBalancer: failed to create new SubConn: %v", err) continue @@ -266,6 +271,8 @@ func (lb *lbBalancer) dialRemoteLB(remoteLBName string) { grpclog.Warningf("grpclb: failed to override the server name in the credentials: %v, using Insecure", err) dopts = append(dopts, grpc.WithInsecure()) } + } else if bundle := lb.grpclbClientConnCreds; bundle != nil { + dopts = append(dopts, grpc.WithCredentialsBundle(bundle)) } else { dopts = append(dopts, grpc.WithInsecure()) } @@ -283,9 +290,12 @@ func (lb *lbBalancer) dialRemoteLB(remoteLBName string) { dopts = append(dopts, grpc.WithChannelzParentID(lb.opt.ChannelzParentID)) } - // DialContext using manualResolver.Scheme, which is a random scheme generated - // when init grpclb. The target name is not important. - cc, err := grpc.DialContext(context.Background(), "grpclb:///grpclb.server", dopts...) + // DialContext using manualResolver.Scheme, which is a random scheme + // generated when init grpclb. The target scheme here is not important. + // + // The grpc dial target will be used by the creds (ALTS) as the authority, + // so it has to be set to remoteLBName that comes from resolver. + cc, err := grpc.DialContext(context.Background(), remoteLBName, dopts...) if err != nil { grpclog.Fatalf("failed to dial: %v", err) } diff --git a/balancer_conn_wrappers.go b/balancer_conn_wrappers.go index c23f81706fba..1ab95fde2578 100644 --- a/balancer_conn_wrappers.go +++ b/balancer_conn_wrappers.go @@ -197,7 +197,7 @@ func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer if ccb.subConns == nil { return nil, fmt.Errorf("grpc: ClientConn balancer wrapper was closed") } - ac, err := ccb.cc.newAddrConn(addrs) + ac, err := ccb.cc.newAddrConn(addrs, opts) if err != nil { return nil, err } @@ -257,6 +257,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) { } if !acbw.ac.tryUpdateAddrs(addrs) { cc := acbw.ac.cc + opts := acbw.ac.scopts acbw.ac.mu.Lock() // Set old ac.acbw to nil so the Shutdown state update will be ignored // by balancer. @@ -272,7 +273,7 @@ func (acbw *acBalancerWrapper) UpdateAddresses(addrs []resolver.Address) { return } - ac, err := cc.newAddrConn(addrs) + ac, err := cc.newAddrConn(addrs, opts) if err != nil { grpclog.Warningf("acBalancerWrapper: UpdateAddresses: failed to newAddrConn: %v", err) return diff --git a/clientconn.go b/clientconn.go index b42b8316bd1e..956b11f26e71 100644 --- a/clientconn.go +++ b/clientconn.go @@ -80,6 +80,9 @@ var ( // being set for ClientConn. Users should either set one or explicitly // call WithInsecure DialOption to disable security. errNoTransportSecurity = errors.New("grpc: no transport security set (use grpc.WithInsecure() explicitly or set credentials)") + // errTransportCredsAndBundle indicates that creds bundle is used together + // with other individual Transport Credentials. + errTransportCredsAndBundle = errors.New("grpc: credentials.Bundle may not be used with individual TransportCredentials") // errTransportCredentialsMissing indicates that users want to transmit security // information (e.g., oauth2 token) which requires secure connection on an insecure // connection. @@ -156,11 +159,14 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } if !cc.dopts.insecure { - if cc.dopts.copts.TransportCredentials == nil { + if cc.dopts.copts.TransportCredentials == nil && cc.dopts.copts.CredsBundle == nil { return nil, errNoTransportSecurity } + if cc.dopts.copts.TransportCredentials != nil && cc.dopts.copts.CredsBundle != nil { + return nil, errTransportCredsAndBundle + } } else { - if cc.dopts.copts.TransportCredentials != nil { + if cc.dopts.copts.TransportCredentials != nil || cc.dopts.copts.CredsBundle != nil { return nil, errCredentialsConflict } for _, cd := range cc.dopts.copts.PerRPCCredentials { @@ -273,6 +279,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } cc.balancerBuildOpts = balancer.BuildOptions{ DialCreds: credsClone, + CredsBundle: cc.dopts.copts.CredsBundle, Dialer: cc.dopts.copts.Dialer, ChannelzParentID: cc.channelzID, } @@ -560,10 +567,11 @@ func (cc *ClientConn) handleSubConnStateChange(sc balancer.SubConn, s connectivi // newAddrConn creates an addrConn for addrs and adds it to cc.conns. // // Caller needs to make sure len(addrs) > 0. -func (cc *ClientConn) newAddrConn(addrs []resolver.Address) (*addrConn, error) { +func (cc *ClientConn) newAddrConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (*addrConn, error) { ac := &addrConn{ cc: cc, addrs: addrs, + scopts: opts, dopts: cc.dopts, czData: new(channelzData), successfulHandshake: true, // make the first nextAddr() call _not_ move addrIdx up by 1 @@ -861,6 +869,7 @@ type addrConn struct { dopts dialOptions events trace.EventLog acbw balancer.SubConn + scopts balancer.NewSubConnOptions transport transport.ClientTransport // The current transport. @@ -1004,6 +1013,9 @@ func (ac *addrConn) resetTransport(resolveNow bool) { addr := ac.addrs[ac.addrIdx] copts := ac.dopts.copts + if ac.scopts.CredsBundle != nil { + copts.CredsBundle = ac.scopts.CredsBundle + } ac.mu.Unlock() if channelz.IsOn() { diff --git a/credentials/alts/alts.go b/credentials/alts/alts.go index 0d3688a26faf..fba041268ff1 100644 --- a/credentials/alts/alts.go +++ b/credentials/alts/alts.go @@ -66,7 +66,7 @@ var ( // ErrUntrustedPlatform is returned from ClientHandshake and // ServerHandshake is running on a platform where the trustworthiness of // the handshaker service is not guaranteed. - ErrUntrustedPlatform = errors.New("untrusted platform") + ErrUntrustedPlatform = errors.New("ALTS: untrusted platform. ALTS is only supported on GCP") ) // AuthInfo exposes security information from the ALTS handshake to the diff --git a/credentials/credentials.go b/credentials/credentials.go index 1dae57ab1881..6c2b811fd50a 100644 --- a/credentials/credentials.go +++ b/credentials/credentials.go @@ -108,6 +108,25 @@ type TransportCredentials interface { OverrideServerName(string) error } +// Bundle is a combination of TransportCredentials and PerRPCCredentials. +// +// It also contains a mode switching method, so it can be used as a combination +// of different credential policies. +// +// Bundle cannot be used together with individual TransportCredentials. +// PerRPCCredentials from Bundle will be appended to other PerRPCCredentials. +// +// This API is experimental. +type Bundle interface { + TransportCredentials() TransportCredentials + PerRPCCredentials() PerRPCCredentials + // NewWithMode should make a copy of Bundle, and switch mode. Modifying the + // existing Bundle may cause races. + // + // NewWithMode returns nil if the requested mode is not supported. + NewWithMode(mode string) (Bundle, error) +} + // TLSInfo contains the auth information for a TLS authenticated connection. // It implements the AuthInfo interface. type TLSInfo struct { diff --git a/credentials/google/google.go b/credentials/google/google.go new file mode 100644 index 000000000000..721eeb420099 --- /dev/null +++ b/credentials/google/google.go @@ -0,0 +1,100 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package google + +import ( + "fmt" + "time" + + "golang.org/x/net/context" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/alts" + "google.golang.org/grpc/credentials/oauth" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal" +) + +const tokenRequestTimeout = 30 * time.Second + +// NewDefaultCredentials returns a credentials bundle that is configured to work +// with google services. +// +// This API is experimental. +func NewDefaultCredentials() credentials.Bundle { + c := &creds{} + bundle, err := c.NewWithMode(internal.CredsBundleModeFallback) + if err != nil { + grpclog.Warningf("google default creds: failed to create new creds: %v", err) + } + return bundle +} + +// creds implements credentials.Bundle. +type creds struct { + // Supported modes are defined in internal/internal.go. + mode string + // The transport credentials associated with this bundle. + transportCreds credentials.TransportCredentials + // The per RPC credentials associated with this bundle. + perRPCCreds credentials.PerRPCCredentials +} + +func (c *creds) TransportCredentials() credentials.TransportCredentials { + return c.transportCreds +} + +func (c *creds) PerRPCCredentials() credentials.PerRPCCredentials { + if c == nil { + return nil + } + return c.perRPCCreds +} + +// NewWithMode should make a copy of Bundle, and switch mode. Modifying the +// existing Bundle may cause races. +func (c *creds) NewWithMode(mode string) (credentials.Bundle, error) { + newCreds := &creds{mode: mode} + + // Create transport credentials. + switch mode { + case internal.CredsBundleModeFallback: + newCreds.transportCreds = credentials.NewTLS(nil) + case internal.CredsBundleModeBackendFromBalancer, internal.CredsBundleModeBalancer: + // Only the clients can use google default credentials, so we only need + // to create new ALTS client creds here. + newCreds.transportCreds = alts.NewClientCreds(alts.DefaultClientOptions()) + default: + return nil, fmt.Errorf("google default creds: unsupported mode: %v", mode) + } + + if mode == internal.CredsBundleModeFallback || mode == internal.CredsBundleModeBackendFromBalancer { + // Create per RPC credentials. + // For the time being, we required per RPC credentials for both TLS and + // ALTS. In the future, this will only be required for TLS. + ctx, cancel := context.WithTimeout(context.Background(), tokenRequestTimeout) + defer cancel() + var err error + newCreds.perRPCCreds, err = oauth.NewApplicationDefault(ctx) + if err != nil { + grpclog.Warningf("google default creds: failed to create application oauth: %v", err) + } + } + + return newCreds, nil +} diff --git a/dialoptions.go b/dialoptions.go index 3d3c9e231dae..99b495272960 100644 --- a/dialoptions.go +++ b/dialoptions.go @@ -286,7 +286,8 @@ func WithInsecure() DialOption { } // WithTransportCredentials returns a DialOption which configures a connection -// level security credentials (e.g., TLS/SSL). +// level security credentials (e.g., TLS/SSL). This should not be used together +// with WithCredentialsBundle. func WithTransportCredentials(creds credentials.TransportCredentials) DialOption { return newFuncDialOption(func(o *dialOptions) { o.copts.TransportCredentials = creds @@ -301,6 +302,17 @@ func WithPerRPCCredentials(creds credentials.PerRPCCredentials) DialOption { }) } +// WithCredentialsBundle returns a DialOption to set a credentials bundle for +// the ClientConn.WithCreds. This should not be used together with +// WithTransportCredentials. +// +// This API is experimental. +func WithCredentialsBundle(b credentials.Bundle) DialOption { + return newFuncDialOption(func(o *dialOptions) { + o.copts.CredsBundle = b + }) +} + // WithTimeout returns a DialOption that configures a timeout for dialing a // ClientConn initially. This is valid if and only if WithBlock() is present. // diff --git a/internal/internal.go b/internal/internal.go index c35afb05ee24..8a7028691372 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -26,3 +26,14 @@ var ( // WithResolverBuilder is exported by clientconn.go WithResolverBuilder interface{} // func (resolver.Builder) grpc.DialOption ) + +const ( + // CredsBundleModeFallback switches GoogleDefaultCreds to fallback mode. + CredsBundleModeFallback = "fallback" + // CredsBundleModeBalancer switches GoogleDefaultCreds to grpclb balancer + // mode. + CredsBundleModeBalancer = "balancer" + // CredsBundleModeBackendFromBalancer switches GoogleDefaultCreds to mode + // that supports backend returned by grpclb balancer. + CredsBundleModeBackendFromBalancer = "backend-from-balancer" +) diff --git a/internal/transport/http2_client.go b/internal/transport/http2_client.go index 0787d6237294..3740fb33ba66 100644 --- a/internal/transport/http2_client.go +++ b/internal/transport/http2_client.go @@ -73,7 +73,7 @@ type http2Client struct { isSecure bool - creds []credentials.PerRPCCredentials + perRPCCreds []credentials.PerRPCCredentials // Boolean to keep track of reading activity on transport. // 1 is true and 0 is false. @@ -169,9 +169,20 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne isSecure bool authInfo credentials.AuthInfo ) - if creds := opts.TransportCredentials; creds != nil { + transportCreds := opts.TransportCredentials + perRPCCreds := opts.PerRPCCredentials + + if b := opts.CredsBundle; b != nil { + if t := b.TransportCredentials(); t != nil { + transportCreds = t + } + if t := b.PerRPCCredentials(); t != nil { + perRPCCreds = append(perRPCCreds, t) + } + } + if transportCreds != nil { scheme = "https" - conn, authInfo, err = creds.ClientHandshake(connectCtx, addr.Authority, conn) + conn, authInfo, err = transportCreds.ClientHandshake(connectCtx, addr.Authority, conn) if err != nil { return nil, connectionErrorf(isTemporary(err), err, "transport: authentication handshake failed: %v", err) } @@ -216,7 +227,7 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr TargetInfo, opts Conne scheme: scheme, activeStreams: make(map[uint32]*Stream), isSecure: isSecure, - creds: opts.PerRPCCredentials, + perRPCCreds: perRPCCreds, kp: kp, statsHandler: opts.StatsHandler, initialWindowSize: initialWindowSize, @@ -453,7 +464,7 @@ func (t *http2Client) createHeaderFields(ctx context.Context, callHdr *CallHdr) func (t *http2Client) createAudience(callHdr *CallHdr) string { // Create an audience string only if needed. - if len(t.creds) == 0 && callHdr.Creds == nil { + if len(t.perRPCCreds) == 0 && callHdr.Creds == nil { return "" } // Construct URI required to get auth request metadata. @@ -468,7 +479,7 @@ func (t *http2Client) createAudience(callHdr *CallHdr) string { func (t *http2Client) getTrAuthData(ctx context.Context, audience string) (map[string]string, error) { authData := map[string]string{} - for _, c := range t.creds { + for _, c := range t.perRPCCreds { data, err := c.GetRequestMetadata(ctx, audience) if err != nil { if _, ok := status.FromError(err); ok { diff --git a/internal/transport/transport.go b/internal/transport/transport.go index 2608c5e5c9d2..1be518a62fb4 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -465,8 +465,12 @@ type ConnectOptions struct { FailOnNonTempDialError bool // PerRPCCredentials stores the PerRPCCredentials required to issue RPCs. PerRPCCredentials []credentials.PerRPCCredentials - // TransportCredentials stores the Authenticator required to setup a client connection. + // TransportCredentials stores the Authenticator required to setup a client + // connection. Only one of TransportCredentials and CredsBundle is non-nil. TransportCredentials credentials.TransportCredentials + // CredsBundle is the credentials bundle to be used. Only one of + // TransportCredentials and CredsBundle is non-nil. + CredsBundle credentials.Bundle // KeepaliveParams stores the keepalive parameters. KeepaliveParams keepalive.ClientParameters // StatsHandler stores the handler for stats. diff --git a/interop/client/client.go b/interop/client/client.go index 69bba92597bb..1fa92f9abb7e 100644 --- a/interop/client/client.go +++ b/interop/client/client.go @@ -27,6 +27,7 @@ import ( _ "google.golang.org/grpc/balancer/grpclb" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/alts" + "google.golang.org/grpc/credentials/google" "google.golang.org/grpc/credentials/oauth" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/interop" @@ -35,10 +36,15 @@ import ( "google.golang.org/grpc/testdata" ) +const ( + googleDefaultCredsName = "google_default_credentials" +) + var ( caFile = flag.String("ca_file", "", "The file containning the CA root cert file") useTLS = flag.Bool("use_tls", false, "Connection uses TLS if true") useALTS = flag.Bool("use_alts", false, "Connection uses ALTS if true (this option can only be used on GCP)") + customCredentialsType = flag.String("custom_credentials_type", "", "Custom creds to use, excluding TLS or ALTS") altsHSAddr = flag.String("alts_handshaker_service_address", "", "ALTS handshaker gRPC service address") testCA = flag.Bool("use_test_ca", false, "Whether to replace platform root CAs with test CA as the CA root") serviceAccountKeyFile = flag.String("service_account_key_file", "", "Path to service account json key file") @@ -70,15 +76,43 @@ var ( unimplemented_service: client attempts to call unimplemented service.`) ) +type credsMode uint8 + +const ( + credsNone credsMode = iota + credsTLS + credsALTS + credsGoogleDefaultCreds +) + func main() { flag.Parse() - resolver.SetDefaultScheme("dns") - if *useTLS && *useALTS { - grpclog.Fatalf("use_tls and use_alts cannot be both set to true") + var useGDC bool // use google default creds + if *customCredentialsType != "" { + if *customCredentialsType != googleDefaultCredsName { + grpclog.Fatalf("custom_credentials_type can only be set to %v or not set", googleDefaultCredsName) + } + useGDC = true } + if (*useTLS && *useALTS) || (*useTLS && useGDC) || (*useALTS && useGDC) { + grpclog.Fatalf("only one of TLS, ALTS and google default creds can be used") + } + + var credsChosen credsMode + switch { + case *useTLS: + credsChosen = credsTLS + case *useALTS: + credsChosen = credsALTS + case useGDC: + credsChosen = credsGoogleDefaultCreds + } + + resolver.SetDefaultScheme("dns") serverAddr := net.JoinHostPort(*serverHost, strconv.Itoa(*serverPort)) var opts []grpc.DialOption - if *useTLS { + switch credsChosen { + case credsTLS: var sn string if *tlsServerName != "" { sn = *tlsServerName @@ -97,17 +131,19 @@ func main() { creds = credentials.NewClientTLSFromCert(nil, sn) } opts = append(opts, grpc.WithTransportCredentials(creds)) - } else if *useALTS { + case credsALTS: altsOpts := alts.DefaultClientOptions() if *altsHSAddr != "" { altsOpts.HandshakerServiceAddress = *altsHSAddr } altsTC := alts.NewClientCreds(altsOpts) opts = append(opts, grpc.WithTransportCredentials(altsTC)) - } else { + case credsGoogleDefaultCreds: + opts = append(opts, grpc.WithCredentialsBundle(google.NewDefaultCredentials())) + default: opts = append(opts, grpc.WithInsecure()) } - if *useTLS || *useALTS { + if credsChosen == credsTLS || credsChosen == credsALTS { if *testCase == "compute_engine_creds" { opts = append(opts, grpc.WithPerRPCCredentials(oauth.NewComputeEngine())) } else if *testCase == "service_account_creds" { @@ -156,32 +192,32 @@ func main() { interop.DoTimeoutOnSleepingServer(tc) grpclog.Infoln("TimeoutOnSleepingServer done") case "compute_engine_creds": - if !*useTLS && !*useALTS { - grpclog.Fatalf("Neither TLS or ALTS are enabled. TLS or ALTS is required to execute compute_engine_creds test case.") + if credsChosen == credsNone { + grpclog.Fatalf("Credentials (TLS, ALTS or google default creds) need to be set for compute_engine_creds test case.") } interop.DoComputeEngineCreds(tc, *defaultServiceAccount, *oauthScope) grpclog.Infoln("ComputeEngineCreds done") case "service_account_creds": - if !*useTLS && !*useALTS { - grpclog.Fatalf("Neither TLS or ALTS are enabled. TLS or ALTS is required to execute service_account_creds test case.") + if credsChosen == credsNone { + grpclog.Fatalf("Credentials (TLS, ALTS or google default creds) need to be set for service_account_creds test case.") } interop.DoServiceAccountCreds(tc, *serviceAccountKeyFile, *oauthScope) grpclog.Infoln("ServiceAccountCreds done") case "jwt_token_creds": - if !*useTLS && !*useALTS { - grpclog.Fatalf("Neither TLS or ALTS are enabled. TLS or ALTS is required to execute jwt_token_creds test case.") + if credsChosen == credsNone { + grpclog.Fatalf("Credentials (TLS, ALTS or google default creds) need to be set for jwt_token_creds test case.") } interop.DoJWTTokenCreds(tc, *serviceAccountKeyFile) grpclog.Infoln("JWTtokenCreds done") case "per_rpc_creds": - if !*useTLS && !*useALTS { - grpclog.Fatalf("Neither TLS or ALTS are enabled. TLS or ALTS is required to execute per_rpc_creds test case.") + if credsChosen == credsNone { + grpclog.Fatalf("Credentials (TLS, ALTS or google default creds) need to be set for per_rpc_creds test case.") } interop.DoPerRPCCreds(tc, *serviceAccountKeyFile, *oauthScope) grpclog.Infoln("PerRPCCreds done") case "oauth2_auth_token": - if !*useTLS && !*useALTS { - grpclog.Fatalf("Neither TLS or ALTS are enabled. TLS or ALTS is required to execute oauth2_auth_token test case.") + if credsChosen == credsNone { + grpclog.Fatalf("Credentials (TLS, ALTS or google default creds) need to be set for oauth2_auth_token test case.") } interop.DoOauth2TokenCreds(tc, *serviceAccountKeyFile, *oauthScope) grpclog.Infoln("Oauth2TokenCreds done") diff --git a/test/balancer_test.go b/test/balancer_test.go new file mode 100644 index 000000000000..d7e1f99ca18a --- /dev/null +++ b/test/balancer_test.go @@ -0,0 +1,134 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "testing" + + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/internal/leakcheck" + "google.golang.org/grpc/resolver" + testpb "google.golang.org/grpc/test/grpc_testing" + "google.golang.org/grpc/testdata" +) + +const testBalancerName = "testbalancer" + +// testBalancer creates one subconn with the first address from resolved +// addresses. +// +// It's used to test options for NewSubConn are applies correctly. +type testBalancer struct { + cc balancer.ClientConn + sc balancer.SubConn + + newSubConnOptions balancer.NewSubConnOptions +} + +func (b *testBalancer) Build(cc balancer.ClientConn, opt balancer.BuildOptions) balancer.Balancer { + b.cc = cc + return b +} + +func (*testBalancer) Name() string { + return testBalancerName +} + +func (b *testBalancer) HandleResolvedAddrs(addrs []resolver.Address, err error) { + // Only create a subconn at the first time. + if err == nil && b.sc == nil { + b.sc, err = b.cc.NewSubConn(addrs, b.newSubConnOptions) + if err != nil { + grpclog.Errorf("testBalancer: failed to NewSubConn: %v", err) + return + } + b.cc.UpdateBalancerState(connectivity.Idle, &picker{sc: b.sc}) + b.sc.Connect() + } +} + +func (b *testBalancer) HandleSubConnStateChange(sc balancer.SubConn, s connectivity.State) { + grpclog.Infof("testBalancer: HandleSubConnStateChange: %p, %v", sc, s) + if b.sc != sc { + grpclog.Infof("testBalancer: ignored state change because sc is not recognized") + return + } + if s == connectivity.Shutdown { + b.sc = nil + return + } + + switch s { + case connectivity.Ready, connectivity.Idle: + b.cc.UpdateBalancerState(s, &picker{sc: sc}) + case connectivity.Connecting: + b.cc.UpdateBalancerState(s, &picker{err: balancer.ErrNoSubConnAvailable}) + case connectivity.TransientFailure: + b.cc.UpdateBalancerState(s, &picker{err: balancer.ErrTransientFailure}) + } +} + +func (b *testBalancer) Close() { +} + +type picker struct { + err error + sc balancer.SubConn +} + +func (p *picker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { + if p.err != nil { + return nil, nil, p.err + } + return p.sc, nil, nil +} + +func TestCredsBundleFromBalancer(t *testing.T) { + balancer.Register(&testBalancer{ + newSubConnOptions: balancer.NewSubConnOptions{ + CredsBundle: &testCredsBundle{}, + }, + }) + defer leakcheck.Check(t) + te := newTest(t, env{name: "creds-bundle", network: "tcp", balancer: ""}) + te.tapHandle = authHandle + te.customDialOptions = []grpc.DialOption{ + grpc.WithBalancerName(testBalancerName), + } + creds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key")) + if err != nil { + t.Fatalf("Failed to generate credentials %v", err) + } + te.customServerOptions = []grpc.ServerOption{ + grpc.Creds(creds), + } + te.startServer(&testServer{}) + defer te.tearDown() + + cc := te.clientConn() + tc := testpb.NewTestServiceClient(cc) + if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} diff --git a/test/creds_test.go b/test/creds_test.go new file mode 100644 index 000000000000..f56440fa2e10 --- /dev/null +++ b/test/creds_test.go @@ -0,0 +1,131 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +// TODO(https://github.com/grpc/grpc-go/issues/2330): move all creds releated +// tests to this file. + +import ( + "testing" + + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/leakcheck" + testpb "google.golang.org/grpc/test/grpc_testing" + "google.golang.org/grpc/testdata" +) + +const ( + bundlePerRPCOnly = "perRPCOnly" + bundleTLSOnly = "tlsOnly" +) + +type testCredsBundle struct { + t *testing.T + mode string +} + +func (c *testCredsBundle) TransportCredentials() credentials.TransportCredentials { + if c.mode == bundlePerRPCOnly { + return nil + } + + creds, err := credentials.NewClientTLSFromFile(testdata.Path("ca.pem"), "x.test.youtube.com") + if err != nil { + c.t.Logf("Failed to load credentials: %v", err) + return nil + } + return creds +} + +func (c *testCredsBundle) PerRPCCredentials() credentials.PerRPCCredentials { + if c.mode == bundleTLSOnly { + return nil + } + return testPerRPCCredentials{} +} + +func (c *testCredsBundle) NewWithMode(mode string) (credentials.Bundle, error) { + return &testCredsBundle{mode: mode}, nil +} + +func TestCredsBundleBoth(t *testing.T) { + defer leakcheck.Check(t) + te := newTest(t, env{name: "creds-bundle", network: "tcp", balancer: "v1", security: "empty"}) + te.tapHandle = authHandle + te.customDialOptions = []grpc.DialOption{ + grpc.WithCredentialsBundle(&testCredsBundle{t: t}), + } + creds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key")) + if err != nil { + t.Fatalf("Failed to generate credentials %v", err) + } + te.customServerOptions = []grpc.ServerOption{ + grpc.Creds(creds), + } + te.startServer(&testServer{}) + defer te.tearDown() + + cc := te.clientConn() + tc := testpb.NewTestServiceClient(cc) + if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} + +func TestCredsBundleTransportCredentials(t *testing.T) { + defer leakcheck.Check(t) + te := newTest(t, env{name: "creds-bundle", network: "tcp", balancer: "v1", security: "empty"}) + te.customDialOptions = []grpc.DialOption{ + grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundleTLSOnly}), + } + creds, err := credentials.NewServerTLSFromFile(testdata.Path("server1.pem"), testdata.Path("server1.key")) + if err != nil { + t.Fatalf("Failed to generate credentials %v", err) + } + te.customServerOptions = []grpc.ServerOption{ + grpc.Creds(creds), + } + te.startServer(&testServer{}) + defer te.tearDown() + + cc := te.clientConn() + tc := testpb.NewTestServiceClient(cc) + if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} + +func TestCredsBundlePerRPCCredentials(t *testing.T) { + defer leakcheck.Check(t) + te := newTest(t, env{name: "creds-bundle", network: "tcp", balancer: "v1", security: "empty"}) + te.tapHandle = authHandle + te.customDialOptions = []grpc.DialOption{ + grpc.WithCredentialsBundle(&testCredsBundle{t: t, mode: bundlePerRPCOnly}), + } + te.startServer(&testServer{}) + defer te.tearDown() + + cc := te.clientConn() + tc := testpb.NewTestServiceClient(cc) + if _, err := tc.EmptyCall(context.Background(), &testpb.Empty{}); err != nil { + t.Fatalf("Test failed. Reason: %v", err) + } +} diff --git a/test/end2end_test.go b/test/end2end_test.go index c80804e8b7ff..6572ad05ebbc 100644 --- a/test/end2end_test.go +++ b/test/end2end_test.go @@ -477,6 +477,7 @@ type test struct { clientInitialConnWindowSize int32 perRPCCreds credentials.PerRPCCredentials customDialOptions []grpc.DialOption + customServerOptions []grpc.ServerOption resolverScheme string cliKeepAlive *keepalive.ClientParameters svrKeepAlive *keepalive.ServerParameters @@ -607,6 +608,7 @@ func (te *test) listenAndServe(ts testpb.TestServiceServer, listen func(network, if te.svrKeepAlive != nil { sopts = append(sopts, grpc.KeepaliveParams(*te.svrKeepAlive)) } + sopts = append(sopts, te.customServerOptions...) s := grpc.NewServer(sopts...) te.srv = s if te.healthServer != nil { @@ -806,6 +808,8 @@ func (te *test) configDial(opts ...grpc.DialOption) ([]grpc.DialOption, string) opts = append(opts, grpc.WithTransportCredentials(creds)) case "clientTimeoutCreds": opts = append(opts, grpc.WithTransportCredentials(&clientTimeoutCreds{})) + case "empty": + // Don't add any transport creds option. default: opts = append(opts, grpc.WithInsecure()) }