Skip to content

Commit

Permalink
Dynamically create jwks clusters for jwt-providers
Browse files Browse the repository at this point in the history
  • Loading branch information
@roncodingenthusiast
roncodingenthusiast committed Jun 29, 2023
1 parent bdf4fad commit dabfaa6
Show file tree
Hide file tree
Showing 12 changed files with 389 additions and 21 deletions.
113 changes: 113 additions & 0 deletions agent/xds/clusters.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ package xds
import (
"errors"
"fmt"
"net/url"
"strconv"
"strings"
"time"

Expand Down Expand Up @@ -141,6 +143,22 @@ func (s *ResourceGenerator) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.C
clusters = append(clusters, upstreamCluster)
}

// add clusters for jwt-providers
for _, prov := range cfgSnap.JWTProviders {
//skip cluster creation for local providers
if prov.JSONWebKeySet == nil || prov.JSONWebKeySet.Remote == nil {
continue
}

cluster, err := makeJWTProviderCluster(prov)
if err != nil {
s.Logger.Warn("failed to make jwt-provider cluster", "provider name", prov.Name, "error", err)
continue
}

clusters = append(clusters, cluster)
}

for _, u := range cfgSnap.Proxy.Upstreams {
if u.DestinationType != structs.UpstreamDestTypePreparedQuery {
continue
Expand Down Expand Up @@ -184,6 +202,101 @@ func (s *ResourceGenerator) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.C
return clusters, nil
}

func makeJWTProviderCluster(p *structs.JWTProviderConfigEntry) (*envoy_cluster_v3.Cluster, error) {
if p.JSONWebKeySet == nil || p.JSONWebKeySet.Remote == nil {
return nil, fmt.Errorf("cannot create JWKS cluster for non remote JWKS. Provider Name: %s", p.Name)
}
hostname, scheme, port, err := processJwtRemoteURL(p.JSONWebKeySet.Remote.URI)
if err != nil {
return nil, err
}

//TODO: expose additional fields: eg. ConnectTimeout, through
// JWTProviderConfigEntry to allow user to configure cluster
cluster := &envoy_cluster_v3.Cluster{
Name: p.Name,
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{
Type: envoy_cluster_v3.Cluster_STRICT_DNS,
},
LbPolicy: envoy_cluster_v3.Cluster_ROUND_ROBIN,
LoadAssignment: &envoy_endpoint_v3.ClusterLoadAssignment{
ClusterName: p.Name,
Endpoints: []*envoy_endpoint_v3.LocalityLbEndpoints{
{
LbEndpoints: []*envoy_endpoint_v3.LbEndpoint{
makeJWTLbEndpoint(hostname, port),
},
},
},
},
}

if scheme == "https" {
// TODO: expose this configuration through JWTProviderConfigEntry to allow
// user to configure certs
jwksTLSContext, err := makeUpstreamTLSTransportSocket(
&envoy_tls_v3.UpstreamTlsContext{
CommonTlsContext: &envoy_tls_v3.CommonTlsContext{
ValidationContextType: &envoy_tls_v3.CommonTlsContext_ValidationContext{
ValidationContext: &envoy_tls_v3.CertificateValidationContext{
TrustedCa: nil,
},
},
},
},
)
if err != nil {
return nil, err
}

cluster.TransportSocket = jwksTLSContext
}
return cluster, nil
}

func makeJWTLbEndpoint(addr string, port int) *envoy_endpoint_v3.LbEndpoint {
return &envoy_endpoint_v3.LbEndpoint{
HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{
Endpoint: &envoy_endpoint_v3.Endpoint{
Address: &envoy_core_v3.Address{
Address: &envoy_core_v3.Address_SocketAddress{
SocketAddress: &envoy_core_v3.SocketAddress{
Address: addr,
PortSpecifier: &envoy_core_v3.SocketAddress_PortValue{
PortValue: uint32(port),
},
},
},
},
},
},
}
}

func processJwtRemoteURL(uri string) (string, string, int, error) {
u, err := url.Parse(uri)
if err != nil {
return "", "", 0, err
}

var port int
if u.Port() != "" {
port, err = strconv.Atoi(u.Port())
if err != nil {
return "", "", port, err
}
}

if port == 0 {
port = 80
if u.Scheme == "https" {
port = 443
}
}

return u.Hostname(), u.Scheme, port, nil
}

func makeExposeClusterName(destinationPort int) string {
return fmt.Sprintf("exposed_cluster_%d", destinationPort)
}
Expand Down
135 changes: 135 additions & 0 deletions agent/xds/clusters_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -959,6 +959,141 @@ func TestEnvoyLBConfig_InjectToCluster(t *testing.T) {
}
}

func TestMakeJWTProviderCluster(t *testing.T) {
// All tests here depend on golden files located under: agent/xds/testdata/jwt_authn/*
tests := map[string]struct {
provider *structs.JWTProviderConfigEntry
}{
"provider-with-hostname-no-port": {
provider: &structs.JWTProviderConfigEntry{
Kind: "jwt-provider",
Name: "okta",
Issuer: "test-issuer",
JSONWebKeySet: &structs.JSONWebKeySet{
Remote: makeTestJWKS("https://example-okta.com/.well-known/jwks.json"),
},
},
},
"provider-with-hostname-and-port": {
provider: &structs.JWTProviderConfigEntry{
Kind: "jwt-provider",
Name: "okta",
Issuer: "test-issuer",
JSONWebKeySet: &structs.JSONWebKeySet{
Remote: makeTestJWKS("https://example-okta.com:90/.well-known/jwks.json"),
},
},
},
"provider-with-ip-no-port": {
provider: &structs.JWTProviderConfigEntry{
Kind: "jwt-provider",
Name: "okta",
Issuer: "test-issuer",
JSONWebKeySet: &structs.JSONWebKeySet{
Remote: makeTestJWKS("http://127.0.0.1"),
},
},
},
"provider-with-ip-and-port": {
provider: &structs.JWTProviderConfigEntry{
Kind: "jwt-provider",
Name: "okta",
Issuer: "test-issuer",
JSONWebKeySet: &structs.JSONWebKeySet{
Remote: makeTestJWKS("https://127.0.0.1:9091"),
},
},
},
}

for name, tt := range tests {
tt := tt
t.Run(name, func(t *testing.T) {
cluster, err := makeJWTProviderCluster(tt.provider)
require.NoError(t, err)
gotJSON := protoToJSON(t, cluster)
require.JSONEq(t, goldenSimple(t, filepath.Join("jwt_authn_clusters", name), gotJSON), gotJSON)
})
}
}

func makeTestJWKS(uri string) *structs.RemoteJWKS {
return &structs.RemoteJWKS{
RequestTimeoutMs: 1000,
FetchAsynchronously: true,
URI: uri,
}
}

func TestProcessJwtRemoteURL(t *testing.T) {
tests := map[string]struct {
uri string
expectedHost string
expectedPort int
expectedScheme string
}{
"https-hostname-no-port": {
uri: "https://test.test.com",
expectedHost: "test.test.com",
expectedPort: 443,
expectedScheme: "https",
},
"https-hostname-with-port": {
uri: "https://test.test.com:4545",
expectedHost: "test.test.com",
expectedPort: 4545,
expectedScheme: "https",
},
"http-hostname-no-port": {
uri: "http://test.test.com",
expectedHost: "test.test.com",
expectedPort: 80,
expectedScheme: "http",
},
"http-hostname-with-port": {
uri: "http://test.test.com:4636",
expectedHost: "test.test.com",
expectedPort: 4636,
expectedScheme: "http",
},
"https-ip-no-port": {
uri: "https://127.0.0.1",
expectedHost: "127.0.0.1",
expectedPort: 443,
expectedScheme: "https",
},
"https-ip-with-port": {
uri: "https://127.0.0.1:3434",
expectedHost: "127.0.0.1",
expectedPort: 3434,
expectedScheme: "https",
},
"http-ip-no-port": {
uri: "http://127.0.0.1",
expectedHost: "127.0.0.1",
expectedPort: 80,
expectedScheme: "http",
},
"http-ip-with-port": {
uri: "http://127.0.0.1:9190",
expectedHost: "127.0.0.1",
expectedPort: 9190,
expectedScheme: "http",
},
}

for name, tt := range tests {
tt := tt
t.Run(name, func(t *testing.T) {
host, scheme, port, err := processJwtRemoteURL(tt.uri)
require.NoError(t, err)
require.Equal(t, host, tt.expectedHost)
require.Equal(t, scheme, tt.expectedScheme)
require.Equal(t, port, tt.expectedPort)
})
}
}

// UID is just a convenience function to aid in writing tests less verbosely.
func UID(input string) proxycfg.UpstreamID {
return proxycfg.UpstreamIDFromString(input)
Expand Down
10 changes: 4 additions & 6 deletions agent/xds/jwt_authn.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ func buildJWTProviderConfig(p *structs.JWTProviderConfigEntry, metadataKeySuffix
}
envoyCfg.JwksSourceSpecifier = specifier
} else if remote := p.JSONWebKeySet.Remote; remote != nil && remote.URI != "" {
envoyCfg.JwksSourceSpecifier = makeRemoteJWKS(remote)
envoyCfg.JwksSourceSpecifier = makeRemoteJWKS(remote, p.Name)
} else {
return nil, fmt.Errorf("invalid jwt provider config; missing JSONWebKeySet for provider: %s", p.Name)
}
Expand Down Expand Up @@ -210,14 +210,12 @@ func makeLocalJWKS(l *structs.LocalJWKS, pName string) (*envoy_http_jwt_authn_v3
return specifier, nil
}

func makeRemoteJWKS(r *structs.RemoteJWKS) *envoy_http_jwt_authn_v3.JwtProvider_RemoteJwks {
func makeRemoteJWKS(r *structs.RemoteJWKS, providerName string) *envoy_http_jwt_authn_v3.JwtProvider_RemoteJwks {
remote_specifier := envoy_http_jwt_authn_v3.JwtProvider_RemoteJwks{
RemoteJwks: &envoy_http_jwt_authn_v3.RemoteJwks{
HttpUri: &envoy_core_v3.HttpUri{
Uri: r.URI,
// TODO(roncodingenthusiast): An explicit cluster is required.
// Need to figure out replacing `jwks_cluster` will an actual cluster
HttpUpstreamType: &envoy_core_v3.HttpUri_Cluster{Cluster: "jwks_cluster"},
Uri: r.URI,
HttpUpstreamType: &envoy_core_v3.HttpUri_Cluster{Cluster: providerName},
},
AsyncFetch: &envoy_http_jwt_authn_v3.JwksAsyncFetch{
FastListener: r.FetchAsynchronously,
Expand Down
19 changes: 11 additions & 8 deletions agent/xds/jwt_authn_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ func TestBuildJWTProviderConfig(t *testing.T) {
RemoteJwks: &envoy_http_jwt_authn_v3.RemoteJwks{
HttpUri: &envoy_core_v3.HttpUri{
Uri: oktaRemoteJWKS.URI,
HttpUpstreamType: &envoy_core_v3.HttpUri_Cluster{Cluster: "jwks_cluster"},
HttpUpstreamType: &envoy_core_v3.HttpUri_Cluster{Cluster: ceRemoteJWKS.Name},
Timeout: &durationpb.Duration{Seconds: 1},
},
AsyncFetch: &envoy_http_jwt_authn_v3.JwksAsyncFetch{
Expand Down Expand Up @@ -520,16 +520,18 @@ func TestMakeLocalJWKS(t *testing.T) {

func TestMakeRemoteJWKS(t *testing.T) {
tests := map[string]struct {
jwks *structs.RemoteJWKS
expected *envoy_http_jwt_authn_v3.JwtProvider_RemoteJwks
jwks *structs.RemoteJWKS
providerName string
expected *envoy_http_jwt_authn_v3.JwtProvider_RemoteJwks
}{
"with-no-cache-duration": {
jwks: oktaRemoteJWKS,
jwks: oktaRemoteJWKS,
providerName: "auth0",
expected: &envoy_http_jwt_authn_v3.JwtProvider_RemoteJwks{
RemoteJwks: &envoy_http_jwt_authn_v3.RemoteJwks{
HttpUri: &envoy_core_v3.HttpUri{
Uri: oktaRemoteJWKS.URI,
HttpUpstreamType: &envoy_core_v3.HttpUri_Cluster{Cluster: "jwks_cluster"},
HttpUpstreamType: &envoy_core_v3.HttpUri_Cluster{Cluster: "auth0"},
Timeout: &durationpb.Duration{Seconds: 1},
},
AsyncFetch: &envoy_http_jwt_authn_v3.JwksAsyncFetch{
Expand All @@ -539,12 +541,13 @@ func TestMakeRemoteJWKS(t *testing.T) {
},
},
"with-retry-policy": {
jwks: extendedRemoteJWKS,
jwks: extendedRemoteJWKS,
providerName: "okta",
expected: &envoy_http_jwt_authn_v3.JwtProvider_RemoteJwks{
RemoteJwks: &envoy_http_jwt_authn_v3.RemoteJwks{
HttpUri: &envoy_core_v3.HttpUri{
Uri: oktaRemoteJWKS.URI,
HttpUpstreamType: &envoy_core_v3.HttpUri_Cluster{Cluster: "jwks_cluster"},
HttpUpstreamType: &envoy_core_v3.HttpUri_Cluster{Cluster: "okta"},
Timeout: &durationpb.Duration{Seconds: 1},
},
AsyncFetch: &envoy_http_jwt_authn_v3.JwksAsyncFetch{
Expand All @@ -560,7 +563,7 @@ func TestMakeRemoteJWKS(t *testing.T) {
for name, tt := range tests {
tt := tt
t.Run(name, func(t *testing.T) {
res := makeRemoteJWKS(tt.jwks)
res := makeRemoteJWKS(tt.jwks, tt.providerName)
require.Equal(t, res, tt.expected)
})
}
Expand Down
2 changes: 1 addition & 1 deletion agent/xds/testdata/jwt_authn/intention-with-path.golden
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"remoteJwks": {
"httpUri": {
"uri": "https://example-okta.com/.well-known/jwks.json",
"cluster": "jwks_cluster",
"cluster": "okta",
"timeout": "1s"
},
"asyncFetch": {
Expand Down
6 changes: 3 additions & 3 deletions agent/xds/testdata/jwt_authn/multiple-providers-and-one-permission.golden
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
"remoteJwks": {
"httpUri": {
"uri": "https://example-okta.com/.well-known/jwks.json",
"cluster": "jwks_cluster",
"cluster": "okta",
"timeout": "1s"
},
"asyncFetch": {
Expand All @@ -23,7 +23,7 @@
"remoteJwks": {
"httpUri": {
"uri": "https://example-okta.com/.well-known/jwks.json",
"cluster": "jwks_cluster",
"cluster": "okta",
"timeout": "1s"
},
"asyncFetch": {
Expand All @@ -37,7 +37,7 @@
"remoteJwks": {
"httpUri": {
"uri": "https://example-auth0.com/.well-known/jwks.json",
"cluster": "jwks_cluster",
"cluster": "auth0",
"timeout": "1s"
},
"asyncFetch": {
Expand Down
Loading

0 comments on commit dabfaa6

Please sign in to comment.