From a2d2828e31be4beaa92d64106d4a9b47ff298b69 Mon Sep 17 00:00:00 2001 From: Jialun Cai Date: Thu, 28 Mar 2024 13:02:49 +0800 Subject: [PATCH] Introduce KeyVaultCredential --- pkg/azclient/arm_conf.go | 4 + pkg/azclient/armauth/auxiliary_auth_policy.go | 59 ++++++++++ pkg/azclient/armauth/keyvault_credential.go | 102 ++++++++++++++++++ pkg/azclient/auth.go | 38 +++++-- pkg/azclient/auth_conf.go | 8 ++ pkg/azclient/go.mod | 5 + pkg/azclient/go.sum | 5 + 7 files changed, 214 insertions(+), 7 deletions(-) create mode 100644 pkg/azclient/armauth/auxiliary_auth_policy.go create mode 100644 pkg/azclient/armauth/keyvault_credential.go diff --git a/pkg/azclient/arm_conf.go b/pkg/azclient/arm_conf.go index e533cf6696..d2e654c85f 100644 --- a/pkg/azclient/arm_conf.go +++ b/pkg/azclient/arm_conf.go @@ -62,3 +62,7 @@ func GetAzCoreClientOption(armConfig *ARMClientConfig) (*policy.ClientOptions, e } return &azCoreClientConfig, nil } + +func IsMultiTenant(armConfig *ARMClientConfig) bool { + return armConfig != nil && armConfig.NetworkResourceTenantID != "" && !strings.EqualFold(armConfig.NetworkResourceTenantID, armConfig.GetTenantID()) +} diff --git a/pkg/azclient/armauth/auxiliary_auth_policy.go b/pkg/azclient/armauth/auxiliary_auth_policy.go new file mode 100644 index 0000000000..0af24a8bf9 --- /dev/null +++ b/pkg/azclient/armauth/auxiliary_auth_policy.go @@ -0,0 +1,59 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package armauth + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" +) + +const ( + HeaderAuthorizationAuxiliary = "x-ms-authorization-auxiliary" +) + +type AuxiliaryAuthPolicy struct { + credentials []azcore.TokenCredential + scope string +} + +func NewAuxiliaryAuthPolicy(credentials []azcore.TokenCredential, scope string) *AuxiliaryAuthPolicy { + return &AuxiliaryAuthPolicy{ + credentials: credentials, + scope: scope, + } +} + +func (p *AuxiliaryAuthPolicy) Do(req *policy.Request) (*http.Response, error) { + tokens := make([]string, 0, len(p.credentials)) + + for _, cred := range p.credentials { + token, err := cred.GetToken(context.TODO(), policy.TokenRequestOptions{ + Scopes: []string{p.scope}, + }) + if err != nil { + return nil, err + } + tokens = append(tokens, fmt.Sprintf("Bearer %s", token.Token)) + } + req.Raw().Header.Set(HeaderAuthorizationAuxiliary, strings.Join(tokens, ", ")) + return req.Next() +} diff --git a/pkg/azclient/armauth/keyvault_credential.go b/pkg/azclient/armauth/keyvault_credential.go new file mode 100644 index 0000000000..13352d729b --- /dev/null +++ b/pkg/azclient/armauth/keyvault_credential.go @@ -0,0 +1,102 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package armauth + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets" +) + +type KeyVaultCredential struct { + secretClient *azsecrets.Client + secretPath string + + token *azcore.AccessToken +} + +type KeyVaultCredentialSecret struct { + AccessToken string `json:"access_token"` + ExpiresOn time.Time `json:"expires_on"` +} + +func NewKeyVaultCredential( + msiCredential azcore.TokenCredential, + keyVaultURL string, + secretName string, +) (*KeyVaultCredential, error) { + cli, err := azsecrets.NewClient(keyVaultURL, msiCredential, nil) + if err != nil { + return nil, fmt.Errorf("create KeyVault client: %w", err) + } + + rv := &KeyVaultCredential{ + secretClient: cli, + secretPath: secretName, + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := rv.refreshToken(ctx); err != nil { + return nil, fmt.Errorf("refresh token: %w", err) + } + + return rv, nil +} + +func (c *KeyVaultCredential) refreshToken(ctx context.Context) error { + const LatestVersion = "" + + resp, err := c.secretClient.GetSecret(ctx, c.secretPath, LatestVersion, nil) + if err != nil { + return err + } + if resp.Value == nil { + return fmt.Errorf("secret value is nil") + } + + var secret KeyVaultCredentialSecret + if err := json.Unmarshal([]byte(*resp.Value), &secret); err != nil { + return fmt.Errorf("unmarshal secret value `%s`: %w", *resp.Value, err) + } + + c.token = &azcore.AccessToken{ + Token: secret.AccessToken, + ExpiresOn: secret.ExpiresOn, + } + + return nil +} + +func (c *KeyVaultCredential) GetToken(ctx context.Context, opts policy.TokenRequestOptions) (azcore.AccessToken, error) { + const RefreshTokenOffset = 5 * time.Minute + + if c.token != nil && c.token.ExpiresOn.Add(RefreshTokenOffset).Before(time.Now()) { + return *c.token, nil + } + + if err := c.refreshToken(ctx); err != nil { + return azcore.AccessToken{}, fmt.Errorf("refresh token: %w", err) + } + + return *c.token, nil +} diff --git a/pkg/azclient/auth.go b/pkg/azclient/auth.go index 503efd3db9..df6316a16e 100644 --- a/pkg/azclient/auth.go +++ b/pkg/azclient/auth.go @@ -24,15 +24,21 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + + "sigs.k8s.io/cloud-provider-azure/pkg/azclient/armauth" ) type AuthProvider struct { - FederatedIdentityCredential azcore.TokenCredential - ManagedIdentityCredential azcore.TokenCredential - ClientSecretCredential azcore.TokenCredential + FederatedIdentityCredential azcore.TokenCredential + + ManagedIdentityCredential azcore.TokenCredential + ClientSecretCredential azcore.TokenCredential + ClientCertificateCredential azcore.TokenCredential + + NetworkTokenCredential azcore.TokenCredential NetworkClientSecretCredential azcore.TokenCredential - MultiTenantCredential azcore.TokenCredential - ClientCertificateCredential azcore.TokenCredential + + MultiTenantCredential azcore.TokenCredential } func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, clientOptionsMutFn ...func(option *policy.ClientOptions)) (*AuthProvider, error) { @@ -76,6 +82,20 @@ func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, client } } + var ( + networkTokenCredential azcore.TokenCredential + ) + if config.UseManagedIdentityExtension && config.AuxiliaryTokenProvider != nil && IsMultiTenant(armConfig) { + networkTokenCredential, err = armauth.NewKeyVaultCredential( + managedIdentityCredential, + config.AuxiliaryTokenProvider.KeyVaultURL, + config.AuxiliaryTokenProvider.SecretName, + ) + if err != nil { + return nil, fmt.Errorf("create KeyVaultCredential for auxiliary token provider: %w", err) + } + } + // ClientSecretCredential is used for client secret var clientSecretCredential azcore.TokenCredential var networkClientSecretCredential azcore.TokenCredential @@ -88,7 +108,7 @@ func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, client if err != nil { return nil, err } - if len(armConfig.NetworkResourceTenantID) > 0 && !strings.EqualFold(armConfig.NetworkResourceTenantID, armConfig.GetTenantID()) { + if IsMultiTenant(armConfig) { credOptions := &azidentity.ClientSecretCredentialOptions{ ClientOptions: *clientOption, } @@ -128,7 +148,7 @@ func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, client if err != nil { return nil, err } - if len(armConfig.NetworkResourceTenantID) > 0 && !strings.EqualFold(armConfig.NetworkResourceTenantID, armConfig.GetTenantID()) { + if IsMultiTenant(armConfig) { networkClientSecretCredential, err = azidentity.NewClientCertificateCredential(armConfig.NetworkResourceTenantID, config.GetAADClientID(), certificate, privateKey, credOptions) if err != nil { return nil, err @@ -150,6 +170,7 @@ func NewAuthProvider(armConfig *ARMClientConfig, config *AzureAuthConfig, client ClientSecretCredential: clientSecretCredential, ClientCertificateCredential: clientCertificateCredential, NetworkClientSecretCredential: networkClientSecretCredential, + NetworkTokenCredential: networkTokenCredential, MultiTenantCredential: multiTenantCredential, }, nil } @@ -173,6 +194,9 @@ func (factory *AuthProvider) GetNetworkAzIdentity() azcore.TokenCredential { if factory.NetworkClientSecretCredential != nil { return factory.NetworkClientSecretCredential } + if factory.NetworkTokenCredential != nil { + return factory.NetworkTokenCredential + } return nil } diff --git a/pkg/azclient/auth_conf.go b/pkg/azclient/auth_conf.go index 46a66e94f3..aa7e464f2e 100644 --- a/pkg/azclient/auth_conf.go +++ b/pkg/azclient/auth_conf.go @@ -42,6 +42,14 @@ type AzureAuthConfig struct { AADFederatedTokenFile string `json:"aadFederatedTokenFile,omitempty" yaml:"aadFederatedTokenFile,omitempty"` // Use workload identity federation for the virtual machine to access Azure ARM APIs UseFederatedWorkloadIdentityExtension bool `json:"useFederatedWorkloadIdentityExtension,omitempty" yaml:"useFederatedWorkloadIdentityExtension,omitempty"` + // Auxiliary token provider for accessing resources from network tenant + // Require MSI to be enabled and have permission to access the KeyVault + AuxiliaryTokenProvider *AzureAuthAuxiliaryTokenProvider `json:"auxiliaryTokenProvider,omitempty" yaml:"auxiliaryTokenProvider,omitempty"` +} + +type AzureAuthAuxiliaryTokenProvider struct { + KeyVaultURL string `json:"keyVaultURL,omitempty" yaml:"keyVaultURL,omitempty"` + SecretName string `json:"secretName" yaml:"secretName"` } func (config *AzureAuthConfig) GetAADClientID() string { diff --git a/pkg/azclient/go.mod b/pkg/azclient/go.mod index 97c9d92b0f..b16777d8ac 100644 --- a/pkg/azclient/go.mod +++ b/pkg/azclient/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 + github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.8.0 @@ -16,6 +17,7 @@ require ( github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.17.1 github.com/onsi/gomega v1.32.0 + github.com/stretchr/testify v1.8.4 go.uber.org/mock v0.4.0 golang.org/x/crypto v0.21.0 golang.org/x/sync v0.6.0 @@ -26,7 +28,9 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/golang-jwt/jwt/v5 v5.2.0 // indirect @@ -35,6 +39,7 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect golang.org/x/net v0.22.0 // indirect golang.org/x/sys v0.18.0 // indirect diff --git a/pkg/azclient/go.sum b/pkg/azclient/go.sum index 3ac7d95c78..b091ba6714 100644 --- a/pkg/azclient/go.sum +++ b/pkg/azclient/go.sum @@ -4,6 +4,10 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2fElBkcqLTOZsAajvKfnSlgBBW8dXYjw= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= +github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0 h1:ui3YNbxfW7J3tTFIZMH6LIGRjCngp+J+nIFlnizfNTE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.6.0/go.mod h1:gZmgV+qBqygoznvqo2J9oKZAFziqhLZ2xE/WVUmzkHA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.2.0 h1:DWlwvVV5r/Wy1561nZ3wrpI1/vDIBRY/Wd1HWaRBZWA= @@ -65,6 +69,7 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=