diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a918790cf..fadbcd64c26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ - **General:** Introduce new Azure Data Explorer Scaler ([#1488](https://github.com/kedacore/keda/issues/1488)) - **General:** Introduce new GCP Storage Scaler ([#2628](https://github.com/kedacore/keda/issues/2628)) - **General:** Introduce ARM-based container image for KEDA ([#2263](https://github.com/kedacore/keda/issues/2263) & [#2262](https://github.com/kedacore/keda/issues/2262)) +- **General:** Provide support for authentication via Azure Key Vault ([#900](https://github.com/kedacore/keda/issues/900)) ### Improvements diff --git a/apis/keda/v1alpha1/triggerauthentication_types.go b/apis/keda/v1alpha1/triggerauthentication_types.go index a071b9ca16c..888dc36b989 100644 --- a/apis/keda/v1alpha1/triggerauthentication_types.go +++ b/apis/keda/v1alpha1/triggerauthentication_types.go @@ -75,6 +75,9 @@ type TriggerAuthenticationSpec struct { // +optional HashiCorpVault *HashiCorpVault `json:"hashiCorpVault,omitempty"` + + // +optional + AzureKeyVault *AzureKeyVault `json:"azureKeyVault,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -175,6 +178,39 @@ type VaultSecret struct { Key string `json:"key"` } +// AzureKeyVault is used to authenticate using Azure Key Vault +type AzureKeyVault struct { + VaultURI string `json:"vaultUri"` + Credentials *AzureKeyVaultCredentials `json:"credentials"` + Secrets []AzureKeyVaultSecret `json:"secrets"` +} + +type AzureKeyVaultCredentials struct { + ClientID string `json:"clientId"` + ClientSecret *AzureKeyVaultClientSecret `json:"clientSecret"` + TenantID string `json:"tenantId"` +} + +type AzureKeyVaultClientSecret struct { + ValueFrom ValueFromSecret `json:"valueFrom"` +} + +type ValueFromSecret struct { + SecretKeyRef SecretKeyRef `json:"secretKeyRef"` +} + +type SecretKeyRef struct { + Name string `json:"name"` + Key string `json:"key"` +} + +type AzureKeyVaultSecret struct { + Parameter string `json:"parameter"` + Name string `json:"name"` + // +optional + Version string `json:"version,omitempty"` +} + func init() { SchemeBuilder.Register(&ClusterTriggerAuthentication{}, &ClusterTriggerAuthenticationList{}) SchemeBuilder.Register(&TriggerAuthentication{}, &TriggerAuthenticationList{}) diff --git a/apis/keda/v1alpha1/zz_generated.deepcopy.go b/apis/keda/v1alpha1/zz_generated.deepcopy.go index d6e286bf9e8..d22a3a5ab1f 100644 --- a/apis/keda/v1alpha1/zz_generated.deepcopy.go +++ b/apis/keda/v1alpha1/zz_generated.deepcopy.go @@ -92,6 +92,82 @@ func (in *AuthSecretTargetRef) DeepCopy() *AuthSecretTargetRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureKeyVault) DeepCopyInto(out *AzureKeyVault) { + *out = *in + if in.Credentials != nil { + in, out := &in.Credentials, &out.Credentials + *out = new(AzureKeyVaultCredentials) + (*in).DeepCopyInto(*out) + } + if in.Secrets != nil { + in, out := &in.Secrets, &out.Secrets + *out = make([]AzureKeyVaultSecret, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureKeyVault. +func (in *AzureKeyVault) DeepCopy() *AzureKeyVault { + if in == nil { + return nil + } + out := new(AzureKeyVault) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureKeyVaultClientSecret) DeepCopyInto(out *AzureKeyVaultClientSecret) { + *out = *in + out.ValueFrom = in.ValueFrom +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureKeyVaultClientSecret. +func (in *AzureKeyVaultClientSecret) DeepCopy() *AzureKeyVaultClientSecret { + if in == nil { + return nil + } + out := new(AzureKeyVaultClientSecret) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureKeyVaultCredentials) DeepCopyInto(out *AzureKeyVaultCredentials) { + *out = *in + if in.ClientSecret != nil { + in, out := &in.ClientSecret, &out.ClientSecret + *out = new(AzureKeyVaultClientSecret) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureKeyVaultCredentials. +func (in *AzureKeyVaultCredentials) DeepCopy() *AzureKeyVaultCredentials { + if in == nil { + return nil + } + out := new(AzureKeyVaultCredentials) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AzureKeyVaultSecret) DeepCopyInto(out *AzureKeyVaultSecret) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureKeyVaultSecret. +func (in *AzureKeyVaultSecret) DeepCopy() *AzureKeyVaultSecret { + if in == nil { + return nil + } + out := new(AzureKeyVaultSecret) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterTriggerAuthentication) DeepCopyInto(out *ClusterTriggerAuthentication) { *out = *in @@ -684,6 +760,21 @@ func (in *ScalingStrategy) DeepCopy() *ScalingStrategy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretKeyRef) DeepCopyInto(out *SecretKeyRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretKeyRef. +func (in *SecretKeyRef) DeepCopy() *SecretKeyRef { + if in == nil { + return nil + } + out := new(SecretKeyRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TriggerAuthentication) DeepCopyInto(out *TriggerAuthentication) { *out = *in @@ -765,6 +856,11 @@ func (in *TriggerAuthenticationSpec) DeepCopyInto(out *TriggerAuthenticationSpec *out = new(HashiCorpVault) (*in).DeepCopyInto(*out) } + if in.AzureKeyVault != nil { + in, out := &in.AzureKeyVault, &out.AzureKeyVault + *out = new(AzureKeyVault) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TriggerAuthenticationSpec. @@ -777,6 +873,22 @@ func (in *TriggerAuthenticationSpec) DeepCopy() *TriggerAuthenticationSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ValueFromSecret) DeepCopyInto(out *ValueFromSecret) { + *out = *in + out.SecretKeyRef = in.SecretKeyRef +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ValueFromSecret. +func (in *ValueFromSecret) DeepCopy() *ValueFromSecret { + if in == nil { + return nil + } + out := new(ValueFromSecret) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VaultSecret) DeepCopyInto(out *VaultSecret) { *out = *in diff --git a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml index c871e853d2c..f6e472206c6 100644 --- a/config/crd/bases/keda.sh_clustertriggerauthentications.yaml +++ b/config/crd/bases/keda.sh_clustertriggerauthentications.yaml @@ -53,6 +53,62 @@ spec: spec: description: TriggerAuthenticationSpec defines the various ways to authenticate properties: + azureKeyVault: + description: AzureKeyVault is used to authenticate using Azure Key + Vault + properties: + credentials: + properties: + clientId: + type: string + clientSecret: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + tenantId: + type: string + required: + - clientId + - clientSecret + - tenantId + type: object + secrets: + items: + properties: + name: + type: string + parameter: + type: string + version: + type: string + required: + - name + - parameter + type: object + type: array + vaultUri: + type: string + required: + - credentials + - secrets + - vaultUri + type: object env: items: description: AuthEnvironment is used to authenticate using environment diff --git a/config/crd/bases/keda.sh_triggerauthentications.yaml b/config/crd/bases/keda.sh_triggerauthentications.yaml index eccd7862528..8729e9c2d5a 100644 --- a/config/crd/bases/keda.sh_triggerauthentications.yaml +++ b/config/crd/bases/keda.sh_triggerauthentications.yaml @@ -52,6 +52,62 @@ spec: spec: description: TriggerAuthenticationSpec defines the various ways to authenticate properties: + azureKeyVault: + description: AzureKeyVault is used to authenticate using Azure Key + Vault + properties: + credentials: + properties: + clientId: + type: string + clientSecret: + properties: + valueFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - key + - name + type: object + required: + - secretKeyRef + type: object + required: + - valueFrom + type: object + tenantId: + type: string + required: + - clientId + - clientSecret + - tenantId + type: object + secrets: + items: + properties: + name: + type: string + parameter: + type: string + version: + type: string + required: + - name + - parameter + type: object + type: array + vaultUri: + type: string + required: + - credentials + - secrets + - vaultUri + type: object env: items: description: AuthEnvironment is used to authenticate using environment diff --git a/pkg/scaling/resolver/azure_keyvault_handler.go b/pkg/scaling/resolver/azure_keyvault_handler.go new file mode 100644 index 00000000000..cd3b2e9ac33 --- /dev/null +++ b/pkg/scaling/resolver/azure_keyvault_handler.go @@ -0,0 +1,76 @@ +/* +Copyright 2022 The KEDA Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resolver + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault" + "github.com/Azure/go-autorest/autorest/azure/auth" + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/client" + + kedav1alpha1 "github.com/kedacore/keda/v2/apis/keda/v1alpha1" +) + +const ( + azureKeyVaultResource = "https://vault.azure.net" +) + +type AzureKeyVaultHandler struct { + vault *kedav1alpha1.AzureKeyVault + keyvaultClient *keyvault.BaseClient +} + +func NewAzureKeyVaultHandler(v *kedav1alpha1.AzureKeyVault) *AzureKeyVaultHandler { + return &AzureKeyVaultHandler{ + vault: v, + } +} + +func (vh *AzureKeyVaultHandler) Initialize(ctx context.Context, client client.Client, logger logr.Logger, triggerNamespace string) error { + clientID := vh.vault.Credentials.ClientID + tenantID := vh.vault.Credentials.TenantID + + clientSecretName := vh.vault.Credentials.ClientSecret.ValueFrom.SecretKeyRef.Name + clientSecretKey := vh.vault.Credentials.ClientSecret.ValueFrom.SecretKeyRef.Key + clientSecret := resolveAuthSecret(ctx, client, logger, clientSecretName, triggerNamespace, clientSecretKey) + + clientCredentialsConfig := auth.NewClientCredentialsConfig(clientID, clientSecret, tenantID) + clientCredentialsConfig.Resource = azureKeyVaultResource + + authorizer, err := clientCredentialsConfig.Authorizer() + if err != nil { + return err + } + + keyvaultClient := keyvault.New() + keyvaultClient.Authorizer = authorizer + + vh.keyvaultClient = &keyvaultClient + + return nil +} + +func (vh *AzureKeyVaultHandler) Read(ctx context.Context, secretName string, version string) (string, error) { + result, err := vh.keyvaultClient.GetSecret(ctx, vh.vault.VaultURI, secretName, version) + if err != nil { + return "", err + } + + return *result.Value, nil +} diff --git a/pkg/scaling/resolver/scale_resolvers.go b/pkg/scaling/resolver/scale_resolvers.go index c30a32c1d92..a0015a117a1 100644 --- a/pkg/scaling/resolver/scale_resolvers.go +++ b/pkg/scaling/resolver/scale_resolvers.go @@ -210,6 +210,23 @@ func resolveAuthRef(ctx context.Context, client client.Client, logger logr.Logge vault.Stop() } } + if triggerAuthSpec.AzureKeyVault != nil && len(triggerAuthSpec.AzureKeyVault.Secrets) > 0 { + vaultHandler := NewAzureKeyVaultHandler(triggerAuthSpec.AzureKeyVault) + err := vaultHandler.Initialize(ctx, client, logger, triggerNamespace) + if err != nil { + logger.Error(err, "Error authenticating to Azure Key Vault", "triggerAuthRef.Name", triggerAuthRef.Name) + } else { + for _, secret := range triggerAuthSpec.AzureKeyVault.Secrets { + res, err := vaultHandler.Read(ctx, secret.Name, secret.Version) + if err != nil { + logger.Error(err, "Error trying to read secret from Azure Key Vault", "triggerAuthRef.Name", triggerAuthRef.Name, + "secret.Name", secret.Name, "secret.Version", secret.Version) + } else { + result[secret.Parameter] = res + } + } + } + } } } diff --git a/tests/.env b/tests/.env index 4928a6315e5..fa3a0c15bb9 100644 --- a/tests/.env +++ b/tests/.env @@ -16,3 +16,4 @@ AZURE_DEVOPS_PAT= AZURE_DEVOPS_PROJECT= AZURE_DEVOPS_BUILD_DEFINITON_ID= AZURE_DEVOPS_POOL_NAME= +AZURE_KEYVAULT_URI= diff --git a/tests/scalers/azure-keyvault-queue.test.ts b/tests/scalers/azure-keyvault-queue.test.ts new file mode 100644 index 00000000000..b2b195a9454 --- /dev/null +++ b/tests/scalers/azure-keyvault-queue.test.ts @@ -0,0 +1,182 @@ +import * as async from 'async' +import * as azure from 'azure-storage' +import * as fs from 'fs' +import * as sh from 'shelljs' +import * as tmp from 'tmp' +import test from 'ava' +import {createNamespace, waitForDeploymentReplicaCount} from "./helpers"; + +const testNamespace = 'azure-keyvault-queue-test' +const queueName = 'queue-name-trigger' +const connectionString = process.env['AZURE_STORAGE_CONNECTION_STRING'] +const keyvaultURI = process.env['AZURE_KEYVAULT_URI'] +const azureADClientID = process.env['AZURE_SP_ID'] +const azureADClientSecret = process.env['AZURE_SP_KEY'] +const azureADTenantID = process.env['AZURE_SP_TENANT'] + +test.before(async t => { + if (!connectionString) { + t.fail('AZURE_STORAGE_CONNECTION_STRING environment variable is required for keyvault tests') + } + + if (!keyvaultURI) { + t.fail('AZURE_KEYVAULT_URI environment variable is required for keyvault tests') + } + + if (!azureADClientID) { + t.fail('AZURE_SP_ID environment variable is required for keyvault tests') + } + + if (!azureADClientSecret) { + t.fail('AZURE_SP_KEY environment variable is required for keyvault tests') + } + + if (!azureADTenantID) { + t.fail('AZURE_SP_TENANT environment variable is required for keyvault tests') + } + + const createQueueAsync = () => new Promise((resolve, _) => { + const queueSvc = azure.createQueueService(connectionString) + queueSvc.messageEncoder = new azure.QueueMessageEncoder.TextBase64QueueMessageEncoder() + queueSvc.createQueueIfNotExists(queueName, _ => { + resolve(undefined); + }) + }) + await createQueueAsync() + + sh.config.silent = true + const base64ConStr = Buffer.from(connectionString).toString('base64') + const base64ClientSecret = Buffer.from(azureADClientSecret).toString('base64') + + const tmpFile = tmp.fileSync() + fs.writeFileSync(tmpFile.name, deployYaml.replace(/{{CONNECTION_STRING_BASE64}}/g, base64ConStr) + .replace(/{{CLIENT_SECRET_BASE64}}/g, base64ClientSecret)) + + createNamespace(testNamespace) + t.is( + 0, + sh.exec(`kubectl apply -f ${tmpFile.name} --namespace ${testNamespace}`).code, + 'creating a deployment should work.' + ) + t.true(await waitForDeploymentReplicaCount(0, 'test-deployment', testNamespace, 60, 1000), 'replica count should be 0 after 1 minute') +}) + +test.serial( + 'Deployment should scale with messages on storage defined through trigger auth', + async t => { + const queueSvc = azure.createQueueService(connectionString) + queueSvc.messageEncoder = new azure.QueueMessageEncoder.TextBase64QueueMessageEncoder() + await async.mapLimit( + Array(1000).keys(), + 20, + (n, cb) => queueSvc.createMessage(queueName, `test ${n}`, cb) + ) + + // Scaling out when messages available + t.true(await waitForDeploymentReplicaCount(1, 'test-deployment', testNamespace, 60, 1000), 'replica count should be 1 after 1 minute') + + queueSvc.clearMessages(queueName, _ => {}) + + // Scaling in when no available messages + t.true(await waitForDeploymentReplicaCount(0, 'test-deployment', testNamespace, 300, 1000), 'replica count should be 0 after 5 minute') + } +) + +test.after.always.cb('clean up azure-queue deployment', t => { + const resources = [ + 'scaledobject.keda.sh/test-scaledobject', + 'triggerauthentications.keda.sh/azure-queue-auth', + 'secret/test-auth-secrets', + 'deployment.apps/test-deployment', + ] + + for (const resource of resources) { + sh.exec(`kubectl delete ${resource} --namespace ${testNamespace}`) + } + sh.exec(`kubectl delete namespace ${testNamespace}`) + + // delete test queue + const queueSvc = azure.createQueueService(connectionString) + queueSvc.deleteQueueIfExists(queueName, err => { + t.falsy(err, 'should delete test queue successfully') + t.end() + }) +}) + +const deployYaml = `apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-deployment + labels: + app: test-deployment +spec: + replicas: 1 + selector: + matchLabels: + app: test-deployment + template: + metadata: + name: + namespace: + labels: + app: test-deployment + spec: + containers: + - name: test-deployment + image: docker.io/kedacore/tests-azure-queue:824031e + resources: + ports: + env: + - name: FUNCTIONS_WORKER_RUNTIME + value: node + - name: AzureWebJobsStorage + valueFrom: + secretKeyRef: + name: test-auth-secrets + key: connectionString +--- +apiVersion: v1 +kind: Secret +metadata: + name: test-auth-secrets + labels: +data: + connectionString: {{CONNECTION_STRING_BASE64}} + clientSecret: {{CLIENT_SECRET_BASE64}} +--- +apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: azure-keyvault-auth +spec: + azureKeyVault: + vaultUri: ${keyvaultURI} + credentials: + clientId: ${azureADClientID} + tenantId: ${azureADTenantID} + clientSecret: + valueFrom: + secretKeyRef: + name: test-auth-secrets + key: clientSecret + secrets: + - parameter: connection + name: E2E-Storage-ConnectionString +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: test-scaledobject +spec: + scaleTargetRef: + name: test-deployment + pollingInterval: 5 + cooldownPeriod: 10 + minReplicaCount: 0 + maxReplicaCount: 1 + triggers: + - type: azure-queue + authenticationRef: + name: azure-keyvault-auth + metadata: + queueName: ${queueName}`