Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

defaultSSLCertificate via key vault URI #166

Merged
merged 42 commits into from
Mar 27, 2024
Merged
Changes from all commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
4cdee56
moving changes from aamgayle/crdwatchreconciler to new verified branch
aamgayle Feb 12, 2024
2eff251
unit tests for nginx_ingress_controller and parameter adjustments
aamgayle Feb 13, 2024
28103b0
tests and fix for app-routing-system namespace
aamgayle Feb 15, 2024
ca2b0e4
keyVaultUri to pointer and associated changes
aamgayle Feb 15, 2024
506e836
Generic BuildSPC function and namespace fixes
aamgayle Feb 15, 2024
0882918
placeholder_pod refactor and additional nic kv uri e2e test
aamgayle Feb 21, 2024
78889dc
obj to nic bug fix, ing annotation change and unit test adjustment
aamgayle Feb 22, 2024
5c7fc53
changed nginx_secret_provider_class namespace to come from config
aamgayle Feb 22, 2024
fc05136
Added CEL validation for keyvault fields and properties size on Defau…
aamgayle Feb 26, 2024
5d1a82d
addressing comments and added placeholder pod check test
aamgayle Feb 28, 2024
3d7564b
Added e2e tests to confirm pulling secret from keyvault
aamgayle Mar 4, 2024
9276555
error checking for spc get
aamgayle Mar 4, 2024
5420ca3
removing unneeded upsert of nic
aamgayle Mar 4, 2024
095f446
Merge branch 'main' into aamgayle/nginxsecretprovider
aamgayle Mar 4, 2024
1af06d7
added secretproviderclass to scheme
aamgayle Mar 4, 2024
2aab948
added wait time to ensure spc is found after nic is created
aamgayle Mar 4, 2024
faa7670
addressing comments
aamgayle Mar 6, 2024
ade7a22
removed waits for spc and service in e2e test
aamgayle Mar 11, 2024
e28d215
Merge branch 'main' into aamgayle/nginxsecretprovider
OliverMKing Mar 12, 2024
cda8f47
Reverted from zap to logr.Discard
aamgayle Mar 12, 2024
0293328
Addressing comments
aamgayle Mar 20, 2024
2ae426a
Merged main
aamgayle Mar 20, 2024
6f6ab58
Merge branch 'main' into aamgayle/nginxsecretprovider
aamgayle Mar 20, 2024
e6a9959
typo fix
aamgayle Mar 20, 2024
48e6789
Addressing comments
aamgayle Mar 22, 2024
8e5b4a5
Merged main
aamgayle Mar 22, 2024
71a8772
Added tests for buildDeployment and moved buildSPC tests to kv_util_test
aamgayle Mar 22, 2024
9e3f751
ingressClassName error check test
aamgayle Mar 25, 2024
c054fb1
string fix in test
aamgayle Mar 25, 2024
3247247
Merge branch 'main' into aamgayle/nginxsecretprovider
aamgayle Mar 26, 2024
affa015
Error() and ToPtr fix
aamgayle Mar 26, 2024
53cb9e8
UserError function and better error checking
aamgayle Mar 26, 2024
4bc5095
changes for UserError
aamgayle Mar 27, 2024
3587980
addressing comments
aamgayle Mar 27, 2024
e33cf09
typo fix
aamgayle Mar 27, 2024
edf6c6e
test fixes
aamgayle Mar 27, 2024
97ca629
added logging for buildSPC error
aamgayle Mar 27, 2024
de6f986
Added logging
aamgayle Mar 27, 2024
d16cefc
Error level logs
aamgayle Mar 27, 2024
ee2f620
Addressing comments
aamgayle Mar 27, 2024
dc90a06
moved logging for placeholder_pod to switch cases
aamgayle Mar 27, 2024
3e66697
default case for placeholder_pod
aamgayle Mar 27, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions api/v1alpha1/nginxingresscontroller_types.go
Original file line number Diff line number Diff line change
@@ -61,10 +61,17 @@ type NginxIngressControllerSpec struct {
Scaling *Scaling `json:"scaling,omitempty"`
}

// DefaultSSLCertificate holds a secret in the form of a secret struct with name and namespace properties or a key vault uri
// +kubebuilder:validation:MaxProperties=1
// +kubebuilder:validation:XValidation:rule="(isURL(self.keyVaultURI) || !has(self.keyVaultURI))"
aamgayle marked this conversation as resolved.
Show resolved Hide resolved
type DefaultSSLCertificate struct {
// Secret is a struct that holds the name and namespace fields used for the default ssl secret
aamgayle marked this conversation as resolved.
Show resolved Hide resolved
// +optional
Secret *Secret `json:"secret,omitempty"`

// Secret in the form of a Key Vault URI
// +optional
KeyVaultURI *string `json:"keyVaultURI"`
}

// Secret is a struct that holds a name and namespace to be used in DefaultSSLCertificate
5 changes: 5 additions & 0 deletions api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -65,7 +65,11 @@ spec:
description: DefaultSSLCertificate defines whether the NginxIngressController
should use a certain SSL certificate by default. If this field is
omitted, no default certificate will be used.
maxProperties: 1
properties:
keyVaultURI:
description: Secret in the form of a Key Vault URI
type: string
secret:
description: Secret is a struct that holds the name and namespace
fields used for the default ssl secret
@@ -85,6 +89,8 @@ spec:
- namespace
type: object
type: object
x-kubernetes-validations:
- rule: (isURL(self.keyVaultURI) || !has(self.keyVaultURI))
ingressClassName:
default: nginx.approuting.kubernetes.azure.com
description: IngressClassName is the name of the IngressClass that
4 changes: 4 additions & 0 deletions pkg/controller/controller.go
Original file line number Diff line number Diff line change
@@ -184,6 +184,10 @@ func setupControllers(mgr ctrl.Manager, conf *config.Config, lgr logr.Logger, cl
if err := keyvault.NewIngressSecretProviderClassReconciler(mgr, conf, ingressManager); err != nil {
return fmt.Errorf("setting up ingress secret provider class reconciler: %w", err)
}
lgr.Info("setting up nginx keyvault secret provider class reconciler")
if err := keyvault.NewNginxSecretProviderClassReconciler(mgr, conf); err != nil {
return fmt.Errorf("setting up nginx secret provider class reconciler: %w", err)
}
lgr.Info("setting up keyvault placeholder pod controller")
if err := keyvault.NewPlaceholderPodController(mgr, conf, ingressManager); err != nil {
return fmt.Errorf("setting up placeholder pod controller: %w", err)
141 changes: 36 additions & 105 deletions pkg/controller/keyvault/ingress_secret_provider_class.go
Original file line number Diff line number Diff line change
@@ -5,25 +5,21 @@ package keyvault

import (
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"

"github.com/Azure/aks-app-routing-operator/pkg/config"
"github.com/Azure/aks-app-routing-operator/pkg/controller/controllername"
"github.com/Azure/aks-app-routing-operator/pkg/controller/metrics"
"github.com/Azure/aks-app-routing-operator/pkg/manifests"
"github.com/Azure/aks-app-routing-operator/pkg/util"
"github.com/go-logr/logr"
netv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
secv1 "sigs.k8s.io/secrets-store-csi-driver/apis/v1"

"github.com/Azure/aks-app-routing-operator/pkg/config"
"github.com/Azure/aks-app-routing-operator/pkg/controller/controllername"
"github.com/Azure/aks-app-routing-operator/pkg/controller/metrics"
"github.com/Azure/aks-app-routing-operator/pkg/manifests"
"github.com/Azure/aks-app-routing-operator/pkg/util"
kvcsi "github.com/Azure/secrets-store-csi-driver-provider-azure/pkg/provider/types"
)

var (
@@ -103,28 +99,45 @@ func (i *IngressSecretProviderClassReconciler) Reconcile(ctx context.Context, re
},
}
logger = logger.WithValues("spc", spc.Name)
ok, err := i.buildSPC(ing, spc)
if err != nil {
logger.Info("failed to build secret provider class for ingress, user input invalid. sending warning event")
i.events.Eventf(ing, "Warning", "InvalidInput", "error while processing Keyvault reference: %s", err)
return result, nil

// Checking if we manage the ingress. All false cases without an error are assumed that we don't manage it
var isManaged bool
if isManaged, err = i.ingressManager.IsManaging(ing); err != nil {
aamgayle marked this conversation as resolved.
Show resolved Hide resolved
logger.Error(err, fmt.Sprintf("failed while checking if ingress was managed with error: %s.", err.Error()))
return result, fmt.Errorf("determining if ingress is managed: %w", err)
}
aamgayle marked this conversation as resolved.
Show resolved Hide resolved
if ok {
logger.Info("reconciling secret provider class for ingress")
err = util.Upsert(ctx, i.client, spc)
if err != nil {
i.events.Eventf(ing, "Warning", "FailedUpdateOrCreateSPC", "error while creating or updating SecretProviderClass needed to pull Keyvault reference: %s", err)

if isManaged {
var upsertSPC bool

if upsertSPC, err = buildSPC(ing, spc, i.config); err != nil {
aamgayle marked this conversation as resolved.
Show resolved Hide resolved
aamgayle marked this conversation as resolved.
Show resolved Hide resolved
var userErr userError
if errors.As(err, &userErr) {
logger.Info(fmt.Sprintf("failed to build secret provider class for ingress with error: %s. sending warning event", userErr.Error()))
i.events.Eventf(ing, "Warning", "InvalidInput", "error while processing Keyvault reference: %s", userErr.UserError())
return result, nil
}

logger.Error(err, fmt.Sprintf("failed to build secret provider class for ingress with error: %s.", err.Error()))
return result, err
aamgayle marked this conversation as resolved.
Show resolved Hide resolved
aamgayle marked this conversation as resolved.
Show resolved Hide resolved
}

if upsertSPC {
logger.Info("reconciling secret provider class for ingress")
if err = util.Upsert(ctx, i.client, spc); err != nil {
i.events.Eventf(ing, "Warning", "FailedUpdateOrCreateSPC", "error while creating or updating SecretProviderClass needed to pull Keyvault reference: %s", err.Error())
aamgayle marked this conversation as resolved.
Show resolved Hide resolved
logger.Error(err, fmt.Sprintf("failed to upsert secret provider class for ingress with error: %s.", err.Error()))
}
return result, err
}
return result, err
}

logger.Info("cleaning unused managed spc for ingress")
logger.Info("getting secret provider class for ingress")

toCleanSPC := &secv1.SecretProviderClass{}

err = i.client.Get(ctx, client.ObjectKeyFromObject(spc), toCleanSPC)
if err != nil {
if err = i.client.Get(ctx, client.ObjectKeyFromObject(spc), toCleanSPC); err != nil {
return result, client.IgnoreNotFound(err)
}

@@ -136,85 +149,3 @@ func (i *IngressSecretProviderClassReconciler) Reconcile(ctx context.Context, re

return result, nil
}

func (i *IngressSecretProviderClassReconciler) buildSPC(ing *netv1.Ingress, spc *secv1.SecretProviderClass) (bool, error) {
if ing.Spec.IngressClassName == nil || ing.Annotations == nil {
return false, nil
}

managed, err := i.ingressManager.IsManaging(ing)
if err != nil {
return false, fmt.Errorf("determining if ingress is managed: %w", err)
}
if !managed {
return false, nil
}

certURI := ing.Annotations[tlsCertKvUriAnnotation]
if certURI == "" {
return false, nil
}

uri, err := url.Parse(certURI)
if err != nil {
return false, err
}
vaultName := strings.Split(uri.Host, ".")[0]
chunks := strings.Split(uri.Path, "/")
if len(chunks) < 3 {
return false, fmt.Errorf("invalid secret uri: %s", certURI)
}
secretName := chunks[2]
p := map[string]interface{}{
"objectName": secretName,
"objectType": "secret",
}
if len(chunks) > 3 {
p["objectVersion"] = chunks[3]
}

params, err := json.Marshal(p)
if err != nil {
return false, err
}
objects, err := json.Marshal(map[string]interface{}{"array": []string{string(params)}})
if err != nil {
return false, err
}

spc.Spec = secv1.SecretProviderClassSpec{
Provider: secv1.Provider("azure"),
SecretObjects: []*secv1.SecretObject{{
SecretName: certSecretName(ing.Name),
Type: "kubernetes.io/tls",
Data: []*secv1.SecretObjectData{
{
ObjectName: secretName,
Key: "tls.key",
},
{
ObjectName: secretName,
Key: "tls.crt",
},
},
}},
// https://azure.github.io/secrets-store-csi-driver-provider-azure/docs/getting-started/usage/#create-your-own-secretproviderclass-object
Parameters: map[string]string{
"keyvaultName": vaultName,
"useVMManagedIdentity": "true",
"userAssignedIdentityID": i.config.MSIClientID,
"tenantId": i.config.TenantID,
"objects": string(objects),
},
}

if i.config.Cloud != "" {
spc.Spec.Parameters[kvcsi.CloudNameParameter] = i.config.Cloud
}

return true, nil
}

func certSecretName(ingressName string) string {
return fmt.Sprintf("keyvault-%s", ingressName)
}
Loading