Skip to content

Commit

Permalink
Adds a test case for L7 policy with TLS
Browse files Browse the repository at this point in the history
This adds a test to check if HTTPS traffic can get inspected in a CNP.
To enable this it adds new helper functions to provision secrets,
certificates and a public CA bundle into the test setup.

Signed-off-by: Maartje Eyskens <[email protected]>
  • Loading branch information
meyskens committed Feb 21, 2023
1 parent 39d311c commit 12024c1
Show file tree
Hide file tree
Showing 8 changed files with 3,685 additions and 1 deletion.
3,293 changes: 3,293 additions & 0 deletions connectivity/check/assets/cacert.pem

Large diffs are not rendered by default.

30 changes: 30 additions & 0 deletions connectivity/check/features.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ const (

FeatureEncryptionPod Feature = "encryption-pod"
FeatureEncryptionNode Feature = "encryption-node"

FeatureSecretBackendK8s Feature = "secret-backend-k8s"
)

// FeatureStatus describes the status of a feature. Some features are either
Expand Down Expand Up @@ -230,6 +232,30 @@ func (ct *ConnectivityTest) extractFeaturesFromNodes(ctx context.Context, client
return nil
}

func (ct *ConnectivityTest) extractFeaturesFromClusterRole(ctx context.Context, client *k8s.Client, result FeatureSet) error {
cr, err := client.GetClusterRole(ctx, "cilium", metav1.GetOptions{})
if err != nil {
return err
}

hasSecretAccess := false

L:
for _, rule := range cr.Rules {
for _, resource := range rule.Resources {
if resource == "secrets" {
hasSecretAccess = true
break L
}
}
}

result[FeatureSecretBackendK8s] = FeatureStatus{
Enabled: hasSecretAccess,
}
return nil
}

func (ct *ConnectivityTest) extractFeaturesFromCiliumStatus(ctx context.Context, ciliumPod Pod, result FeatureSet) error {
stdout, err := ciliumPod.K8sClient.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name,
defaults.AgentContainerName, []string{"cilium", "status", "-o", "json"})
Expand Down Expand Up @@ -355,6 +381,10 @@ func (ct *ConnectivityTest) detectFeatures(ctx context.Context) error {
if err != nil {
return err
}
err = ct.extractFeaturesFromClusterRole(ctx, ciliumPod.K8sClient, features)
if err != nil {
return err
}
err = features.deriveFeatures()
if err != nil {
return err
Expand Down
124 changes: 124 additions & 0 deletions connectivity/check/secrets.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
package check

import (
"bytes"
"context"
"errors"
"fmt"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/cilium/cilium-cli/k8s"
)

// addSecrets adds one or more secret(s) resources to the test.
func (t *Test) addSecrets(secrets ...*corev1.Secret) error {
if t.secrets == nil {
t.secrets = make(map[string]*corev1.Secret)
}

for _, s := range secrets {
if s == nil {
return errors.New("cannot add nil Secret to test")
}
if s.Name == "" {
return fmt.Errorf("adding Secret with empty name to test: %v", s)
}
if _, ok := t.secrets[s.Name]; ok {
return fmt.Errorf("Secret with name %s already in test scope", s.Name)
}

t.secrets[s.Name] = s
}

return nil
}

// applySecrets applies all the test's registered secrets.
func (t *Test) applySecrets(ctx context.Context) error {
if len(t.secrets) == 0 {
return nil
}

for _, secret := range t.secrets {
for _, client := range t.Context().clients.clients() {
t.Infof("📜 Applying secret '%s' to namespace '%s'..", secret.Name, secret.Namespace)
if _, err := updateOrCreateSecret(ctx, client, secret); err != nil {
return fmt.Errorf("secret application failed: %w", err)
}
}
}

// Register a finalizer with the Test immediately to enable cleanup.
t.finalizers = append(t.finalizers, func() error {
// Use a detached context to make sure this call is not affected by
// context cancellation. This deletion needs to happen event when the
// user interrupted the program.
if err := t.deleteSecrets(context.TODO()); err != nil {
t.ciliumLogs(ctx)
return err
}

return nil
})

t.Debugf("📜 Successfully applied %d secret(s)", len(t.secrets))

return nil
}

// deleteSecrets deletes a given set of secrets from the cluster.
func (t *Test) deleteSecrets(ctx context.Context) error {
if len(t.secrets) == 0 {
return nil
}

// Delete all the Test's secrers from all clients.
for _, secret := range t.secrets {
t.Infof("📜 Deleting secret '%s' from namespace '%s'..", secret.Name, secret.Namespace)
for _, client := range t.Context().clients.clients() {
if err := deleteSecret(ctx, client, secret); err != nil {
return fmt.Errorf("deleting secret: %w", err)
}
}
}

t.Debugf("📜 Successfully deleted %d secret(s)", len(t.secrets))

return nil
}

func updateOrCreateSecret(ctx context.Context, client *k8s.Client, secret *corev1.Secret) (bool, error) {
mod := false

if existing, err := client.GetSecret(ctx, secret.Namespace, secret.Name, metav1.GetOptions{}); err == nil {
// compare data map
if len(existing.Data) != len(secret.Data) {
mod = true
} else {
for k, v := range existing.Data {
if v2, ok := secret.Data[k]; !ok || !bytes.Equal(v, v2) {
mod = true
break
}
}
}

_, err = client.UpdateSecret(ctx, secret.Namespace, secret, metav1.UpdateOptions{})
return mod, err
}

// Creating, so a resource will definitely be modified.
mod = true
_, err := client.CreateSecret(ctx, secret.Namespace, secret, metav1.CreateOptions{})
return mod, err
}

func deleteSecret(ctx context.Context, client *k8s.Client, secret *corev1.Secret) error {
if err := client.DeleteSecret(ctx, secret.Namespace, secret.Name, metav1.DeleteOptions{}); err != nil {
return fmt.Errorf("%s/%s/%s secret delete failed: %w", client.ClusterName(), secret.Namespace, secret.Name, err)
}

return nil
}
117 changes: 116 additions & 1 deletion connectivity/check/test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package check
import (
"bytes"
"context"
_ "embed"
"fmt"
"io"
"sync"
Expand All @@ -16,6 +17,17 @@ import (

"github.com/cilium/cilium-cli/defaults"
"github.com/cilium/cilium-cli/sysdump"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/cloudflare/cfssl/cli/genkey"
"github.com/cloudflare/cfssl/config"
"github.com/cloudflare/cfssl/csr"
"github.com/cloudflare/cfssl/helpers"
"github.com/cloudflare/cfssl/initca"
"github.com/cloudflare/cfssl/signer"
"github.com/cloudflare/cfssl/signer/local"
)

const (
Expand All @@ -30,6 +42,11 @@ const (
anySourceLabelPrefix = "any."
)

var (
//go:embed assets/cacert.pem
caBundle []byte
)

type Test struct {
// Reference to the enclosing test suite for logging etc.
ctx *ConnectivityTest
Expand Down Expand Up @@ -57,6 +74,9 @@ type Test struct {
// Policies active during this test.
cnps map[string]*ciliumv2.CiliumNetworkPolicy

// Secrets that have to be present during the test.
secrets map[string]*corev1.Secret

expectFunc ExpectationsFunc

// Start time of the test.
Expand Down Expand Up @@ -107,9 +127,15 @@ func (t *Test) Context() *ConnectivityTest {
return t.ctx
}

// setup sets up the environment for the Test to execute in, like applying CNPs.
// setup sets up the environment for the Test to execute in, like applying secrets and CNPs.
func (t *Test) setup(ctx context.Context) error {

// Apply Secrets to the cluster.
if err := t.applySecrets(ctx); err != nil {
t.ciliumLogs(ctx)
return fmt.Errorf("applying Secrets: %w", err)
}

// Apply CNPs to the cluster.
if err := t.applyPolicies(ctx); err != nil {
t.ciliumLogs(ctx)
Expand Down Expand Up @@ -297,6 +323,95 @@ func (t *Test) WithFeatureRequirements(reqs ...FeatureRequirement) *Test {
return t
}

// WithSecret takes a Secret and adds it to the cluster during the test
func (t *Test) WithSecret(secret *corev1.Secret) *Test {

// Change namespace of the secret to the test namespace
secret.SetNamespace(t.ctx.params.TestNamespace)

if err := t.addSecrets(secret); err != nil {
t.Fatalf("Adding secret: %s", err)
}
return t
}

// WithCABundleSecret takes a Secret and adds it to the cluster during the test
func (t *Test) WithCABundleSecret() *Test {
if len(caBundle) == 0 {
t.Fatalf("CA bundle is empty")
}

secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "cabundle",
Namespace: t.ctx.params.TestNamespace,
},
Data: map[string][]byte{
"ca.crt": caBundle,
},
}

if err := t.addSecrets(secret); err != nil {
t.Fatalf("Adding CA bundle secret: %s", err)
}
return t
}

// WithCertificate makes a secret with a certificate and adds it to the cluster
func (t *Test) WithCertificate(name, hostname string) *Test {
caCert, _, caKey, err := initca.New(&csr.CertificateRequest{
KeyRequest: csr.NewKeyRequest(),
CN: "Cilium Test CA",
})
if err != nil {
t.Fatalf("Unable to create CA: %s", err)
}

g := &csr.Generator{Validator: genkey.Validator}
csrBytes, keyBytes, err := g.ProcessRequest(&csr.CertificateRequest{
CN: hostname,
Hosts: []string{hostname},
})
if err != nil {
t.Fatalf("Unable to create CSR: %s", err)
}
parsedCa, err := helpers.ParseCertificatePEM(caCert)
if err != nil {
t.Fatalf("Unable to parse CA: %s", err)
}
caPriv, err := helpers.ParsePrivateKeyPEM(caKey)
if err != nil {
t.Fatalf("Unable to parse CA key: %s", err)
}

signConf := &config.Signing{
Default: &config.SigningProfile{
Expiry: 365 * 24 * time.Hour,
Usage: []string{"key encipherment", "server auth", "digital signature"},
},
}

s, err := local.NewSigner(caPriv, parsedCa, signer.DefaultSigAlgo(caPriv), signConf)
if err != nil {
t.Fatalf("Unable to create signer: %s", err)
}
certBytes, err := s.Sign(signer.SignRequest{Request: string(csrBytes)})
if err != nil {
t.Fatalf("Unable to sign certificate: %s", err)
}

return t.WithSecret(&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Type: corev1.SecretTypeTLS,
Data: map[string][]byte{
corev1.TLSCertKey: certBytes,
corev1.TLSPrivateKeyKey: keyBytes,
},
})
}

// NewAction creates a new Action. s must be the Scenario the Action is created
// for, name should be a visually-distinguishable name, src is the execution
// Pod of the action, and dst is the network target the Action will connect to.
Expand Down
39 changes: 39 additions & 0 deletions connectivity/manifests/client-egress-l7-tls.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
apiVersion: "cilium.io/v2"
kind: CiliumNetworkPolicy
metadata:
name: "l7-policy-tls"
specs:
- description: "L7 policy with TLS"
endpointSelector:
matchLabels:
kind: client
egress:
# Allow DNS
- toPorts:
- ports:
- port: "53"
protocol: ANY
rules:
dns:
- matchPattern: "*"
# Allow HTTPS when X-Very-Secret-Token is set
- toFQDNs:
- matchName: "{{.ExternalTarget}}"
toPorts:
- ports:
- port: "443"
protocol: "TCP"
terminatingTLS:
secret:
namespace: "{{.TestNamespace}}"
name: externaltarget-tls # internal certificate to terminate in cluster
originatingTLS:
secret:
namespace: "{{.TestNamespace}}"
name: cabundle # public CA bundle to validate external target
rules:
http:
- method: "GET"
path: "/"
headers:
- "X-Very-Secret-Token: 42"
Loading

0 comments on commit 12024c1

Please sign in to comment.