Skip to content

Commit

Permalink
Adds a test case for L7 policy with TLS
Browse files Browse the repository at this point in the history
This adds a test to check if HTTPS traffic can get inspected in a CNP.
To enable this it adds new helper functions to provision secrets,
certificates and a public CA bundle into the test setup.
It also adds the functionality to insert a file into a pod,
this is done for having the certificate chain checked in curl.

Signed-off-by: Maartje Eyskens <[email protected]>
  • Loading branch information
meyskens committed Feb 28, 2023
1 parent fc5860b commit ea3ec51
Show file tree
Hide file tree
Showing 9 changed files with 3,747 additions and 1 deletion.
25 changes: 25 additions & 0 deletions connectivity/check/action.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package check

import (
"context"
"encoding/base64"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -199,6 +200,30 @@ func (a *Action) fail() {
a.failed = true
}

// WriteDataToPod writes data to a file in the source pod
// It does this by using a shell command, writing huge files should be avoided
func (a *Action) WriteDataToPod(ctx context.Context, filePath string, data []byte) {
if err := ctx.Err(); err != nil {
a.Fatal("Skipping writing data to pod:", ctx.Err())
}

// Encode data to base64 to be decoded in the shell
encodedData := base64.StdEncoding.EncodeToString(data)

if a.src == nil {
a.Fatalf("No source Pod to add file to: %s", filePath)
}
pod := a.src

output, err := pod.K8sClient.ExecInPod(ctx,
pod.Pod.Namespace, pod.Pod.Name, pod.Pod.Labels["name"],
[]string{"sh", "-c", fmt.Sprintf("echo %s | base64 -d > %s", encodedData, filePath)})

if err != nil {
a.Fatalf("Writing data to pod failed: %s: %s", err, output.String())
}
}

func (a *Action) ExecInPod(ctx context.Context, cmd []string) {
if err := ctx.Err(); err != nil {
a.Fatal("Skipping command execution:", ctx.Err())
Expand Down
3,293 changes: 3,293 additions & 0 deletions connectivity/check/assets/cacert.pem

Large diffs are not rendered by default.

30 changes: 30 additions & 0 deletions connectivity/check/features.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ const (
FeatureIPv6 Feature = "ipv6"

FeatureFlavor Feature = "flavor"

FeatureSecretBackendK8s Feature = "secret-backend-k8s"
)

// FeatureStatus describes the status of a feature. Some features are either
Expand Down Expand Up @@ -249,6 +251,30 @@ func (ct *ConnectivityTest) extractFeaturesFromNodes(ctx context.Context, client
return nil
}

func (ct *ConnectivityTest) extractFeaturesFromClusterRole(ctx context.Context, client *k8s.Client, result FeatureSet) error {
cr, err := client.GetClusterRole(ctx, defaults.AgentClusterRoleName, metav1.GetOptions{})
if err != nil {
return err
}

hasSecretAccess := false

L:
for _, rule := range cr.Rules {
for _, resource := range rule.Resources {
if resource == "secrets" {
hasSecretAccess = true
break L
}
}
}

result[FeatureSecretBackendK8s] = FeatureStatus{
Enabled: hasSecretAccess,
}
return nil
}

func (ct *ConnectivityTest) extractFeaturesFromCiliumStatus(ctx context.Context, ciliumPod Pod, result FeatureSet) error {
stdout, err := ciliumPod.K8sClient.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name,
defaults.AgentContainerName, []string{"cilium", "status", "-o", "json"})
Expand Down Expand Up @@ -383,6 +409,10 @@ func (ct *ConnectivityTest) detectFeatures(ctx context.Context) error {
if err != nil {
return err
}
err = ct.extractFeaturesFromClusterRole(ctx, ciliumPod.K8sClient, features)
if err != nil {
return err
}
ct.extractFeaturesFromK8sCluster(ctx, features)
err = features.deriveFeatures()
if err != nil {
Expand Down
124 changes: 124 additions & 0 deletions connectivity/check/secrets.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
package check

import (
"bytes"
"context"
"errors"
"fmt"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/cilium/cilium-cli/k8s"
)

// addSecrets adds one or more secret(s) resources to the test.
func (t *Test) addSecrets(secrets ...*corev1.Secret) error {
if t.secrets == nil {
t.secrets = make(map[string]*corev1.Secret)
}

for _, s := range secrets {
if s == nil {
return errors.New("cannot add nil Secret to test")
}
if s.Name == "" {
return fmt.Errorf("cannot add Secret with empty name to test: %v", s)
}
if _, ok := t.secrets[s.Name]; ok {
return fmt.Errorf("Secret with name %s already in test scope", s.Name)
}

t.secrets[s.Name] = s
}

return nil
}

// applySecrets applies all the test's registered secrets.
func (t *Test) applySecrets(ctx context.Context) error {
if len(t.secrets) == 0 {
return nil
}

for _, secret := range t.secrets {
for _, client := range t.Context().clients.clients() {
t.Infof("📜 Applying secret '%s' to namespace '%s'..", secret.Name, secret.Namespace)
if _, err := updateOrCreateSecret(ctx, client, secret); err != nil {
return fmt.Errorf("secret application failed: %w", err)
}
}
}

// Register a finalizer with the Test immediately to enable cleanup.
t.finalizers = append(t.finalizers, func() error {
// Use a detached context to make sure this call is not affected by
// context cancellation. This deletion needs to happen event when the
// user interrupted the program.
if err := t.deleteSecrets(context.TODO()); err != nil {
t.ciliumLogs(ctx)
return err
}

return nil
})

t.Debugf("📜 Successfully applied %d secret(s)", len(t.secrets))

return nil
}

// deleteSecrets deletes a given set of secrets from the cluster.
func (t *Test) deleteSecrets(ctx context.Context) error {
if len(t.secrets) == 0 {
return nil
}

// Delete all the Test's secrers from all clients.
for _, secret := range t.secrets {
t.Infof("📜 Deleting secret '%s' from namespace '%s'..", secret.Name, secret.Namespace)
for _, client := range t.Context().clients.clients() {
if err := deleteSecret(ctx, client, secret); err != nil {
return fmt.Errorf("deleting secret: %w", err)
}
}
}

t.Debugf("📜 Successfully deleted %d secret(s)", len(t.secrets))

return nil
}

func updateOrCreateSecret(ctx context.Context, client *k8s.Client, secret *corev1.Secret) (bool, error) {
mod := false

if existing, err := client.GetSecret(ctx, secret.Namespace, secret.Name, metav1.GetOptions{}); err == nil {
// compare data map
if len(existing.Data) != len(secret.Data) {
mod = true
} else {
for k, v := range existing.Data {
if v2, ok := secret.Data[k]; !ok || !bytes.Equal(v, v2) {
mod = true
break
}
}
}

_, err = client.UpdateSecret(ctx, secret.Namespace, secret, metav1.UpdateOptions{})
return mod, err
}

// Creating, so a resource will definitely be modified.
mod = true
_, err := client.CreateSecret(ctx, secret.Namespace, secret, metav1.CreateOptions{})
return mod, err
}

func deleteSecret(ctx context.Context, client *k8s.Client, secret *corev1.Secret) error {
if err := client.DeleteSecret(ctx, secret.Namespace, secret.Name, metav1.DeleteOptions{}); err != nil {
return fmt.Errorf("%s/%s/%s secret delete failed: %w", client.ClusterName(), secret.Namespace, secret.Name, err)
}

return nil
}
130 changes: 129 additions & 1 deletion connectivity/check/test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package check
import (
"bytes"
"context"
_ "embed"
"fmt"
"io"
"sync"
Expand All @@ -16,6 +17,17 @@ import (

"github.com/cilium/cilium-cli/defaults"
"github.com/cilium/cilium-cli/sysdump"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/cloudflare/cfssl/cli/genkey"
"github.com/cloudflare/cfssl/config"
"github.com/cloudflare/cfssl/csr"
"github.com/cloudflare/cfssl/helpers"
"github.com/cloudflare/cfssl/initca"
"github.com/cloudflare/cfssl/signer"
"github.com/cloudflare/cfssl/signer/local"
)

const (
Expand All @@ -30,6 +42,11 @@ const (
anySourceLabelPrefix = "any."
)

var (
//go:embed assets/cacert.pem
caBundle []byte
)

type Test struct {
// Reference to the enclosing test suite for logging etc.
ctx *ConnectivityTest
Expand Down Expand Up @@ -57,6 +74,12 @@ type Test struct {
// Policies active during this test.
cnps map[string]*ciliumv2.CiliumNetworkPolicy

// Secrets that have to be present during the test.
secrets map[string]*corev1.Secret

// CA certificates of the certificates that have to be present during the test.
certificateCAs map[string][]byte

expectFunc ExpectationsFunc

// Start time of the test.
Expand Down Expand Up @@ -107,9 +130,15 @@ func (t *Test) Context() *ConnectivityTest {
return t.ctx
}

// setup sets up the environment for the Test to execute in, like applying CNPs.
// setup sets up the environment for the Test to execute in, like applying secrets and CNPs.
func (t *Test) setup(ctx context.Context) error {

// Apply Secrets to the cluster.
if err := t.applySecrets(ctx); err != nil {
t.ciliumLogs(ctx)
return fmt.Errorf("applying Secrets: %w", err)
}

// Apply CNPs to the cluster.
if err := t.applyPolicies(ctx); err != nil {
t.ciliumLogs(ctx)
Expand Down Expand Up @@ -297,6 +326,100 @@ func (t *Test) WithFeatureRequirements(reqs ...FeatureRequirement) *Test {
return t
}

// WithSecret takes a Secret and adds it to the cluster during the test
func (t *Test) WithSecret(secret *corev1.Secret) *Test {

// Change namespace of the secret to the test namespace
secret.SetNamespace(t.ctx.params.TestNamespace)

if err := t.addSecrets(secret); err != nil {
t.Fatalf("Adding secret: %s", err)
}
return t
}

// WithCABundleSecret makes the secret `cabundle` with a CA bundle and adds it to the cluster
func (t *Test) WithCABundleSecret() *Test {
if len(caBundle) == 0 {
t.Fatalf("CA bundle is empty")
}

secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "cabundle",
Namespace: t.ctx.params.TestNamespace,
},
Data: map[string][]byte{
"ca.crt": caBundle,
},
}

if err := t.addSecrets(secret); err != nil {
t.Fatalf("Adding CA bundle secret: %s", err)
}
return t
}

// WithCertificate makes a secret with a certificate and adds it to the cluster
func (t *Test) WithCertificate(name, hostname string) *Test {
caCert, _, caKey, err := initca.New(&csr.CertificateRequest{
KeyRequest: csr.NewKeyRequest(),
CN: "Cilium Test CA",
})
if err != nil {
t.Fatalf("Unable to create CA: %s", err)
}

g := &csr.Generator{Validator: genkey.Validator}
csrBytes, keyBytes, err := g.ProcessRequest(&csr.CertificateRequest{
CN: hostname,
Hosts: []string{hostname},
})
if err != nil {
t.Fatalf("Unable to create CSR: %s", err)
}
parsedCa, err := helpers.ParseCertificatePEM(caCert)
if err != nil {
t.Fatalf("Unable to parse CA: %s", err)
}
caPriv, err := helpers.ParsePrivateKeyPEM(caKey)
if err != nil {
t.Fatalf("Unable to parse CA key: %s", err)
}

signConf := &config.Signing{
Default: &config.SigningProfile{
Expiry: 365 * 24 * time.Hour,
Usage: []string{"key encipherment", "server auth", "digital signature"},
},
}

s, err := local.NewSigner(caPriv, parsedCa, signer.DefaultSigAlgo(caPriv), signConf)
if err != nil {
t.Fatalf("Unable to create signer: %s", err)
}
certBytes, err := s.Sign(signer.SignRequest{Request: string(csrBytes)})
if err != nil {
t.Fatalf("Unable to sign certificate: %s", err)
}

if t.certificateCAs == nil {
t.certificateCAs = make(map[string][]byte)
}
t.certificateCAs[name] = caCert

return t.WithSecret(&corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: name,
},
Type: corev1.SecretTypeTLS,
Data: map[string][]byte{
corev1.TLSCertKey: certBytes,
corev1.TLSPrivateKeyKey: keyBytes,
},
})
}

// NewAction creates a new Action. s must be the Scenario the Action is created
// for, name should be a visually-distinguishable name, src is the execution
// Pod of the action, and dst is the network target the Action will connect to.
Expand Down Expand Up @@ -370,3 +493,8 @@ func (t *Test) ForEachIPFamily(do func(IPFamily)) {
}
}
}

// CertificateCAs returns the CAs used to sign the certificates within the test.
func (t *Test) CertificateCAs() map[string][]byte {
return t.certificateCAs
}
Loading

0 comments on commit ea3ec51

Please sign in to comment.