Skip to content

Commit

Permalink
store helm values into a Kubernetes secret
Browse files Browse the repository at this point in the history 
Similarly to helm behavior, Cilium-cli will store its generated yaml
values file in a secret.

Signed-off-by: André Martins <[email protected]>
  • Loading branch information
@aanm
aanm committed Apr 9, 2022
1 parent aad7a72 commit 93c64e5
Show file tree
Hide file tree
Showing 10 changed files with 115 additions and 63 deletions.
3 changes: 2 additions & 1 deletion defaults/defaults.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ const (
IngressClassName = "cilium"
IngressControllerName = "cilium.io/ingress-controller"

ExtraConfigMapUserOptsKey = "io.cilium.cilium-cli"
HelmValuesSecretName = "cilium-cli-helm-values"
HelmValuesSecretKeyName = "io.cilium.cilium-cli"
)

var (
Expand Down
75 changes: 50 additions & 25 deletions hubble/hubble.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
"github.com/spf13/pflag"
"helm.sh/helm/v3/pkg/chartutil"
"helm.sh/helm/v3/pkg/cli/values"
"helm.sh/helm/v3/pkg/strvals"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
Expand All @@ -39,6 +38,7 @@ const (

type k8sHubbleImplementation interface {
CreateSecret(ctx context.Context, namespace string, secret *corev1.Secret, opts metav1.CreateOptions) (*corev1.Secret, error)
UpdateSecret(ctx context.Context, namespace string, secret *corev1.Secret, opts metav1.UpdateOptions) (*corev1.Secret, error)
DeleteSecret(ctx context.Context, namespace, name string, opts metav1.DeleteOptions) error
GetSecret(ctx context.Context, namespace, name string, opts metav1.GetOptions) (*corev1.Secret, error)
CreateServiceAccount(ctx context.Context, namespace string, account *corev1.ServiceAccount, opts metav1.CreateOptions) (*corev1.ServiceAccount, error)
Expand Down Expand Up @@ -72,6 +72,7 @@ type K8sHubble struct {
ciliumVersion string
manifests map[string]string
semVerCiliumVersion semver.Version
helmYAMLValues string
}

var (
Expand Down Expand Up @@ -125,6 +126,10 @@ type Parameters struct {
// HelmGenValuesFile points to the file that will store the generated helm
// options.
HelmGenValuesFile string

// HelmValuesSecretName is the name of the secret where helm values will be
// stored.
HelmValuesSecretName string
}

func (p *Parameters) Log(format string, a ...interface{}) {
Expand Down Expand Up @@ -214,21 +219,25 @@ func (k *K8sHubble) Disable(ctx context.Context) error {

k.semVerCiliumVersion = k.getCiliumVersion()

cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{})
helmSecret, err := k.client.GetSecret(ctx, k.params.Namespace, k.params.HelmValuesSecretName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("unable to retrieve ConfigMap %q: %w", defaults.ConfigMapName, err)
return fmt.Errorf("unable to retrieve helm values secret %s/%s: %w", k.params.Namespace, k.params.HelmValuesSecretName, err)
}

value, ok := cm.Data[defaults.ExtraConfigMapUserOptsKey]
yamlSecret, ok := helmSecret.Data[defaults.HelmValuesSecretKeyName]
if !ok {
return fmt.Errorf("configmap option not found")
return fmt.Errorf("unable to retrieve helm values from secret %s/%s: %w", k.params.Namespace, k.params.HelmValuesSecretName, err)
}

vals, err := chartutil.ReadValues(yamlSecret)
if err != nil {
return fmt.Errorf("unable to parse helm values from secret %s/%s: %w", k.params.Namespace, k.params.HelmValuesSecretName, err)
}

// Generate the manifests has if hubble was being enabled so that we can
// retrieve all UI and Relay's resource names.
k.params.UI = true
k.params.Relay = true
err = k.generateManifestsEnable(ctx, false, value)
err = k.generateManifestsEnable(ctx, false, vals)
if err != nil {
return err
}
Expand All @@ -243,7 +252,7 @@ func (k *K8sHubble) Disable(ctx context.Context) error {

// Now that we have delete all UI and Relay's resource names then we can
// generate the manifests with UI and Relay disabled.
err = k.generateManifestsDisable(ctx, value)
err = k.generateManifestsDisable(ctx, vals)
if err != nil {
return err
}
Expand All @@ -252,6 +261,14 @@ func (k *K8sHubble) Disable(ctx context.Context) error {
return err
}

k.Log("ℹ️ Storing helm values file in %s/%s Secret", k.params.Namespace, k.params.HelmValuesSecretName)

helmSecret.Data[defaults.HelmValuesSecretKeyName] = []byte(k.helmYAMLValues)
if _, err := k.client.UpdateSecret(ctx, k.params.Namespace, helmSecret, metav1.UpdateOptions{}); err != nil {
k.Log("❌ Unable to store helm values file %s/%s Secret", k.params.Namespace, k.params.HelmValuesSecretName)
return err
}

k.Log("✅ Hubble was successfully disabled.")

return nil
Expand Down Expand Up @@ -313,7 +330,7 @@ func (k *K8sHubble) getCiliumVersion() semver.Version {
return v
}

func (k *K8sHubble) generateManifestsEnable(ctx context.Context, printHelmTemplate bool, helmValues string) error {
func (k *K8sHubble) generateManifestsEnable(ctx context.Context, printHelmTemplate bool, helmValues chartutil.Values) error {
ciliumVer := k.semVerCiliumVersion

helmMapOpts := map[string]string{}
Expand Down Expand Up @@ -369,7 +386,7 @@ func (k *K8sHubble) generateManifestsEnable(ctx context.Context, printHelmTempla
return k.genManifests(ctx, printHelmTemplate, helmValues, helmMapOpts, ciliumVer)
}

func (k *K8sHubble) generateManifestsDisable(ctx context.Context, helmValues string) error {
func (k *K8sHubble) generateManifestsDisable(ctx context.Context, helmValues chartutil.Values) error {
ciliumVer := k.semVerCiliumVersion

helmMapOpts := map[string]string{}
Expand All @@ -391,24 +408,19 @@ func (k *K8sHubble) generateManifestsDisable(ctx context.Context, helmValues str
return k.genManifests(ctx, false, helmValues, helmMapOpts, ciliumVer)
}

func (k *K8sHubble) genManifests(ctx context.Context, printHelmTemplate bool, helmValues string, helmMapOpts map[string]string, ciliumVer semver.Version) error {
func (k *K8sHubble) genManifests(ctx context.Context, printHelmTemplate bool, prevHelmValues chartutil.Values, helmMapOpts map[string]string, ciliumVer semver.Version) error {
// Store all the options passed by --config into helm extraConfig
prevHelmValues := map[string]interface{}{}
err := strvals.ParseInto(helmValues, prevHelmValues)
vals, err := helm.MergeVals(k, printHelmTemplate, k.params.HelmOpts, helmMapOpts, prevHelmValues, nil, k.params.HelmChartDirectory, ciliumVer.String(), k.params.Namespace)
if err != nil {
return fmt.Errorf("error parsing helm options %q: %w", helmValues, err)
return err
}

vals, err := helm.MergeVals(k, printHelmTemplate, k.params.HelmOpts, helmMapOpts, prevHelmValues, nil, k.params.HelmChartDirectory, ciliumVer.String(), k.params.Namespace)
yamlValue, err := chartutil.Values(vals).YAML()
if err != nil {
return err
}

if k.params.HelmGenValuesFile != "" {
yamlValue, err := chartutil.Values(vals).YAML()
if err != nil {
return err
}
return os.WriteFile(k.params.HelmGenValuesFile, []byte(yamlValue), 0o600)
}

Expand All @@ -427,6 +439,7 @@ func (k *K8sHubble) genManifests(ctx context.Context, printHelmTemplate bool, he
}

k.manifests = manifests
k.helmYAMLValues = yamlValue
return nil
}

Expand Down Expand Up @@ -462,17 +475,21 @@ func (k *K8sHubble) Enable(ctx context.Context) error {
}
}

cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{})
helmSecret, err := k.client.GetSecret(ctx, k.params.Namespace, k.params.HelmValuesSecretName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("unable to retrieve ConfigMap %q: %w", defaults.ConfigMapName, err)
return fmt.Errorf("unable to retrieve helm values secret %s/%s: %w", k.params.Namespace, k.params.HelmValuesSecretName, err)
}

value, ok := cm.Data[defaults.ExtraConfigMapUserOptsKey]
yamlSecret, ok := helmSecret.Data[defaults.HelmValuesSecretKeyName]
if !ok {
return fmt.Errorf("configmap option not found")
return fmt.Errorf("unable to retrieve helm values from secret %s/%s: %w", k.params.Namespace, k.params.HelmValuesSecretName, err)
}

err = k.generateManifestsEnable(ctx, true, value)
vals, err := chartutil.ReadValues(yamlSecret)
if err != nil {
return fmt.Errorf("unable to parse helm values from secret %s/%s: %w", k.params.Namespace, k.params.HelmValuesSecretName, err)
}

err = k.generateManifestsEnable(ctx, true, vals)
if err != nil {
return err
}
Expand Down Expand Up @@ -541,6 +558,14 @@ func (k *K8sHubble) Enable(ctx context.Context) error {
}
}

k.Log("ℹ️ Storing helm values file in %s/%s Secret", k.params.Namespace, k.params.HelmValuesSecretName)

helmSecret.Data[defaults.HelmValuesSecretKeyName] = []byte(k.helmYAMLValues)
if _, err := k.client.UpdateSecret(ctx, k.params.Namespace, helmSecret, metav1.UpdateOptions{}); err != nil {
k.Log("❌ Unable to store helm values file %s/%s Secret", k.params.Namespace, k.params.HelmValuesSecretName)
return err
}

k.Log("✅ Hubble was successfully enabled!")

return nil
Expand Down
17 changes: 11 additions & 6 deletions hubble/relay.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/cilium/cilium-cli/internal/utils"

"github.com/cilium/cilium/pkg/versioncheck"
"helm.sh/helm/v3/pkg/chartutil"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -247,21 +248,25 @@ func (k *K8sHubble) PortForwardCommand(ctx context.Context) error {

k.semVerCiliumVersion = k.getCiliumVersion()

cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{})
helmSecret, err := k.client.GetSecret(ctx, k.params.Namespace, k.params.HelmValuesSecretName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("unable to retrieve ConfigMap %q: %w", defaults.ConfigMapName, err)
return fmt.Errorf("unable to retrieve helm values secret %s/%s: %w", k.params.Namespace, k.params.HelmValuesSecretName, err)
}

value, ok := cm.Data[defaults.ExtraConfigMapUserOptsKey]
yamlSecret, ok := helmSecret.Data[defaults.HelmValuesSecretKeyName]
if !ok {
return fmt.Errorf("configmap option not found")
return fmt.Errorf("unable to retrieve helm values from secret %s/%s: %w", k.params.Namespace, k.params.HelmValuesSecretName, err)
}

vals, err := chartutil.ReadValues(yamlSecret)
if err != nil {
return fmt.Errorf("unable to parse helm values from secret %s/%s: %w", k.params.Namespace, k.params.HelmValuesSecretName, err)
}

// Generate the manifests has if hubble was being enabled so that we can
// retrieve all UI and Relay's resource names.
k.params.UI = true
k.params.Relay = true
err = k.generateManifestsEnable(ctx, false, value)
err = k.generateManifestsEnable(ctx, false, vals)
if err != nil {
return err
}
Expand Down
10 changes: 6 additions & 4 deletions install/helm.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,11 +236,12 @@ func (k *K8sInstaller) generateManifests(ctx context.Context) error {
return err
}

yamlValue, err := chartutil.Values(vals).YAML()
if err != nil {
return err
}

if k.params.HelmGenValuesFile != "" {
yamlValue, err := chartutil.Values(vals).YAML()
if err != nil {
return err
}
return os.WriteFile(k.params.HelmGenValuesFile, []byte(yamlValue), 0o600)
}

Expand All @@ -259,5 +260,6 @@ func (k *K8sInstaller) generateManifests(ctx context.Context) error {
}

k.manifests = manifests
k.helmYAMLValues = yamlValue
return nil
}
32 changes: 26 additions & 6 deletions install/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ type k8sInstallerImplementation interface {
DeletePod(ctx context.Context, namespace, name string, options metav1.DeleteOptions) error
ExecInPod(ctx context.Context, namespace, pod, container string, command []string) (bytes.Buffer, error)
CreateSecret(ctx context.Context, namespace string, secret *corev1.Secret, opts metav1.CreateOptions) (*corev1.Secret, error)
UpdateSecret(ctx context.Context, namespace string, secret *corev1.Secret, opts metav1.UpdateOptions) (*corev1.Secret, error)
DeleteSecret(ctx context.Context, namespace, name string, opts metav1.DeleteOptions) error
GetSecret(ctx context.Context, namespace, name string, opts metav1.GetOptions) (*corev1.Secret, error)
PatchSecret(ctx context.Context, namespace, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*corev1.Secret, error)
Expand All @@ -137,12 +138,13 @@ type k8sInstallerImplementation interface {
}

type K8sInstaller struct {
client k8sInstallerImplementation
params Parameters
flavor k8s.Flavor
certManager *certs.CertManager
rollbackSteps []rollbackStep
manifests map[string]string
client k8sInstallerImplementation
params Parameters
flavor k8s.Flavor
certManager *certs.CertManager
rollbackSteps []rollbackStep
manifests map[string]string
helmYAMLValues string
}

const (
Expand Down Expand Up @@ -243,6 +245,10 @@ type Parameters struct {
// ImageTag will set the tags that will be set on all docker images
// generated by cilium-cli
ImageTag string

// HelmValuesSecretName is the name of the secret where helm values will be
// stored.
HelmValuesSecretName string
}

type rollbackStep func(context.Context)
Expand Down Expand Up @@ -537,6 +543,20 @@ func (k *K8sInstaller) Install(ctx context.Context) error {
if err != nil {
return err
}
k.Log("ℹ️ Storing helm values file in %s/%s Secret", k.params.Namespace, k.params.HelmValuesSecretName)

helmSecret := k8s.NewSecret(k.params.HelmValuesSecretName, k.params.Namespace, map[string][]byte{defaults.HelmValuesSecretKeyName: []byte(k.helmYAMLValues)})
if _, err := k.client.GetSecret(ctx, k.params.Namespace, k.params.HelmValuesSecretName, metav1.GetOptions{}); err == nil {
if _, err := k.client.UpdateSecret(ctx, k.params.Namespace, helmSecret, metav1.UpdateOptions{}); err != nil {
k.Log("❌ Unable to store helm values file %s/%s Secret", k.params.Namespace, k.params.HelmValuesSecretName)
return err
}
} else {
if _, err := k.client.CreateSecret(ctx, k.params.Namespace, helmSecret, metav1.CreateOptions{}); err != nil {
k.Log("❌ Unable to store helm values file %s/%s Secret", k.params.Namespace, k.params.HelmValuesSecretName)
return err
}
}

if k.params.HelmGenValuesFile != "" {
k.Log("ℹ️ Generated helm values file %q successfully written", k.params.HelmGenValuesFile)
Expand Down
12 changes: 8 additions & 4 deletions install/uninstall.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@ import (
)

type UninstallParameters struct {
Namespace string
TestNamespace string
Writer io.Writer
Wait bool
Namespace string
TestNamespace string
Writer io.Writer
Wait bool
HelmValuesSecretName string
}

type K8sUninstaller struct {
Expand Down Expand Up @@ -116,6 +117,9 @@ func (k *K8sUninstaller) Uninstall(ctx context.Context) error {
}
}

k.Log("🔥 Deleting secret with the helm values configuration...")
k.client.DeleteSecret(ctx, k.params.Namespace, k.params.HelmValuesSecretName, metav1.DeleteOptions{})

k.Log("✅ Cilium was successfully uninstalled.")

return nil
Expand Down
3 changes: 3 additions & 0 deletions internal/cli/cmd/hubble.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ func newCmdHubbleEnable() *cobra.Command {
cmd.Flags().StringArrayVar(&params.HelmOpts.StringValues, "helm-set-string", []string{}, "Set helm STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
cmd.Flags().StringArrayVar(&params.HelmOpts.FileValues, "helm-set-file", []string{}, "Set helm values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)")
cmd.Flags().StringVar(&params.HelmGenValuesFile, "helm-auto-gen-values", "", "Write an auto-generated helm values into this file")
cmd.Flags().StringVar(&params.HelmValuesSecretName, "helm-values-secret-name", defaults.HelmValuesSecretName, "Secret name to store the auto-generated helm values file. The namespace is the same as where Cilium will be installed")

for flagName := range hubble.FlagsToHelmOpts {
// TODO(aanm) Do not mark the flags has deprecated for now.
Expand Down Expand Up @@ -111,6 +112,7 @@ func newCmdHubbleDisable() *cobra.Command {
},
}

cmd.Flags().StringVar(&params.HelmValuesSecretName, "helm-values-secret-name", defaults.HelmValuesSecretName, "Secret name to store the auto-generated helm values file. The namespace is the same as where Cilium will be installed")
cmd.Flags().StringVarP(&params.Namespace, "namespace", "n", "kube-system", "Namespace Cilium is running in")
cmd.Flags().StringVar(&contextName, "context", "", "Kubernetes configuration context")

Expand All @@ -135,6 +137,7 @@ func newCmdPortForwardCommand() *cobra.Command {
},
}

cmd.Flags().StringVar(&params.HelmValuesSecretName, "helm-values-secret-name", defaults.HelmValuesSecretName, "Secret name to store the auto-generated helm values file. The namespace is the same as where Cilium will be installed")
cmd.Flags().StringVarP(&params.Namespace, "namespace", "n", "kube-system", "Namespace Cilium is running in")
cmd.Flags().StringVar(&params.Context, "context", "", "Kubernetes configuration context")
cmd.Flags().IntVar(&params.PortForward, "port-forward", 4245, "Local port to forward to")
Expand Down
2 changes: 2 additions & 0 deletions internal/cli/cmd/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@ cilium install --context kind-cluster1 --cluster-id 1 --cluster-name cluster1
cmd.Flags().StringArrayVar(&params.HelmOpts.StringValues, "helm-set-string", []string{}, "Set helm STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
cmd.Flags().StringArrayVar(&params.HelmOpts.FileValues, "helm-set-file", []string{}, "Set helm values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)")
cmd.Flags().StringVar(&params.HelmGenValuesFile, "helm-auto-gen-values", "", "Write an auto-generated helm values into this file")
cmd.Flags().StringVar(&params.HelmValuesSecretName, "helm-values-secret-name", defaults.HelmValuesSecretName, "Secret name to store the auto-generated helm values file. The namespace is the same as where Cilium will be installed")
cmd.Flags().StringVar(&params.ImageSuffix, "image-suffix", "", "Set all generated images with this suffix")
cmd.Flags().StringVar(&params.ImageTag, "image-tag", "", "Set all images with this tag")

Expand Down Expand Up @@ -143,6 +144,7 @@ func newCmdUninstall() *cobra.Command {
}

cmd.Flags().StringVarP(&params.Namespace, "namespace", "n", "kube-system", "Namespace to uninstall Cilium from")
cmd.Flags().StringVar(&params.HelmValuesSecretName, "helm-values-secret-name", defaults.HelmValuesSecretName, "Secret name to store the auto-generated helm values file. The namespace is the same as where Cilium will be installed")
cmd.Flags().StringVar(&params.TestNamespace, "test-namespace", defaults.ConnectivityCheckNamespace, "Namespace to uninstall Cilium tests from")
cmd.Flags().StringVar(&contextName, "context", "", "Kubernetes configuration context")
cmd.Flags().BoolVar(&params.Wait, "wait", false, "Wait for uninstallation to have completed")
Expand Down
Loading

0 comments on commit 93c64e5

Please sign in to comment.