From b1c6659f0637da3ed04dff4ee64d3c57f77d685d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Martins?= Date: Sat, 2 Apr 2022 01:03:30 +0200 Subject: [PATCH 1/3] refactor helm code into dedicated package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These changes are needed for the upcoming changes that adds helm support to the remaining CLI options. Signed-off-by: André Martins --- install/helm.go | 191 +++----------------------------- install/install.go | 31 ------ internal/helm/helm.go | 250 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 267 insertions(+), 205 deletions(-) create mode 100644 internal/helm/helm.go diff --git a/install/helm.go b/install/helm.go index a68bf041ba..41e92049d0 100644 --- a/install/helm.go +++ b/install/helm.go @@ -5,100 +5,24 @@ package install import ( - "bytes" "context" "fmt" "os" - "regexp" - "sort" "strconv" "strings" + "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/helm" + "github.com/cilium/cilium-cli/k8s" + "github.com/cilium/cilium/pkg/versioncheck" "github.com/spf13/pflag" - "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chartutil" - "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/getter" - "helm.sh/helm/v3/pkg/releaseutil" - "helm.sh/helm/v3/pkg/strvals" - - "github.com/cilium/cilium-cli/defaults" - "github.com/cilium/cilium-cli/k8s" ) -var settings = cli.New() - -// FilterManifests a map of generated manifests. The Key is the filename and the -// Value is its manifest. -func FilterManifests(manifest string) map[string]string { - // This is necessary to ensure consistent manifest ordering when using --show-only - // with globs or directory names. - var manifests bytes.Buffer - fmt.Fprintln(&manifests, strings.TrimSpace(manifest)) - - splitManifests := releaseutil.SplitManifests(manifests.String()) - manifestsKeys := make([]string, 0, len(splitManifests)) - for k := range splitManifests { - manifestsKeys = append(manifestsKeys, k) - } - sort.Sort(releaseutil.BySplitManifestsOrder(manifestsKeys)) - - manifestNameRegex := regexp.MustCompile("# Source: [^/]+/(.+)") - - var ( - manifestsToRender = map[string]string{} - ) - - for _, manifestKey := range manifestsKeys { - manifest := splitManifests[manifestKey] - submatch := manifestNameRegex.FindStringSubmatch(manifest) - if len(submatch) == 0 { - continue - } - manifestName := submatch[1] - // manifest.Name is rendered using linux-style filepath separators on Windows as - // well as macOS/linux. - manifestPathSplit := strings.Split(manifestName, "/") - // manifest.Path is connected using linux-style filepath separators on Windows as - // well as macOS/linux - manifestPath := strings.Join(manifestPathSplit, "/") - - manifestsToRender[manifestPath] = manifest - } - return manifestsToRender -} - func (k *K8sInstaller) generateManifests(ctx context.Context) error { - k8sVersionStr := k.params.K8sVersion - if k8sVersionStr == "" { - k8sVersion, err := k.client.GetServerVersion() - if err != nil { - return fmt.Errorf("error getting Kubernetes version, try --k8s-version: %s", err) - } - k8sVersionStr = k8sVersion.String() - } - - helmClient, err := newHelmClient(k.params.Namespace, k8sVersionStr) - if err != nil { - return err - } - ciliumVer := k.getCiliumVersion() - var helmChart *chart.Chart - if helmDir := k.params.HelmChartDirectory; helmDir != "" { - helmChart, err = newHelmChartFromDirectory(helmDir) - if err != nil { - return err - } - } else { - helmChart, err = newHelmChartFromCiliumVersion(ciliumVer.String()) - if err != nil { - return err - } - } - helmMapOpts := map[string]string{} deprecatedCfgOpts := map[string]string{} @@ -301,68 +225,13 @@ func (k *K8sInstaller) generateManifests(ctx context.Context) error { return fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } - // Create helm values from helmMapOpts - var helmOpts []string - for k, v := range helmMapOpts { - if v == "" { - panic(fmt.Sprintf("empty value form helm option %q", k)) - } - helmOpts = append(helmOpts, fmt.Sprintf("%s=%s", k, v)) - } - - helmOptsStr := strings.Join(helmOpts, ",") - - ciliumCliHelmValues := map[string]interface{}{} - err = strvals.ParseInto(helmOptsStr, ciliumCliHelmValues) - if err != nil { - return fmt.Errorf("error parsing helm options %q: %w", helmOptsStr, err) - } - - // Get the user-defined helm options passed by flag - p := getter.All(settings) - userVals, err := k.params.HelmOpts.MergeValues(p) - if err != nil { - return err - } - - // User-defined helm options will overwrite the default cilium-cli helm options - userVals = mergeMaps(ciliumCliHelmValues, userVals) - // Store all the options passed by --config into helm extraConfig extraConfigMap := map[string]interface{}{} for k, v := range deprecatedCfgOpts { extraConfigMap[k] = v } - extraConfig := map[string]interface{}{} - if len(extraConfigMap) != 0 { - extraConfig["extraConfig"] = extraConfigMap - } - - // Merge the user-defined helm options into the `--config` map. This - // effectively means that any --helm-set=extraConfig. will overwrite - // the values of --config - vals := mergeMaps(extraConfig, userVals) - - valsStr := valuesToString("", vals) - - if helmChartDir := k.params.HelmChartDirectory; helmChartDir != "" { - k.Log("ℹ️ helm template --namespace %s cilium %q --version %s --set %s", k.params.Namespace, helmChartDir, ciliumVer, valsStr) - } else { - k.Log("ℹ️ helm template --namespace %s cilium cilium/cilium --version %s --set %s", k.params.Namespace, ciliumVer, valsStr) - } - - // Store the current helm-opts used in this installation in Cilium's - // ConfigMap - extraConfigMap[defaults.ExtraConfigMapUserOptsKey] = valsStr - extraConfig = map[string]interface{}{ - "extraConfig": extraConfigMap, - } - - // User-defined helm options will overwrite the default cilium-cli helm options - vals = mergeMaps(extraConfig, vals) - - rel, err := helmClient.RunWithContext(ctx, helmChart, vals) + vals, err := helm.MergeVals(k, k.params.HelmOpts, helmMapOpts, nil, extraConfigMap, k.params.HelmChartDirectory, ciliumVer.String(), k.params.Namespace) if err != nil { return err } @@ -375,46 +244,20 @@ func (k *K8sInstaller) generateManifests(ctx context.Context) error { return os.WriteFile(k.params.HelmGenValuesFile, []byte(yamlValue), 0o600) } - k.manifests = FilterManifests(rel.Manifest) - return nil -} - -func mergeMaps(a, b map[string]interface{}) map[string]interface{} { - out := make(map[string]interface{}, len(a)) - for k, v := range a { - out[k] = v - } - for k, v := range b { - if v, ok := v.(map[string]interface{}); ok { - if bv, ok := out[k]; ok { - if bv, ok := bv.(map[string]interface{}); ok { - out[k] = mergeMaps(bv, v) - continue - } - } + k8sVersionStr := k.params.K8sVersion + if k8sVersionStr == "" { + k8sVersion, err := k.client.GetServerVersion() + if err != nil { + return fmt.Errorf("error getting Kubernetes version, try --k8s-version: %s", err) } - out[k] = v + k8sVersionStr = k8sVersion.String() } - return out -} -func valuesToString(prevKey string, b map[string]interface{}) string { - var out []string - for k, v := range b { - if v, ok := v.(map[string]interface{}); ok { - if prevKey != "" { - out = append(out, valuesToString(fmt.Sprintf("%s.%s", prevKey, k), v)) - } else { - out = append(out, valuesToString(k, v)) - } - continue - } - if prevKey != "" { - out = append(out, fmt.Sprintf("%s.%s=%v", prevKey, k, v)) - } else { - out = append(out, fmt.Sprintf("%s=%v", k, v)) - } + manifests, err := helm.GenManifests(ctx, k.params.HelmChartDirectory, k8sVersionStr, ciliumVer.String(), k.params.Namespace, vals) + if err != nil { + return err } - sort.Strings(out) - return strings.Join(out, ",") + + k.manifests = manifests + return nil } diff --git a/install/install.go b/install/install.go index 556edbf0a3..cd2431063d 100644 --- a/install/install.go +++ b/install/install.go @@ -12,14 +12,10 @@ import ( "time" "github.com/blang/semver/v4" - ciliumhelm "github.com/cilium/charts" "github.com/cilium/cilium/api/v1/models" ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" "github.com/cilium/cilium/pkg/versioncheck" "github.com/spf13/pflag" - "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/chart" - "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli/values" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -286,33 +282,6 @@ func (k *K8sInstaller) fqOperatorImage(imagePathMode utils.ImagePathMode) string return utils.BuildImagePath(k.params.OperatorImage, k.params.Version, defaultImage, defaults.Version, imagePathMode) } -func newHelmClient(namespace, k8sVersion string) (*action.Install, error) { - actionConfig := new(action.Configuration) - helmClient := action.NewInstall(actionConfig) - helmClient.DryRun = true - helmClient.ReleaseName = "release-name" - helmClient.Replace = true // Skip the name check - helmClient.ClientOnly = true - helmClient.APIVersions = []string{k8sVersion} - helmClient.Namespace = namespace - - return helmClient, nil -} - -func newHelmChartFromCiliumVersion(ciliumVersion string) (*chart.Chart, error) { - helmTgz, err := ciliumhelm.HelmFS.ReadFile(fmt.Sprintf("cilium-%s.tgz", ciliumVersion)) - if err != nil { - return nil, fmt.Errorf("cilium version not found: %s", err) - } - - // Check chart dependencies to make sure all are present in /charts - return loader.LoadArchive(bytes.NewReader(helmTgz)) -} - -func newHelmChartFromDirectory(directory string) (*chart.Chart, error) { - return loader.LoadDir(directory) -} - func NewK8sInstaller(client k8sInstallerImplementation, p Parameters) (*K8sInstaller, error) { if err := (&p).validate(); err != nil { return nil, fmt.Errorf("invalid parameters: %w", err) diff --git a/internal/helm/helm.go b/internal/helm/helm.go new file mode 100644 index 0000000000..00e80fa26a --- /dev/null +++ b/internal/helm/helm.go @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2020 Authors of Cilium +// Copyright The Helm Authors. + +package helm + +import ( + "bytes" + "context" + "fmt" + "regexp" + "sort" + "strings" + + "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/utils" + + helm "github.com/cilium/charts" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "helm.sh/helm/v3/pkg/cli" + "helm.sh/helm/v3/pkg/cli/values" + "helm.sh/helm/v3/pkg/getter" + "helm.sh/helm/v3/pkg/releaseutil" + "helm.sh/helm/v3/pkg/strvals" +) + +var settings = cli.New() + +// filterManifests a map of generated manifests. The Key is the filename and the +// Value is its manifest. +func filterManifests(manifest string) map[string]string { + // This is necessary to ensure consistent manifest ordering when using --show-only + // with globs or directory names. + var manifests bytes.Buffer + fmt.Fprintln(&manifests, strings.TrimSpace(manifest)) + + splitManifests := releaseutil.SplitManifests(manifests.String()) + manifestsKeys := make([]string, 0, len(splitManifests)) + for k := range splitManifests { + manifestsKeys = append(manifestsKeys, k) + } + sort.Sort(releaseutil.BySplitManifestsOrder(manifestsKeys)) + + manifestNameRegex := regexp.MustCompile("# Source: [^/]+/(.+)") + + var ( + manifestsToRender = map[string]string{} + ) + + for _, manifestKey := range manifestsKeys { + manifest := splitManifests[manifestKey] + submatch := manifestNameRegex.FindStringSubmatch(manifest) + if len(submatch) == 0 { + continue + } + manifestName := submatch[1] + // manifest.Name is rendered using linux-style filepath separators on Windows as + // well as macOS/linux. + manifestPathSplit := strings.Split(manifestName, "/") + // manifest.Path is connected using linux-style filepath separators on Windows as + // well as macOS/linux + manifestPath := strings.Join(manifestPathSplit, "/") + + manifestsToRender[manifestPath] = manifest + } + return manifestsToRender +} + +func mergeMaps(a, b map[string]interface{}) map[string]interface{} { + out := make(map[string]interface{}, len(a)) + for k, v := range a { + out[k] = v + } + for k, v := range b { + if v, ok := v.(map[string]interface{}); ok { + if bv, ok := out[k]; ok { + if bv, ok := bv.(map[string]interface{}); ok { + out[k] = mergeMaps(bv, v) + continue + } + } + } + out[k] = v + } + return out +} + +func valuesToString(prevKey string, b map[string]interface{}) string { + var out []string + for k, v := range b { + if v, ok := v.(map[string]interface{}); ok { + if prevKey != "" { + out = append(out, valuesToString(fmt.Sprintf("%s.%s", prevKey, k), v)) + } else { + out = append(out, valuesToString(k, v)) + } + continue + } + if prevKey != "" { + out = append(out, fmt.Sprintf("%s.%s=%v", prevKey, k, v)) + } else { + out = append(out, fmt.Sprintf("%s=%v", k, v)) + } + } + sort.Strings(out) + return strings.Join(out, ",") +} + +func newClient(namespace, k8sVersion string) (*action.Install, error) { + actionConfig := new(action.Configuration) + helmClient := action.NewInstall(actionConfig) + helmClient.DryRun = true + helmClient.ReleaseName = "release-name" + helmClient.Replace = true // Skip the name check + helmClient.ClientOnly = true + helmClient.APIVersions = []string{k8sVersion} + helmClient.Namespace = namespace + + return helmClient, nil +} + +func newChartFromCiliumVersion(ciliumVersion string) (*chart.Chart, error) { + helmTgz, err := helm.HelmFS.ReadFile(fmt.Sprintf("cilium-%s.tgz", ciliumVersion)) + if err != nil { + return nil, fmt.Errorf("cilium version not found: %s", err) + } + + // Check chart dependencies to make sure all are present in /charts + return loader.LoadArchive(bytes.NewReader(helmTgz)) +} + +func newChartFromDirectory(directory string) (*chart.Chart, error) { + return loader.LoadDir(directory) +} + +// GenManifests returns the generated manifests in a map that maps the manifest +// name to its contents. +func GenManifests(ctx context.Context, helmChartDirectory, k8sVersion, ciliumVer, namespace string, helmValues map[string]interface{}) (map[string]string, error) { + var ( + helmChart *chart.Chart + err error + ) + if helmDir := helmChartDirectory; helmDir != "" { + helmChart, err = newChartFromDirectory(helmDir) + if err != nil { + return nil, err + } + } else { + helmChart, err = newChartFromCiliumVersion(ciliumVer) + if err != nil { + return nil, err + } + } + + helmClient, err := newClient(namespace, k8sVersion) + if err != nil { + return nil, err + } + + rel, err := helmClient.RunWithContext(ctx, helmChart, helmValues) + if err != nil { + return nil, err + } + + return filterManifests(rel.Manifest), nil +} + +// MergeVals merges all values from flag options ('helmFlagOpts'), +// auto-generated helm options based on environment ('helmMapOpts'), +// helm values from a previous installation ('helmValues'), +// extra options that are not defined as helm flags ('extraConfigMapOpts') +// and returns a single map with all of these options merged. +// It will log a message so that users can replicate the same behavior as the +// CLI. The log message will be slightly different depending on if +// 'helmChartDirectory' is set or not. +// Both 'helmMapOpts', 'helmValues', 'extraConfigMapOpts' can be nil. +func MergeVals( + logger utils.Logger, + helmFlagOpts values.Options, + helmMapOpts map[string]string, + helmValues map[string]interface{}, + extraConfigMapOpts map[string]interface{}, + helmChartDirectory, ciliumVer, namespace string, +) (map[string]interface{}, error) { + + // Create helm values from helmMapOpts + var helmOpts []string + for k, v := range helmMapOpts { + if v == "" { + panic(fmt.Sprintf("empty value form helm option %q", k)) + } + helmOpts = append(helmOpts, fmt.Sprintf("%s=%s", k, v)) + } + + helmOptsStr := strings.Join(helmOpts, ",") + + if helmValues == nil { + helmValues = map[string]interface{}{} + } + err := strvals.ParseInto(helmOptsStr, helmValues) + if err != nil { + return nil, fmt.Errorf("error parsing helm options %q: %w", helmOptsStr, err) + } + + // Get the user-defined helm options passed by flag + p := getter.All(settings) + userVals, err := helmFlagOpts.MergeValues(p) + if err != nil { + return nil, err + } + + // User-defined helm options will overwrite the default cilium-cli helm options + userVals = mergeMaps(helmValues, userVals) + + // Merge the user-defined helm options into the `--config` map. This + // effectively means that any --helm-set=extraConfig. will overwrite + // the values of --config + extraConfig := map[string]interface{}{} + if len(extraConfigMapOpts) != 0 { + extraConfig["extraConfig"] = extraConfigMapOpts + } else if extraConfigMapOpts == nil { + extraConfigMapOpts = map[string]interface{}{} + } + + vals := mergeMaps(extraConfig, userVals) + + valsStr := valuesToString("", vals) + + if helmChartDirectory != "" { + logger.Log("ℹ️ helm template --namespace %s cilium %q --version %s --set %s", namespace, helmChartDirectory, ciliumVer, valsStr) + } else { + logger.Log("ℹ️ helm template --namespace %s cilium cilium/cilium --version %s --set %s", namespace, ciliumVer, valsStr) + } + + // Store the current helm-opts used in this installation in Cilium's + // ConfigMap + // To avoid verbosity in the terminal we don't print the `valsStr` in the + // log above + extraConfigMapOpts[defaults.ExtraConfigMapUserOptsKey] = valsStr + extraConfig = map[string]interface{}{ + "extraConfig": extraConfigMapOpts, + } + + // User-defined helm options will overwrite the default cilium-cli helm options + vals = mergeMaps(extraConfig, vals) + + return vals, nil +} From c2133c557149301420b5232d274df31d5843f618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Martins?= Date: Tue, 5 Apr 2022 16:07:00 +0200 Subject: [PATCH 2/3] internal/helm: fix bug in helm keys that had a . MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since in YAML dots are used to represent a path, when converting helm values into a string representation YAML keys that had dots were wrongly represented. This commit escapes the dots to correctly represent the helm keys. Signed-off-by: André Martins --- internal/helm/helm.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/helm/helm.go b/internal/helm/helm.go index 00e80fa26a..c919ebbdc9 100644 --- a/internal/helm/helm.go +++ b/internal/helm/helm.go @@ -99,6 +99,9 @@ func valuesToString(prevKey string, b map[string]interface{}) string { continue } if prevKey != "" { + if strings.Contains(k, ".") { + k = strings.ReplaceAll(k, ".", `\\.`) + } out = append(out, fmt.Sprintf("%s.%s=%v", prevKey, k, v)) } else { out = append(out, fmt.Sprintf("%s=%v", k, v)) From ebd12e37e080570dadae4e09176703b77e66a32c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Martins?= Date: Tue, 5 Apr 2022 00:41:45 +0200 Subject: [PATCH 3/3] hubble: add helm-based installation to hubble mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By storing helm-based config values into Cilium's ConfigMap we are able to generate hubble deployment files by reading those values. Once the manifests are generated, the new helm flags used to enable hubble and/or hubble-ui and/or hubble-relay will also be store in Cilium's ConfigMap. Similarly to 'hubble enable', 'hubble disable' command also stores the helm-based config values into Cilium's ConfigMap. Signed-off-by: André Martins --- defaults/defaults.go | 42 +--- hubble/hubble.go | 430 ++++++++++++++++++++++++++----- hubble/relay.go | 503 ++++++++++++++----------------------- hubble/relay_test.go | 5 + hubble/ui.go | 431 +++++++++++++------------------ install/helm.go | 2 +- internal/certs/certs.go | 10 + internal/cli/cmd/hubble.go | 35 ++- internal/helm/helm.go | 14 +- k8s/client.go | 4 + 10 files changed, 795 insertions(+), 681 deletions(-) diff --git a/defaults/defaults.go b/defaults/defaults.go index 8932063ee4..4aff84022c 100644 --- a/defaults/defaults.go +++ b/defaults/defaults.go @@ -30,27 +30,20 @@ const ( OperatorImageAWS = "quay.io/cilium/operator-aws" OperatorImageAzure = "quay.io/cilium/operator-azure" - HubbleSocketPath = "/var/run/cilium/hubble.sock" HubbleServerSecretName = "hubble-server-certs" - RelayContainerName = "hubble-relay" - RelayDeploymentName = "hubble-relay" - RelayClusterRoleName = "hubble-relay" - RelayServiceAccountName = "hubble-relay" - RelayServiceName = "hubble-relay" - RelayConfigMapName = "hubble-relay-config" - RelayImage = "quay.io/cilium/hubble-relay" - RelayListenHost = "" - RelayPort = 4245 - RelayServicePlaintextPort = 80 - RelayServiceTLSPort = 443 - RelayServerSecretName = "hubble-relay-server-certs" - RelayClientSecretName = "hubble-relay-client-certs" - - HubbleUIServiceName = "hubble-ui" + RelayContainerName = "hubble-relay" + RelayDeploymentName = "hubble-relay" + RelayClusterRoleName = "hubble-relay" + RelayServiceAccountName = "hubble-relay" + RelayConfigMapName = "hubble-relay-config" + RelayImage = "quay.io/cilium/hubble-relay" + RelayServerSecretName = "hubble-relay-server-certs" + RelayClientSecretName = "hubble-relay-client-certs" + HubbleUIClientSecretName = "hubble-ui-client-certs" + HubbleUIClusterRoleName = "hubble-ui" HubbleUIServiceAccountName = "hubble-ui" - HubbleUIConfigMapName = "hubble-ui-envoy" HubbleUIDeploymentName = "hubble-ui" HubbleUIImage = "quay.io/cilium/hubble-ui" HubbleUIBackendImage = "quay.io/cilium/hubble-ui-backend" @@ -72,9 +65,8 @@ const ( ConnectivityPerformanceImage = "quay.io/cilium/network-perf:bf58fb8bc57c4933dfa6e2a9581d3925c0a0571e@sha256:9bef508b2dcaeb3e288a496b8d3f065e8636a4937ba3aebcb1732afffaccea34" ConnectivityCheckJSONMockImage = "quay.io/cilium/json-mock:v1.3.0@sha256:2729064827fa9dbfface8d3df424feb6c792a0ba07117b844349635c93c06d2b" - ConfigMapName = "cilium-config" - Version = "v1.11.3" - HubbleUIVersion = "v0.8.5" + ConfigMapName = "cilium-config" + Version = "v1.11.3" TunnelType = "vxlan" @@ -98,16 +90,6 @@ const ( ) var ( - // RelayDeploymentLabels are the labels set on the Hubble Relay Deployment by default. - RelayDeploymentLabels = map[string]string{ - "k8s-app": "hubble-relay", - } - - // HubbleUIDeploymentLabels are the labels set on the Hubble UI Deployment by default. - HubbleUIDeploymentLabels = map[string]string{ - "k8s-app": "hubble-ui", - } - // ClusterMeshDeploymentLabels are the labels set on the clustermesh API server by default. ClusterMeshDeploymentLabels = map[string]string{ "k8s-app": "clustermesh-apiserver", diff --git a/hubble/hubble.go b/hubble/hubble.go index 20f958c533..fae7048f52 100644 --- a/hubble/hubble.go +++ b/hubble/hubble.go @@ -5,23 +5,31 @@ package hubble import ( "context" + "encoding/base64" "fmt" "io" + "os" "strings" "time" + "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/certs" + "github.com/cilium/cilium-cli/internal/helm" + "github.com/cilium/cilium-cli/internal/utils" + "github.com/cilium/cilium-cli/status" + + "github.com/blang/semver/v4" "github.com/cilium/cilium/api/v1/models" ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" + "github.com/cilium/cilium/pkg/versioncheck" + "github.com/spf13/pflag" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/cli/values" + "helm.sh/helm/v3/pkg/strvals" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - "github.com/cilium/cilium-cli/defaults" - "github.com/cilium/cilium-cli/internal/certs" - "github.com/cilium/cilium-cli/internal/utils" - "github.com/cilium/cilium-cli/status" ) const ( @@ -29,10 +37,6 @@ const ( configNameListenAddress = "hubble-listen-address" ) -var ( - hostPathDirectoryOrCreate = corev1.HostPathDirectoryOrCreate -) - type k8sHubbleImplementation interface { CreateSecret(ctx context.Context, namespace string, secret *corev1.Secret, opts metav1.CreateOptions) (*corev1.Secret, error) DeleteSecret(ctx context.Context, namespace, name string, opts metav1.DeleteOptions) error @@ -46,7 +50,7 @@ type k8sHubbleImplementation interface { CreateConfigMap(ctx context.Context, namespace string, config *corev1.ConfigMap, opts metav1.CreateOptions) (*corev1.ConfigMap, error) DeleteConfigMap(ctx context.Context, namespace, name string, opts metav1.DeleteOptions) error GetConfigMap(ctx context.Context, namespace, name string, opts metav1.GetOptions) (*corev1.ConfigMap, error) - PatchConfigMap(ctx context.Context, namespace, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*corev1.ConfigMap, error) + UpdateConfigMap(ctx context.Context, configMap *corev1.ConfigMap, opts metav1.UpdateOptions) (*corev1.ConfigMap, error) CreateDeployment(ctx context.Context, namespace string, deployment *appsv1.Deployment, opts metav1.CreateOptions) (*appsv1.Deployment, error) GetDeployment(ctx context.Context, namespace, name string, opts metav1.GetOptions) (*appsv1.Deployment, error) DeleteDeployment(ctx context.Context, namespace, name string, opts metav1.DeleteOptions) error @@ -58,15 +62,32 @@ type k8sHubbleImplementation interface { CiliumStatus(ctx context.Context, namespace, pod string) (*models.StatusResponse, error) ListCiliumEndpoints(ctx context.Context, namespace string, opts metav1.ListOptions) (*ciliumv2.CiliumEndpointList, error) GetRunningCiliumVersion(ctx context.Context, namespace string) (string, error) + GetServerVersion() (*semver.Version, error) } type K8sHubble struct { - client k8sHubbleImplementation - params Parameters - certManager *certs.CertManager - ciliumVersion string + client k8sHubbleImplementation + params Parameters + certManager *certs.CertManager + ciliumVersion string + manifests map[string]string + semVerCiliumVersion semver.Version } +var ( + // FlagsToHelmOpts maps the deprecated install flags to the helm + // options + FlagsToHelmOpts = map[string][]string{ + "relay-image": {"hubble.relay.image.override"}, + "relay-version": {"hubble.relay.image.tag"}, + "ui-image": {"hubble.ui.frontend.image.override"}, + "ui-backend-image": {"hubble.ui.backend.image.override"}, + "ui-version": {"hubble.ui.frontend.image.tag", "hubble.ui.backend.image.tag"}, + } + // FlagValues maps all FlagsToHelmOpts keys to their values + FlagValues = map[string]pflag.Value{} +) + type Parameters struct { Namespace string Relay bool @@ -84,6 +105,26 @@ type Parameters struct { Context string // Only for 'kubectl' pass-through commands Wait bool WaitDuration time.Duration + + // BaseVersion is used to explicitly specify Cilium version for generating the config map + // in case it cannot be inferred from the Version field (e.g. commit SHA tags for CI images). + BaseVersion string + + // K8sVersion is the Kubernetes version that will be used to generate the + // kubernetes manifests. If the auto-detection fails, this flag can be used + // as a workaround. + K8sVersion string + // HelmChartDirectory points to the location of a helm chart directory. + // Useful to test from upstream where a helm release is not available yet. + HelmChartDirectory string + + // HelmOpts are all the options the user used to pass into the Cilium cli + // template. + HelmOpts values.Options + + // HelmGenValuesFile points to the file that will store the generated helm + // options. + HelmGenValuesFile string } func (p *Parameters) Log(format string, a ...interface{}) { @@ -158,51 +199,40 @@ func (k *K8sHubble) Validate(ctx context.Context) error { } -var hubbleCfg = map[string]string{ - // Enable Hubble gRPC service. - "enable-hubble": "true", - // UNIX domain socket for Hubble server to listen to. - "hubble-socket-path": defaults.HubbleSocketPath, - // An additional address for Hubble server to listen to (e.g. ":4244"). - "hubble-listen-address": ":4244", - "hubble-disable-tls": "false", - "hubble-tls-cert-file": "/var/lib/cilium/tls/hubble/server.crt", - "hubble-tls-key-file": "/var/lib/cilium/tls/hubble/server.key", - "hubble-tls-client-ca-files": "/var/lib/cilium/tls/hubble/client-ca.crt", +func (k *K8sHubble) disableHubble(ctx context.Context) error { + k.Log("✨ Patching ConfigMap %s to disable Hubble...", defaults.ConfigMapName) + + return k.updateConfigMap(ctx) } -func (k *K8sHubble) disableHubble(ctx context.Context) error { - cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{}) +func (k *K8sHubble) Disable(ctx context.Context) error { + var err error + k.ciliumVersion, err = k.client.GetRunningCiliumVersion(ctx, k.params.Namespace) if err != nil { - return fmt.Errorf("unable to get ConfigMap %s: %w", defaults.ConfigMapName, err) + return err } - var changes []string - for k := range hubbleCfg { - if _, ok := cm.Data[k]; ok { - changes = append(changes, `{"op": "remove", "path": "/data/`+k+`"}`) - } - } - if len(changes) > 0 { - patch := []byte(`[` + strings.Join(changes, ",") + `]`) + k.semVerCiliumVersion = k.getCiliumVersion() - k.Log("✨ Patching ConfigMap %s to disable Hubble...", defaults.ConfigMapName) - _, err := k.client.PatchConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, types.JSONPatchType, patch, metav1.PatchOptions{}) - if err != nil { - return fmt.Errorf("unable to patch ConfigMap %s with patch %q: %w", defaults.ConfigMapName, patch, err) - } + cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("unable to retrieve ConfigMap %q: %w", defaults.ConfigMapName, err) } - if err := k.client.DeletePodCollection(ctx, k.params.Namespace, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: defaults.CiliumPodSelector}); err != nil { - k.Log("⚠️ Unable to restart Clium pods: %s", err) - } else { - k.Log("♻️ Restarted Cilium pods") + value, ok := cm.Data[defaults.ExtraConfigMapUserOptsKey] + if !ok { + return fmt.Errorf("configmap option not found") } - return nil -} + // Generate the manifests has if hubble was being enabled so that we can + // retrieve all UI and Relay's resource names. + k.params.UI = true + k.params.Relay = true + err = k.generateManifestsEnable(ctx, false, value) + if err != nil { + return err + } -func (k *K8sHubble) Disable(ctx context.Context) error { if err := k.disableUI(ctx); err != nil { return err } @@ -211,6 +241,13 @@ func (k *K8sHubble) Disable(ctx context.Context) error { return err } + // Now that we have delete all UI and Relay's resource names then we can + // generate the manifests with UI and Relay disabled. + err = k.generateManifestsDisable(ctx, value) + if err != nil { + return err + } + if err := k.disableHubble(ctx); err != nil { return err } @@ -220,26 +257,176 @@ func (k *K8sHubble) Disable(ctx context.Context) error { return nil } -func (k *K8sHubble) enableHubble(ctx context.Context) error { - var changes []string - for k, v := range hubbleCfg { - changes = append(changes, `"`+k+`":"`+v+`"`) +func (k *K8sHubble) generateConfigMap() (*corev1.ConfigMap, error) { + var ( + cmFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + cmFilename = "templates/cilium-configmap.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } - patch := []byte(`{"data":{` + strings.Join(changes, ",") + `}}`) + cmFile := k.manifests[cmFilename] + + var cm corev1.ConfigMap + utils.MustUnmarshalYAML([]byte(cmFile), &cm) + k.Log("🚀 Creating ConfigMap for Cilium version %s...", ciliumVer.String()) + return &cm, nil +} + +func (k *K8sHubble) enableHubble(ctx context.Context) error { k.Log("✨ Patching ConfigMap %s to enable Hubble...", defaults.ConfigMapName) - _, err := k.client.PatchConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + + return k.updateConfigMap(ctx) +} + +func (k *K8sHubble) updateConfigMap(ctx context.Context) error { + cm, err := k.generateConfigMap() if err != nil { - return fmt.Errorf("unable to patch ConfigMap %s with patch %q: %w", defaults.ConfigMapName, patch, err) + return err + } + + _, err = k.client.UpdateConfigMap(ctx, cm, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("unable to patch ConfigMap %s with %s: %w", defaults.ConfigMapName, cm, err) } if err := k.client.DeletePodCollection(ctx, k.params.Namespace, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: defaults.CiliumPodSelector}); err != nil { - k.Log("⚠️ Unable to restart Clium pods: %s", err) + k.Log("⚠️ Unable to restart Cilium pods: %s", err) } else { k.Log("♻️ Restarted Cilium pods") } + return nil +} + +func (k *K8sHubble) getCiliumVersion() semver.Version { + v, err := utils.ParseCiliumVersion(k.ciliumVersion, k.params.BaseVersion) + if err != nil { + v = versioncheck.MustVersion(defaults.Version) + k.Log("Unable to parse the provided version %q, assuming %v for ConfigMap compatibility", k.ciliumVersion, defaults.Version) + } + return v +} + +func (k *K8sHubble) generateManifestsEnable(ctx context.Context, printHelmTemplate bool, helmValues string) error { + ciliumVer := k.semVerCiliumVersion + + helmMapOpts := map[string]string{} + + switch { + // It's likely that certain helm options have changed since 1.9.0 + // These were tested for the >=1.11.0. In case something breaks for versions + // older than 1.11.0 we will fix it afterwards. + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + // case versioncheck.MustCompile(">=1.11.0")(ciliumVer): + + // Pre-define all deprecated flags as helm options + for flagName, helmOpts := range FlagsToHelmOpts { + if v, ok := FlagValues[flagName]; ok { + if val := v.String(); val != "" { + for _, helmOpt := range helmOpts { + helmMapOpts[helmOpt] = val + switch helmOpt { + // If the images or tag are overwritten then we need + // to disable 'useDigest' + case "hubble.relay.image.override", "hubble.relay.image.tag": + helmMapOpts["hubble.relay.image.useDigest"] = "false" + } + } + } + } + } + + helmMapOpts["hubble.enabled"] = "true" + helmMapOpts["hubble.tls.ca.cert"] = base64.StdEncoding.EncodeToString(k.certManager.CACertBytes()) + helmMapOpts["hubble.tls.ca.key"] = base64.StdEncoding.EncodeToString(k.certManager.CAKeyBytes()) + + if k.params.UI { + helmMapOpts["hubble.ui.enabled"] = "true" + // See for https://github.com/cilium/cilium/pull/19338 more details + switch { + case versioncheck.MustCompile(">=1.11.4")(ciliumVer): + default: + helmMapOpts["hubble.ui.securityContext.enabled"] = "false" + } + } + if k.params.Relay { + helmMapOpts["hubble.relay.enabled"] = "true" + // TODO we won't generate hubble-ui certificates because we don't want + // to give a bad UX for hubble-cli (which connects to hubble-relay) + // helmMapOpts["hubble.relay.tls.server.enabled"] = "true" + } + + default: + return fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) + } + + return k.genManifests(ctx, printHelmTemplate, helmValues, helmMapOpts, ciliumVer) +} + +func (k *K8sHubble) generateManifestsDisable(ctx context.Context, helmValues string) error { + ciliumVer := k.semVerCiliumVersion + + helmMapOpts := map[string]string{} + + switch { + // It's likely that certain helm options have changed since 1.9.0 + // These were tested for the >=1.11.0. In case something breaks for versions + // older than 1.11.0 we will fix it afterwards. + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + // case versioncheck.MustCompile(">=1.11.0")(ciliumVer): + helmMapOpts["hubble.enabled"] = "false" + helmMapOpts["hubble.ui.enabled"] = "false" + helmMapOpts["hubble.relay.enabled"] = "false" + + default: + return fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) + } + + return k.genManifests(ctx, false, helmValues, helmMapOpts, ciliumVer) +} +func (k *K8sHubble) genManifests(ctx context.Context, printHelmTemplate bool, helmValues string, helmMapOpts map[string]string, ciliumVer semver.Version) error { + // Store all the options passed by --config into helm extraConfig + prevHelmValues := map[string]interface{}{} + err := strvals.ParseInto(helmValues, prevHelmValues) + if err != nil { + return fmt.Errorf("error parsing helm options %q: %w", helmValues, err) + } + + vals, err := helm.MergeVals(k, printHelmTemplate, k.params.HelmOpts, helmMapOpts, prevHelmValues, nil, k.params.HelmChartDirectory, ciliumVer.String(), k.params.Namespace) + if err != nil { + return err + } + + if k.params.HelmGenValuesFile != "" { + yamlValue, err := chartutil.Values(vals).YAML() + if err != nil { + return err + } + return os.WriteFile(k.params.HelmGenValuesFile, []byte(yamlValue), 0o600) + } + + k8sVersionStr := k.params.K8sVersion + if k8sVersionStr == "" { + k8sVersion, err := k.client.GetServerVersion() + if err != nil { + return fmt.Errorf("error getting Kubernetes version, try --k8s-version: %s", err) + } + k8sVersionStr = k8sVersion.String() + } + + manifests, err := helm.GenManifests(ctx, k.params.HelmChartDirectory, k8sVersionStr, ciliumVer.String(), k.params.Namespace, vals) + if err != nil { + return err + } + + k.manifests = manifests return nil } @@ -254,6 +441,8 @@ func (k *K8sHubble) Enable(ctx context.Context) error { return err } + k.semVerCiliumVersion = k.getCiliumVersion() + caSecret, created, err := k.certManager.GetOrCreateCASecret(ctx, defaults.CASecretName, k.params.CreateCA) if err != nil { k.Log("❌ Unable to get or create the Cilium CA Secret: %s", err) @@ -273,6 +462,21 @@ func (k *K8sHubble) Enable(ctx context.Context) error { } } + cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("unable to retrieve ConfigMap %q: %w", defaults.ConfigMapName, err) + } + + value, ok := cm.Data[defaults.ExtraConfigMapUserOptsKey] + if !ok { + return fmt.Errorf("configmap option not found") + } + + err = k.generateManifestsEnable(ctx, true, value) + if err != nil { + return err + } + if err := k.enableHubble(ctx); err != nil { return err } @@ -299,34 +503,32 @@ func (k *K8sHubble) Enable(ctx context.Context) error { } } + var warnFreePods []string if k.params.Relay { - if err := k.enableRelay(ctx); err != nil { + podsName, err := k.enableRelay(ctx) + if err != nil { return err } + + warnFreePods = append(warnFreePods, podsName) } if k.params.UI { - if err := k.enableUI(ctx); err != nil { + podsName, err := k.enableUI(ctx) + if err != nil { return err } + + warnFreePods = append(warnFreePods, podsName) } if k.params.Wait { - var pods []string - - if k.params.Relay { - pods = append(pods, defaults.RelayDeploymentName) - } - if k.params.UI { - pods = append(pods, defaults.HubbleUIDeploymentName) - } - k.Log("⌛ Waiting for Hubble to be installed...") collector, err := status.NewK8sStatusCollector(k.client, status.K8sStatusParameters{ Namespace: k.params.Namespace, Wait: true, WaitDuration: k.params.WaitDuration - dur, - WarningFreePods: pods, + WarningFreePods: warnFreePods, }) if err != nil { return err @@ -343,3 +545,93 @@ func (k *K8sHubble) Enable(ctx context.Context) error { return nil } + +func (k *K8sHubble) NewServiceAccount(name string) *corev1.ServiceAccount { + var ( + saFileName string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">1.10.99")(ciliumVer): + switch name { + case defaults.RelayServiceAccountName: + saFileName = "templates/hubble-relay/serviceaccount.yaml" + case defaults.HubbleUIServiceAccountName: + saFileName = "templates/hubble-ui/serviceaccount.yaml" + } + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + switch name { + case defaults.RelayServiceAccountName: + saFileName = "templates/hubble-relay-serviceaccount.yaml" + case defaults.HubbleUIServiceAccountName: + saFileName = "templates/hubble-ui-serviceaccount.yaml" + } + } + + saFile := k.manifests[saFileName] + + var sa corev1.ServiceAccount + utils.MustUnmarshalYAML([]byte(saFile), &sa) + return &sa +} + +func (k *K8sHubble) NewClusterRole(name string) *rbacv1.ClusterRole { + var ( + crFileName string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">1.10.99")(ciliumVer): + switch name { + case defaults.RelayClusterRoleName: + crFileName = "templates/hubble-relay/clusterrole.yaml" + case defaults.HubbleUIClusterRoleName: + crFileName = "templates/hubble-ui/clusterrole.yaml" + } + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + switch name { + case defaults.RelayClusterRoleName: + crFileName = "templates/hubble-relay-clusterrole.yaml" + case defaults.HubbleUIClusterRoleName: + crFileName = "templates/hubble-ui-clusterrole.yaml" + } + } + + crFile := k.manifests[crFileName] + + var cr rbacv1.ClusterRole + utils.MustUnmarshalYAML([]byte(crFile), &cr) + return &cr +} + +func (k *K8sHubble) NewClusterRoleBinding(crbName string) *rbacv1.ClusterRoleBinding { + var ( + crbFileName string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">1.10.99")(ciliumVer): + switch crbName { + case defaults.RelayClusterRoleName: + crbFileName = "templates/hubble-relay/clusterrolebinding.yaml" + case defaults.HubbleUIClusterRoleName: + crbFileName = "templates/hubble-ui/clusterrolebinding.yaml" + } + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + switch crbName { + case defaults.RelayClusterRoleName: + crbFileName = "templates/hubble-relay-clusterrolebinding.yaml" + case defaults.HubbleUIClusterRoleName: + crbFileName = "templates/hubble-ui-clusterrolebinding.yaml" + } + } + + crbFile := k.manifests[crbFileName] + + var crb rbacv1.ClusterRoleBinding + utils.MustUnmarshalYAML([]byte(crbFile), &crb) + return &crb +} diff --git a/hubble/relay.go b/hubble/relay.go index beb4a98534..f804fdc30b 100644 --- a/hubble/relay.go +++ b/hubble/relay.go @@ -6,399 +6,282 @@ package hubble import ( "context" "fmt" - "time" - "github.com/cloudflare/cfssl/config" - "github.com/cloudflare/cfssl/csr" + "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/utils" + + "github.com/cilium/cilium/pkg/versioncheck" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "github.com/cilium/cilium-cli/defaults" - "github.com/cilium/cilium-cli/internal/utils" - "github.com/cilium/cilium-cli/k8s" ) -const ( - relayPort = int32(defaults.RelayPort) - relayPortName = "grpc" -) +func (k *K8sHubble) generateRelayService() (*corev1.Service, error) { + var ( + svcFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + svcFilename = "templates/hubble-relay/service.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) + } -var ( - secretDefaultMode = int32(0400) - relayReplicas = int32(1) - relayPortIntstr = intstr.FromInt(defaults.RelayPort) - deploymentMaxSurge = intstr.FromInt(1) - deploymentMaxUnavailable = intstr.FromInt(1) -) + svcFile := k.manifests[svcFilename] -var relayClusterRole = &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.RelayClusterRoleName, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"componentstatuses", "endpoints", "namespaces", "nodes", "pods", "services"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, + var svc corev1.Service + utils.MustUnmarshalYAML([]byte(svcFile), &svc) + return &svc, nil } -func (k *K8sHubble) generateRelayService() *corev1.Service { - // NOTE: assuming "disable-server-tls: true", see generateRelayConfigMap(). - s := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.RelayServiceName, - Labels: defaults.RelayDeploymentLabels, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceType(k.params.RelayServiceType), - Ports: []corev1.ServicePort{ - { - Port: int32(defaults.RelayServicePlaintextPort), - TargetPort: relayPortIntstr, - }, - }, - Selector: defaults.RelayDeploymentLabels, - }, +func (k *K8sHubble) generateRelayDeployment() (*appsv1.Deployment, error) { + var ( + deployFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + deployFilename = "templates/hubble-relay/deployment.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } - return s -} -func (k *K8sHubble) generateRelayDeployment() *appsv1.Deployment { - d := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.RelayDeploymentName, - Labels: defaults.RelayDeploymentLabels, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &relayReplicas, - Selector: &metav1.LabelSelector{ - MatchLabels: defaults.RelayDeploymentLabels, - }, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.RollingUpdateDeploymentStrategyType, - RollingUpdate: &appsv1.RollingUpdateDeployment{ - MaxUnavailable: &deploymentMaxUnavailable, - MaxSurge: &deploymentMaxSurge, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.RelayDeploymentName, - Labels: defaults.RelayDeploymentLabels, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyAlways, - ServiceAccountName: defaults.RelayServiceAccountName, - Containers: []corev1.Container{ - { - Name: defaults.RelayContainerName, - Command: []string{"hubble-relay"}, - Args: []string{ - "serve", - }, - Image: k.relayImage(utils.ImagePathIncludeDigest), - ImagePullPolicy: corev1.PullIfNotPresent, - Ports: []corev1.ContainerPort{ - { - Name: relayPortName, - ContainerPort: relayPort, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "hubble-sock-dir", - MountPath: "/var/run/cilium", - ReadOnly: true, - }, - { - Name: "config", - MountPath: "/etc/hubble-relay", - ReadOnly: true, - }, - { - Name: "tls", - MountPath: "/var/lib/hubble-relay/tls", - ReadOnly: true, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: relayPortIntstr, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: relayPortIntstr, - }, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: defaults.RelayConfigMapName, - }, - Items: []corev1.KeyToPath{ - {Key: "config.yaml", Path: "config.yaml"}, - }, - }, - }, - }, - { - Name: "hubble-sock-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/run/cilium", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "tls", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - DefaultMode: &secretDefaultMode, - Sources: []corev1.VolumeProjection{ - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: defaults.RelayClientSecretName, - }, - Items: []corev1.KeyToPath{ - { - Key: corev1.TLSCertKey, - Path: "client.crt", - }, - { - Key: corev1.TLSPrivateKeyKey, - Path: "client.key", - }, - { - Key: defaults.CASecretCertName, - Path: "hubble-server-ca.crt", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - return d + deploymentFile := k.manifests[deployFilename] + + var deploy appsv1.Deployment + utils.MustUnmarshalYAML([]byte(deploymentFile), &deploy) + return &deploy, nil } -func (k *K8sHubble) generateRelayConfigMap() *corev1.ConfigMap { - - var config = ` -peer-service: ` + defaults.HubbleSocketPath + ` -listen-address: ` + fmt.Sprintf("%s:%d", defaults.RelayListenHost, defaults.RelayPort) + ` -dial-timeout: ~ -retry-timeout: ~ -sort-buffer-len-max: ~ -sort-buffer-drain-timeout: ~ -tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt -tls-client-key-file: /var/lib/hubble-relay/tls/client.key -tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt -disable-server-tls: true -` - - //{{- if .Values.hubble.relay.tls.server.enabled }} - //tls-server-cert-file: /var/lib/hubble-relay/tls/server.crt - //tls-server-key-file: /var/lib/hubble-relay/tls/server.key - //{{- else }} - //{{- end }} - //{{- end }} - - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.RelayConfigMapName, - }, - Data: map[string]string{ - "config.yaml": config, - }, +func (k *K8sHubble) generateRelayConfigMap() (*corev1.ConfigMap, error) { + var ( + cmFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + cmFilename = "templates/hubble-relay/configmap.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } -} -func (k *K8sHubble) relayImage(imagePathMode utils.ImagePathMode) string { - return utils.BuildImagePath(k.params.RelayImage, k.params.RelayVersion, defaults.RelayImage, k.ciliumVersion, imagePathMode) + cmFile := k.manifests[cmFilename] + + var cm corev1.ConfigMap + utils.MustUnmarshalYAML([]byte(cmFile), &cm) + return &cm, nil } func (k *K8sHubble) disableRelay(ctx context.Context) error { k.Log("🔥 Deleting Relay...") - k.client.DeleteService(ctx, k.params.Namespace, defaults.RelayServiceName, metav1.DeleteOptions{}) - k.client.DeleteDeployment(ctx, k.params.Namespace, defaults.RelayDeploymentName, metav1.DeleteOptions{}) - k.client.DeleteClusterRoleBinding(ctx, defaults.RelayClusterRoleName, metav1.DeleteOptions{}) - k.client.DeleteClusterRole(ctx, defaults.RelayClusterRoleName, metav1.DeleteOptions{}) - k.client.DeleteServiceAccount(ctx, k.params.Namespace, defaults.RelayServiceAccountName, metav1.DeleteOptions{}) - k.client.DeleteConfigMap(ctx, k.params.Namespace, defaults.RelayConfigMapName, metav1.DeleteOptions{}) - k.deleteRelayCertificates(ctx) + relaySvc, err := k.generateRelayService() + if err != nil { + return err + } + k.client.DeleteService(ctx, relaySvc.GetNamespace(), relaySvc.GetName(), metav1.DeleteOptions{}) - return nil + relayDeployment, err := k.generateRelayDeployment() + if err != nil { + return err + } + k.client.DeleteDeployment(ctx, relayDeployment.GetNamespace(), relayDeployment.GetName(), metav1.DeleteOptions{}) + + crb := k.NewClusterRoleBinding(defaults.RelayClusterRoleName) + k.client.DeleteClusterRoleBinding(ctx, crb.GetName(), metav1.DeleteOptions{}) + + cr := k.NewClusterRole(defaults.RelayClusterRoleName) + k.client.DeleteClusterRole(ctx, cr.GetName(), metav1.DeleteOptions{}) + + sa := k.NewServiceAccount(defaults.RelayServiceAccountName) + k.client.DeleteServiceAccount(ctx, sa.GetNamespace(), sa.GetName(), metav1.DeleteOptions{}) + + relayConfigMap, err := k.generateRelayConfigMap() + if err != nil { + return err + } + k.client.DeleteConfigMap(ctx, relayConfigMap.GetNamespace(), relayConfigMap.GetName(), metav1.DeleteOptions{}) + + return k.deleteRelayCertificates(ctx) } -func (k *K8sHubble) enableRelay(ctx context.Context) error { - _, err := k.client.GetDeployment(ctx, k.params.Namespace, defaults.RelayDeploymentName, metav1.GetOptions{}) +func (k *K8sHubble) enableRelay(ctx context.Context) (string, error) { + relayDeployment, err := k.generateRelayDeployment() + if err != nil { + return "", err + } + + _, err = k.client.GetDeployment(ctx, relayDeployment.GetNamespace(), relayDeployment.GetName(), metav1.GetOptions{}) if err == nil { k.Log("✅ Relay is already deployed") - return nil + return relayDeployment.GetName(), nil } + k.Log("✨ Generating certificates...") + if err := k.createRelayCertificates(ctx); err != nil { - return err + return "", err } - // k.Log("✨ Generating certificates...") - - k.Log("✨ Deploying Relay from %s...", k.relayImage(utils.ImagePathExcludeDigest)) - if _, err := k.client.CreateConfigMap(ctx, k.params.Namespace, k.generateRelayConfigMap(), metav1.CreateOptions{}); err != nil { - return err + relayCm, err := k.generateRelayConfigMap() + if err != nil { + return "", err } - - if _, err := k.client.CreateServiceAccount(ctx, k.params.Namespace, k8s.NewServiceAccount(defaults.RelayServiceAccountName), metav1.CreateOptions{}); err != nil { - return err + k.Log("✨ Deploying Relay...") + if _, err := k.client.CreateConfigMap(ctx, relayCm.GetNamespace(), relayCm, metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateClusterRole(ctx, relayClusterRole, metav1.CreateOptions{}); err != nil { - return err + sa := k.NewServiceAccount(defaults.RelayServiceAccountName) + if _, err := k.client.CreateServiceAccount(ctx, sa.GetNamespace(), sa, metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateClusterRoleBinding(ctx, k8s.NewClusterRoleBinding(defaults.RelayClusterRoleName, k.params.Namespace, defaults.RelayServiceAccountName), metav1.CreateOptions{}); err != nil { - return err + if _, err := k.client.CreateDeployment(ctx, relayDeployment.GetNamespace(), relayDeployment, metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateDeployment(ctx, k.params.Namespace, k.generateRelayDeployment(), metav1.CreateOptions{}); err != nil { - return err + relaySvc, err := k.generateRelayService() + if err != nil { + return "", err } - - //relayService.Spec.Type = corev1.ServiceType(k.params.ServiceType) - if _, err := k.client.CreateService(ctx, k.params.Namespace, k.generateRelayService(), metav1.CreateOptions{}); err != nil { - return err + if _, err := k.client.CreateService(ctx, relaySvc.GetNamespace(), relaySvc, metav1.CreateOptions{}); err != nil { + return "", err } - return nil + return relayDeployment.GetName(), nil } func (k *K8sHubble) deleteRelayCertificates(ctx context.Context) error { k.Log("🔥 Deleting Relay certificates...") - k.client.DeleteSecret(ctx, k.params.Namespace, defaults.RelayServerSecretName, metav1.DeleteOptions{}) - k.client.DeleteSecret(ctx, k.params.Namespace, defaults.RelayClientSecretName, metav1.DeleteOptions{}) + secret, err := k.generateRelayCertificate(defaults.RelayServerSecretName) + if err != nil { + return err + } + + k.client.DeleteSecret(ctx, secret.GetNamespace(), secret.GetName(), metav1.DeleteOptions{}) + + secret, err = k.generateRelayCertificate(defaults.RelayClientSecretName) + if err != nil { + return err + } + k.client.DeleteSecret(ctx, secret.GetNamespace(), secret.GetName(), metav1.DeleteOptions{}) return nil } func (k *K8sHubble) createRelayCertificates(ctx context.Context) error { k.Log("🔑 Generating certificates for Relay...") - if err := k.createRelayServerCertificate(ctx); err != nil { - return err - } + // TODO we won't generate hubble-ui certificates because we don't want + // to give a bad UX for hubble-cli (which connects to hubble-relay) + // if err := k.createRelayServerCertificate(ctx); err != nil { + // return err + // } return k.createRelayClientCertificate(ctx) } -func (k *K8sHubble) createRelayServerCertificate(ctx context.Context) error { - certReq := &csr.CertificateRequest{ - Names: []csr.Name{{C: "US", ST: "San Francisco", L: "CA"}}, - KeyRequest: csr.NewKeyRequest(), - Hosts: []string{"*.hubble-relay.cilium.io"}, - CN: "*.hubble-relay.cilium.io", - } - - signConf := &config.Signing{ - Default: &config.SigningProfile{Expiry: 5 * 365 * 24 * time.Hour}, - Profiles: map[string]*config.SigningProfile{ - defaults.RelayServerSecretName: { - Expiry: 3 * 365 * 24 * time.Hour, - Usage: []string{"signing", "key encipherment", "server auth", "client auth"}, - }, - }, - } +// TODO we won't generate hubble-ui certificates because we don't want +// to give a bad UX for hubble-cli (which connects to hubble-relay) +// func (k *K8sHubble) createRelayServerCertificate(ctx context.Context) error { +// secret, err := k.generateRelayCertificate(defaults.RelayServerSecretName) +// if err != nil { +// return err +// } +// +// _, err = k.client.CreateSecret(ctx, secret.GetNamespace(), &secret, metav1.CreateOptions{}) +// if err != nil { +// return fmt.Errorf("unable to create secret %s/%s: %w", secret.GetNamespace(), secret.GetName(), err) +// } +// +// return nil +// } - cert, key, err := k.certManager.GenerateCertificate(defaults.RelayServerSecretName, certReq, signConf) +func (k *K8sHubble) createRelayClientCertificate(ctx context.Context) error { + secret, err := k.generateRelayCertificate(defaults.RelayClientSecretName) if err != nil { - return fmt.Errorf("unable to generate certificate %s: %w", defaults.RelayServerSecretName, err) - } - - data := map[string][]byte{ - corev1.TLSCertKey: cert, - corev1.TLSPrivateKeyKey: key, - defaults.CASecretCertName: k.certManager.CACertBytes(), + return err } - _, err = k.client.CreateSecret(ctx, k.params.Namespace, k8s.NewTLSSecret(defaults.RelayServerSecretName, k.params.Namespace, data), metav1.CreateOptions{}) + _, err = k.client.CreateSecret(ctx, secret.GetNamespace(), &secret, metav1.CreateOptions{}) if err != nil { - return fmt.Errorf("unable to create secret %s/%s: %w", k.params.Namespace, defaults.RelayServerSecretName, err) + return fmt.Errorf("unable to create secret %s/%s: %w", secret.GetNamespace(), secret.GetName(), err) } return nil } -func (k *K8sHubble) createRelayClientCertificate(ctx context.Context) error { - certReq := &csr.CertificateRequest{ - Names: []csr.Name{{C: "US", ST: "San Francisco", L: "CA"}}, - KeyRequest: csr.NewKeyRequest(), - Hosts: []string{"*.hubble-relay.cilium.io"}, - CN: "*.hubble-relay.cilium.io", +func (k *K8sHubble) generateRelayCertificate(name string) (corev1.Secret, error) { + var ( + relaySecretFilename string + ) + + ciliumVer := k.semVerCiliumVersion + + switch { + case versioncheck.MustCompile(">1.10.99")(ciliumVer): + switch name { + case defaults.RelayServerSecretName: + relaySecretFilename = "templates/hubble/tls-helm/relay-server-secret.yaml" + case defaults.RelayClientSecretName: + relaySecretFilename = "templates/hubble/tls-helm/relay-client-secret.yaml" + } } - signConf := &config.Signing{ - Default: &config.SigningProfile{Expiry: 5 * 365 * 24 * time.Hour}, - Profiles: map[string]*config.SigningProfile{ - defaults.RelayClientSecretName: { - Expiry: 3 * 365 * 24 * time.Hour, - Usage: []string{"signing", "key encipherment", "server auth", "client auth"}, - }, - }, + relayFile := k.manifests[relaySecretFilename] + + var secret corev1.Secret + utils.MustUnmarshalYAML([]byte(relayFile), &secret) + return secret, nil +} + +func (k *K8sHubble) PortForwardCommand(ctx context.Context) error { + var err error + k.ciliumVersion, err = k.client.GetRunningCiliumVersion(ctx, k.params.Namespace) + if err != nil { + return err } - cert, key, err := k.certManager.GenerateCertificate(defaults.RelayClientSecretName, certReq, signConf) + k.semVerCiliumVersion = k.getCiliumVersion() + + cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("unable to generate certificate %s: %w", defaults.RelayClientSecretName, err) + return fmt.Errorf("unable to retrieve ConfigMap %q: %w", defaults.ConfigMapName, err) } - data := map[string][]byte{ - corev1.TLSCertKey: cert, - corev1.TLSPrivateKeyKey: key, - defaults.CASecretCertName: k.certManager.CACertBytes(), + value, ok := cm.Data[defaults.ExtraConfigMapUserOptsKey] + if !ok { + return fmt.Errorf("configmap option not found") } - _, err = k.client.CreateSecret(ctx, k.params.Namespace, k8s.NewTLSSecret(defaults.RelayClientSecretName, k.params.Namespace, data), metav1.CreateOptions{}) + // Generate the manifests has if hubble was being enabled so that we can + // retrieve all UI and Relay's resource names. + k.params.UI = true + k.params.Relay = true + err = k.generateManifestsEnable(ctx, false, value) if err != nil { - return fmt.Errorf("unable to create secret %s/%s: %w", k.params.Namespace, defaults.RelayClientSecretName, err) + return err } - return nil -} - -func (p *Parameters) PortForwardCommand(ctx context.Context) error { + relaySvc, err := k.generateRelayService() + if err != nil { + return err + } args := []string{ "port-forward", - "-n", p.Namespace, + "-n", k.params.Namespace, "svc/hubble-relay", "--address", "0.0.0.0", "--address", "::", - fmt.Sprintf("%d:%d", p.PortForward, defaults.RelayServicePlaintextPort)} + fmt.Sprintf("%d:%d", k.params.PortForward, relaySvc.Spec.Ports[0].Port)} - if p.Context != "" { - args = append([]string{"--context", p.Context}, args...) + if k.params.Context != "" { + args = append([]string{"--context", k.params.Context}, args...) } - _, err := utils.Exec(p, "kubectl", args...) + _, err = utils.Exec(k, "kubectl", args...) return err } diff --git a/hubble/relay_test.go b/hubble/relay_test.go index 2844ef8061..447b204415 100644 --- a/hubble/relay_test.go +++ b/hubble/relay_test.go @@ -4,6 +4,7 @@ import ( "strconv" "testing" + "github.com/cilium/cilium-cli/defaults" "github.com/cilium/cilium-cli/internal/utils" ) @@ -45,3 +46,7 @@ func TestK8sHubbleRelayImage(t *testing.T) { }) } } + +func (k *K8sHubble) relayImage(imagePathMode utils.ImagePathMode) string { + return utils.BuildImagePath(k.params.RelayImage, k.params.RelayVersion, defaults.RelayImage, k.ciliumVersion, imagePathMode) +} diff --git a/hubble/ui.go b/hubble/ui.go index 1d03a37c6a..96f28c134c 100644 --- a/hubble/ui.go +++ b/hubble/ui.go @@ -9,307 +9,216 @@ import ( "io" "time" + "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/utils" + + "github.com/cilium/cilium/pkg/versioncheck" "github.com/pkg/browser" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "github.com/cilium/cilium-cli/defaults" - "github.com/cilium/cilium-cli/internal/utils" - "github.com/cilium/cilium-cli/k8s" ) -var ( - hubbleUIReplicas = int32(1) - hubbleUIPortIntstr = intstr.FromInt(8081) - hubbleUIUser = int64(1001) -) +func (k *K8sHubble) generateHubbleUIService() (*corev1.Service, error) { + var ( + svcFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + svcFilename = "templates/hubble-ui/service.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) + } -var hubbleUIClusterRole = &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.HubbleUIClusterRoleName, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"componentstatuses", "endpoints", "namespaces", "nodes", "pods", "services"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"apiextensions.k8s.io"}, - Resources: []string{"customresourcedefinitions"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"cilium.io"}, - Resources: []string{"*"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, -} + svcFile := k.manifests[svcFilename] -func (k *K8sHubble) generateHubbleUIService() *corev1.Service { - s := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.HubbleUIServiceName, - Labels: defaults.HubbleUIDeploymentLabels, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{ - { - Port: int32(80), - TargetPort: hubbleUIPortIntstr, - }, - }, - Selector: defaults.HubbleUIDeploymentLabels, - }, - } - return s + var svc corev1.Service + utils.MustUnmarshalYAML([]byte(svcFile), &svc) + return &svc, nil } -func (k *K8sHubble) generateHubbleUIConfigMap() *corev1.ConfigMap { - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.HubbleUIConfigMapName, - }, - Data: map[string]string{ - "envoy.yaml": `static_resources: - listeners: - - name: listener_hubble_ui - address: - socket_address: - address: 0.0.0.0 - port_value: 8081 - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - codec_type: auto - stat_prefix: ingress_http - route_config: - name: local_route - virtual_hosts: - - name: local_service - domains: ["*"] - routes: - - match: - prefix: "/api/" - route: - cluster: backend - prefix_rewrite: "/" - timeout: 0s - max_stream_duration: - grpc_timeout_header_max: 0s - - match: - prefix: "/" - route: - cluster: frontend - cors: - allow_origin_string_match: - - prefix: "*" - allow_methods: GET, PUT, DELETE, POST, OPTIONS - allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout - max_age: "1728000" - expose_headers: grpc-status,grpc-message - http_filters: - - name: envoy.filters.http.grpc_web - - name: envoy.filters.http.cors - - name: envoy.filters.http.router - clusters: - - name: frontend - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: frontend - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 8080 - - name: backend - connect_timeout: 0.25s - type: logical_dns - lb_policy: round_robin - http2_protocol_options: {} - load_assignment: - cluster_name: backend - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 8090 -`, - }, +func (k *K8sHubble) generateHubbleUIConfigMap() (*corev1.ConfigMap, error) { + var ( + cmFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + cmFilename = "templates/hubble-ui/configmap.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } + + cmFile := k.manifests[cmFilename] + + var cm corev1.ConfigMap + utils.MustUnmarshalYAML([]byte(cmFile), &cm) + return &cm, nil } -func (k *K8sHubble) generateHubbleUIDeployment() *appsv1.Deployment { - d := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.HubbleUIDeploymentName, - Labels: defaults.HubbleUIDeploymentLabels, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &hubbleUIReplicas, - Selector: &metav1.LabelSelector{ - MatchLabels: defaults.HubbleUIDeploymentLabels, - }, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.RollingUpdateDeploymentStrategyType, - RollingUpdate: &appsv1.RollingUpdateDeployment{ - MaxUnavailable: &deploymentMaxUnavailable, - MaxSurge: &deploymentMaxSurge, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.HubbleUIDeploymentName, - Labels: defaults.HubbleUIDeploymentLabels, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyAlways, - ServiceAccountName: defaults.HubbleUIServiceAccountName, - SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: &hubbleUIUser, - }, - Containers: []corev1.Container{ - { - Name: "frontend", - Image: k.uiImage(utils.ImagePathIncludeDigest), - ImagePullPolicy: corev1.PullIfNotPresent, - Ports: []corev1.ContainerPort{ - { - Name: "http", - ContainerPort: 8080, - }, - }, - }, - { - Name: "backend", - Image: k.uiBackendImage(utils.ImagePathIncludeDigest), - ImagePullPolicy: corev1.PullIfNotPresent, - Env: []corev1.EnvVar{ - {Name: "EVENTS_SERVER_PORT", Value: "8090"}, - {Name: "FLOWS_API_ADDR", Value: "hubble-relay:80"}, - }, - Ports: []corev1.ContainerPort{ - { - Name: "grpc", - ContainerPort: 8090, - }, - }, - }, - { - Name: "proxy", - Image: "docker.io/envoyproxy/envoy:v1.18.2@sha256:e8b37c1d75787dd1e712ff389b0d37337dc8a174a63bed9c34ba73359dc67da7", - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"envoy"}, - Args: []string{"-c", "/etc/envoy.yaml", "-l", "info"}, - Env: []corev1.EnvVar{ - {Name: "EVENTS_SERVER_PORT", Value: "8090"}, - {Name: "FLOWS_API_ADDR", Value: "hubble-relay:80"}, - }, - Ports: []corev1.ContainerPort{ - { - Name: "http", - ContainerPort: 8081, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "hubble-ui-envoy-yaml", - MountPath: "/etc/envoy.yaml", - SubPath: "envoy.yaml", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "hubble-ui-envoy-yaml", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: defaults.HubbleUIConfigMapName, - }, - }, - }, - }, - }, - }, - }, - }, +func (k *K8sHubble) generateHubbleUIDeployment() (*appsv1.Deployment, error) { + var ( + deployFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + deployFilename = "templates/hubble-ui/deployment.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } - return d -} -func (k *K8sHubble) uiImage(imagePathMode utils.ImagePathMode) string { - return utils.BuildImagePath(k.params.UIImage, k.params.UIVersion, defaults.HubbleUIImage, defaults.HubbleUIVersion, imagePathMode) -} + deploymentFile := k.manifests[deployFilename] -func (k *K8sHubble) uiBackendImage(imagePathMode utils.ImagePathMode) string { - return utils.BuildImagePath(k.params.UIBackendImage, k.params.UIVersion, defaults.HubbleUIBackendImage, defaults.HubbleUIVersion, imagePathMode) + var deploy appsv1.Deployment + utils.MustUnmarshalYAML([]byte(deploymentFile), &deploy) + return &deploy, nil } func (k *K8sHubble) disableUI(ctx context.Context) error { k.Log("🔥 Deleting Hubble UI...") - k.client.DeleteService(ctx, k.params.Namespace, defaults.HubbleUIServiceName, metav1.DeleteOptions{}) - k.client.DeleteDeployment(ctx, k.params.Namespace, defaults.HubbleUIDeploymentName, metav1.DeleteOptions{}) - k.client.DeleteClusterRoleBinding(ctx, defaults.HubbleUIClusterRoleName, metav1.DeleteOptions{}) - k.client.DeleteClusterRole(ctx, defaults.HubbleUIClusterRoleName, metav1.DeleteOptions{}) - k.client.DeleteServiceAccount(ctx, k.params.Namespace, defaults.HubbleUIServiceAccountName, metav1.DeleteOptions{}) - k.client.DeleteConfigMap(ctx, k.params.Namespace, defaults.HubbleUIConfigMapName, metav1.DeleteOptions{}) + + hubbleUISvc, err := k.generateHubbleUIService() + if err != nil { + return err + } + k.client.DeleteService(ctx, hubbleUISvc.GetNamespace(), hubbleUISvc.GetName(), metav1.DeleteOptions{}) + + hubbleUIDeploy, err := k.generateHubbleUIDeployment() + if err != nil { + return err + } + k.client.DeleteDeployment(ctx, hubbleUIDeploy.GetNamespace(), hubbleUIDeploy.GetName(), metav1.DeleteOptions{}) + + crb := k.NewClusterRoleBinding(defaults.HubbleUIClusterRoleName) + k.client.DeleteClusterRoleBinding(ctx, crb.GetName(), metav1.DeleteOptions{}) + + cr := k.NewClusterRole(defaults.HubbleUIClusterRoleName) + k.client.DeleteClusterRole(ctx, cr.GetName(), metav1.DeleteOptions{}) + + sa := k.NewServiceAccount(defaults.HubbleUIServiceAccountName) + k.client.DeleteServiceAccount(ctx, sa.GetNamespace(), sa.GetName(), metav1.DeleteOptions{}) + + hubbleUICM, err := k.generateHubbleUIConfigMap() + if err != nil { + return err + } + k.client.DeleteConfigMap(ctx, hubbleUICM.GetNamespace(), hubbleUICM.GetName(), metav1.DeleteOptions{}) + + return k.deleteUICertificates(ctx) +} + +func (k *K8sHubble) deleteUICertificates(ctx context.Context) error { + k.Log("🔥 Deleting Hubble UI certificates...") + secret, err := k.generateUICertificate(defaults.HubbleUIClientSecretName) + if err != nil { + return err + } + + k.client.DeleteSecret(ctx, secret.GetNamespace(), secret.GetName(), metav1.DeleteOptions{}) return nil } -func (k *K8sHubble) enableUI(ctx context.Context) error { - _, err := k.client.GetDeployment(ctx, k.params.Namespace, defaults.HubbleUIDeploymentName, metav1.GetOptions{}) +func (k *K8sHubble) enableUI(ctx context.Context) (string, error) { + hubbleUIDeploy, err := k.generateHubbleUIDeployment() + if err != nil { + return "", err + } + + _, err = k.client.GetDeployment(ctx, hubbleUIDeploy.GetNamespace(), hubbleUIDeploy.GetName(), metav1.GetOptions{}) if err == nil { k.Log("✅ Hubble UI is already deployed") - return nil + return hubbleUIDeploy.GetName(), nil } - k.Log("✨ Deploying Hubble UI from %s and Hubble UI Backend from %s...", k.uiImage(utils.ImagePathExcludeDigest), k.uiBackendImage(utils.ImagePathExcludeDigest)) - if _, err := k.client.CreateConfigMap(ctx, k.params.Namespace, k.generateHubbleUIConfigMap(), metav1.CreateOptions{}); err != nil { - return err + // TODO we won't generate hubble-ui certificates because we don't want + // to give a bad UX for hubble-cli (which connects to hubble-relay) + // k.Log("✨ Generating certificates...") + // + // if err := k.createUICertificates(ctx); err != nil { + // return "", err + // } + + hubbleUICM, err := k.generateHubbleUIConfigMap() + if err != nil { + return "", err } - if _, err := k.client.CreateServiceAccount(ctx, k.params.Namespace, k8s.NewServiceAccount(defaults.HubbleUIServiceAccountName), metav1.CreateOptions{}); err != nil { - return err + k.Log("✨ Deploying Hubble UI and Hubble UI Backend...") + if _, err := k.client.CreateConfigMap(ctx, hubbleUICM.GetNamespace(), hubbleUICM, metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateClusterRole(ctx, hubbleUIClusterRole, metav1.CreateOptions{}); err != nil { - return err + sa := k.NewServiceAccount(defaults.HubbleUIServiceAccountName) + if _, err := k.client.CreateServiceAccount(ctx, sa.GetNamespace(), sa, metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateClusterRoleBinding(ctx, k8s.NewClusterRoleBinding(defaults.HubbleUIClusterRoleName, k.params.Namespace, defaults.HubbleUIServiceAccountName), metav1.CreateOptions{}); err != nil { - return err + if _, err := k.client.CreateClusterRole(ctx, k.NewClusterRole(defaults.HubbleUIClusterRoleName), metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateDeployment(ctx, k.params.Namespace, k.generateHubbleUIDeployment(), metav1.CreateOptions{}); err != nil { - return err + if _, err := k.client.CreateClusterRoleBinding(ctx, k.NewClusterRoleBinding(defaults.HubbleUIClusterRoleName), metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateService(ctx, k.params.Namespace, k.generateHubbleUIService(), metav1.CreateOptions{}); err != nil { - return err + if _, err := k.client.CreateDeployment(ctx, hubbleUIDeploy.GetNamespace(), hubbleUIDeploy, metav1.CreateOptions{}); err != nil { + return "", err } - return nil + hubbleUISvc, err := k.generateHubbleUIService() + if err != nil { + return "", err + } + if _, err := k.client.CreateService(ctx, hubbleUISvc.GetNamespace(), hubbleUISvc, metav1.CreateOptions{}); err != nil { + return "", err + } + + return hubbleUIDeploy.GetName(), nil +} + +// TODO we won't generate hubble-ui certificates because we don't want +// to give a bad UX for hubble-cli (which connects to hubble-relay) +// func (k *K8sHubble) createUICertificates(ctx context.Context) error { +// k.Log("🔑 Generating certificates for UI...") +// secret, err := k.generateUICertificate(defaults.HubbleUIClientSecretName) +// if err != nil { +// return err +// } +// +// _, err = k.client.CreateSecret(ctx, secret.GetNamespace(), &secret, metav1.CreateOptions{}) +// if err != nil { +// return fmt.Errorf("unable to create secret %s/%s: %w", secret.GetNamespace(), secret.GetName(), err) +// } +// +// return nil +// } + +func (k *K8sHubble) generateUICertificate(name string) (corev1.Secret, error) { + var ( + relaySecretFilename string + ) + + ciliumVer := k.semVerCiliumVersion + + switch { + case versioncheck.MustCompile(">1.10.99")(ciliumVer): + switch name { + case defaults.HubbleUIClientSecretName: + relaySecretFilename = "templates/hubble/tls-helm/ui-client-certs.yaml" + } + } + + relayFile := k.manifests[relaySecretFilename] + + var secret corev1.Secret + utils.MustUnmarshalYAML([]byte(relayFile), &secret) + return secret, nil } func (p *Parameters) UIPortForwardCommand(ctx context.Context) error { diff --git a/install/helm.go b/install/helm.go index 41e92049d0..1a1cced319 100644 --- a/install/helm.go +++ b/install/helm.go @@ -231,7 +231,7 @@ func (k *K8sInstaller) generateManifests(ctx context.Context) error { extraConfigMap[k] = v } - vals, err := helm.MergeVals(k, k.params.HelmOpts, helmMapOpts, nil, extraConfigMap, k.params.HelmChartDirectory, ciliumVer.String(), k.params.Namespace) + vals, err := helm.MergeVals(k, true, k.params.HelmOpts, helmMapOpts, nil, extraConfigMap, k.params.HelmChartDirectory, ciliumVer.String(), k.params.Namespace) if err != nil { return err } diff --git a/internal/certs/certs.go b/internal/certs/certs.go index 57627c95e0..a4f2bcd029 100644 --- a/internal/certs/certs.go +++ b/internal/certs/certs.go @@ -159,3 +159,13 @@ func (c *CertManager) CACertBytes() []byte { copy(crt, c.caCert) return crt } + +// CAKeyBytes return the CA private certificate bytes, or nil when it is not +// set. +func (c *CertManager) CAKeyBytes() []byte { + // NOTE: return a copy just to avoid the caller modifiying our CA + // certificate. + crt := make([]byte, len(c.caKey)) + copy(crt, c.caKey) + return crt +} diff --git a/internal/cli/cmd/hubble.go b/internal/cli/cmd/hubble.go index 27b05aba01..1d2c004fb0 100644 --- a/internal/cli/cmd/hubble.go +++ b/internal/cli/cmd/hubble.go @@ -7,10 +7,10 @@ import ( "context" "os" - "github.com/spf13/cobra" - "github.com/cilium/cilium-cli/defaults" "github.com/cilium/cilium-cli/hubble" + + "github.com/spf13/cobra" ) func newCmdHubble() *cobra.Command { @@ -50,18 +50,46 @@ func newCmdHubbleEnable() *cobra.Command { cmd.Flags().StringVarP(¶ms.Namespace, "namespace", "n", "kube-system", "Namespace Cilium is running in") cmd.Flags().BoolVar(¶ms.Relay, "relay", true, "Deploy Hubble Relay") + // It can be deprecated since we have a helm option for it cmd.Flags().StringVar(¶ms.RelayImage, "relay-image", "", "Image path to use for Relay") + // It can be deprecated since we have a helm option for it cmd.Flags().StringVar(¶ms.RelayVersion, "relay-version", "", "Version of Relay to deploy") + // It can be deprecated since there is not a helm option for it and cmd.Flags().StringVar(¶ms.RelayServiceType, "relay-service-type", "ClusterIP", "Type of Kubernetes service to expose Hubble Relay") + cmd.Flags().MarkDeprecated("relay-service-type", "value is no longer used for relay-service") cmd.Flags().BoolVar(¶ms.UI, "ui", false, "Enable Hubble UI") + + // It can be deprecated since we have a helm option for it cmd.Flags().StringVar(¶ms.UIImage, "ui-image", "", "Image path to use for UI") + // It can be deprecated since we have a helm option for it cmd.Flags().StringVar(¶ms.UIBackendImage, "ui-backend-image", "", "Image path to use for UI backend") + // It can be deprecated since we have a helm option for it cmd.Flags().StringVar(¶ms.UIVersion, "ui-version", "", "Version of UI to deploy") cmd.Flags().BoolVar(¶ms.CreateCA, "create-ca", true, "Automatically create CA if needed") cmd.Flags().StringVar(&contextName, "context", "", "Kubernetes configuration context") cmd.Flags().BoolVar(¶ms.Wait, "wait", true, "Wait for status to report success (no errors)") cmd.Flags().DurationVar(¶ms.WaitDuration, "wait-duration", defaults.StatusWaitDuration, "Maximum time to wait for status") + cmd.Flags().StringVar(¶ms.K8sVersion, "k8s-version", "", "Kubernetes server version in case auto-detection fails") + cmd.Flags().StringVar(¶ms.BaseVersion, "base-version", defaults.Version, + "Specify the base Cilium version for configuration purpose in case the --version flag doesn't indicate the actual Cilium version") + cmd.Flags().StringVar(¶ms.HelmChartDirectory, "chart-directory", "", "Helm chart directory") + cmd.Flags().StringSliceVar(¶ms.HelmOpts.ValueFiles, "helm-values", []string{}, "Specify helm values in a YAML file or a URL (can specify multiple)") + cmd.Flags().StringArrayVar(¶ms.HelmOpts.Values, "helm-set", []string{}, "Set helm values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)") + cmd.Flags().StringArrayVar(¶ms.HelmOpts.StringValues, "helm-set-string", []string{}, "Set helm STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)") + cmd.Flags().StringArrayVar(¶ms.HelmOpts.FileValues, "helm-set-file", []string{}, "Set helm values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)") + cmd.Flags().StringVar(¶ms.HelmGenValuesFile, "helm-auto-gen-values", "", "Write an auto-generated helm values into this file") + + for flagName := range hubble.FlagsToHelmOpts { + // TODO(aanm) Do not mark the flags has deprecated for now. + // msg := fmt.Sprintf("use --helm-set=%s<=value> instead", helmOpt) + // err := cmd.Flags().MarkDeprecated(flagName, msg) + // if err != nil { + // panic(err) + // } + hubble.FlagValues[flagName] = cmd.Flags().Lookup(flagName).Value + } + return cmd } @@ -99,7 +127,8 @@ func newCmdPortForwardCommand() *cobra.Command { Short: "Forward the relay port to the local machine", Long: ``, RunE: func(cmd *cobra.Command, args []string) error { - if err := params.PortForwardCommand(context.Background()); err != nil { + h := hubble.NewK8sHubble(k8sClient, params) + if err := h.PortForwardCommand(context.Background()); err != nil { fatalf("Unable to port forward: %s", err) } return nil diff --git a/internal/helm/helm.go b/internal/helm/helm.go index c919ebbdc9..87e8d07198 100644 --- a/internal/helm/helm.go +++ b/internal/helm/helm.go @@ -181,6 +181,7 @@ func GenManifests(ctx context.Context, helmChartDirectory, k8sVersion, ciliumVer // Both 'helmMapOpts', 'helmValues', 'extraConfigMapOpts' can be nil. func MergeVals( logger utils.Logger, + printHelmTemplate bool, helmFlagOpts values.Options, helmMapOpts map[string]string, helmValues map[string]interface{}, @@ -191,9 +192,6 @@ func MergeVals( // Create helm values from helmMapOpts var helmOpts []string for k, v := range helmMapOpts { - if v == "" { - panic(fmt.Sprintf("empty value form helm option %q", k)) - } helmOpts = append(helmOpts, fmt.Sprintf("%s=%s", k, v)) } @@ -231,10 +229,12 @@ func MergeVals( valsStr := valuesToString("", vals) - if helmChartDirectory != "" { - logger.Log("ℹ️ helm template --namespace %s cilium %q --version %s --set %s", namespace, helmChartDirectory, ciliumVer, valsStr) - } else { - logger.Log("ℹ️ helm template --namespace %s cilium cilium/cilium --version %s --set %s", namespace, ciliumVer, valsStr) + if printHelmTemplate { + if helmChartDirectory != "" { + logger.Log("ℹ️ helm template --namespace %s cilium %q --version %s --set %s", namespace, helmChartDirectory, ciliumVer, valsStr) + } else { + logger.Log("ℹ️ helm template --namespace %s cilium cilium/cilium --version %s --set %s", namespace, ciliumVer, valsStr) + } } // Store the current helm-opts used in this installation in Cilium's diff --git a/k8s/client.go b/k8s/client.go index c21d8db802..ccca3314e9 100644 --- a/k8s/client.go +++ b/k8s/client.go @@ -163,6 +163,10 @@ func (c *Client) PatchConfigMap(ctx context.Context, namespace, name string, pt return c.Clientset.CoreV1().ConfigMaps(namespace).Patch(ctx, name, pt, data, opts) } +func (c *Client) UpdateConfigMap(ctx context.Context, configMap *corev1.ConfigMap, opts metav1.UpdateOptions) (*corev1.ConfigMap, error) { + return c.Clientset.CoreV1().ConfigMaps(configMap.Namespace).Update(ctx, configMap, opts) +} + func (c *Client) CreateService(ctx context.Context, namespace string, service *corev1.Service, opts metav1.CreateOptions) (*corev1.Service, error) { return c.Clientset.CoreV1().Services(namespace).Create(ctx, service, opts) }