From 04427b2d73205373b9860f4e2311544704eafb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Martins?= Date: Tue, 5 Apr 2022 00:41:45 +0200 Subject: [PATCH] hubble: add helm-based installation to hubble mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By storing helm-based config values into Cilium's ConfigMap we are able to generate hubble deployment files by reading those values. Once the manifests are generated, the new helm flags used to enable hubble and/or hubble-ui and/or hubble-relay will also be store in Cilium's ConfigMap. Similarly to 'hubble enable', 'hubble disable' command also stores the helm-based config values into Cilium's ConfigMap. Signed-off-by: André Martins --- defaults/defaults.go | 40 +--- hubble/hubble.go | 423 ++++++++++++++++++++++++++------ hubble/relay.go | 479 ++++++++++++++----------------------- hubble/relay_test.go | 5 + hubble/ui.go | 372 +++++++++------------------- internal/cli/cmd/hubble.go | 35 ++- k8s/client.go | 4 + 7 files changed, 692 insertions(+), 666 deletions(-) diff --git a/defaults/defaults.go b/defaults/defaults.go index 8932063ee4..808d8afafc 100644 --- a/defaults/defaults.go +++ b/defaults/defaults.go @@ -33,24 +33,17 @@ const ( HubbleSocketPath = "/var/run/cilium/hubble.sock" HubbleServerSecretName = "hubble-server-certs" - RelayContainerName = "hubble-relay" - RelayDeploymentName = "hubble-relay" - RelayClusterRoleName = "hubble-relay" - RelayServiceAccountName = "hubble-relay" - RelayServiceName = "hubble-relay" - RelayConfigMapName = "hubble-relay-config" - RelayImage = "quay.io/cilium/hubble-relay" - RelayListenHost = "" - RelayPort = 4245 - RelayServicePlaintextPort = 80 - RelayServiceTLSPort = 443 - RelayServerSecretName = "hubble-relay-server-certs" - RelayClientSecretName = "hubble-relay-client-certs" - - HubbleUIServiceName = "hubble-ui" + RelayContainerName = "hubble-relay" + RelayDeploymentName = "hubble-relay" + RelayClusterRoleName = "hubble-relay" + RelayServiceAccountName = "hubble-relay" + RelayConfigMapName = "hubble-relay-config" + RelayImage = "quay.io/cilium/hubble-relay" + RelayServerSecretName = "hubble-relay-server-certs" + RelayClientSecretName = "hubble-relay-client-certs" + HubbleUIClusterRoleName = "hubble-ui" HubbleUIServiceAccountName = "hubble-ui" - HubbleUIConfigMapName = "hubble-ui-envoy" HubbleUIDeploymentName = "hubble-ui" HubbleUIImage = "quay.io/cilium/hubble-ui" HubbleUIBackendImage = "quay.io/cilium/hubble-ui-backend" @@ -72,9 +65,8 @@ const ( ConnectivityPerformanceImage = "quay.io/cilium/network-perf:bf58fb8bc57c4933dfa6e2a9581d3925c0a0571e@sha256:9bef508b2dcaeb3e288a496b8d3f065e8636a4937ba3aebcb1732afffaccea34" ConnectivityCheckJSONMockImage = "quay.io/cilium/json-mock:v1.3.0@sha256:2729064827fa9dbfface8d3df424feb6c792a0ba07117b844349635c93c06d2b" - ConfigMapName = "cilium-config" - Version = "v1.11.3" - HubbleUIVersion = "v0.8.5" + ConfigMapName = "cilium-config" + Version = "v1.11.3" TunnelType = "vxlan" @@ -98,16 +90,6 @@ const ( ) var ( - // RelayDeploymentLabels are the labels set on the Hubble Relay Deployment by default. - RelayDeploymentLabels = map[string]string{ - "k8s-app": "hubble-relay", - } - - // HubbleUIDeploymentLabels are the labels set on the Hubble UI Deployment by default. - HubbleUIDeploymentLabels = map[string]string{ - "k8s-app": "hubble-ui", - } - // ClusterMeshDeploymentLabels are the labels set on the clustermesh API server by default. ClusterMeshDeploymentLabels = map[string]string{ "k8s-app": "clustermesh-apiserver", diff --git a/hubble/hubble.go b/hubble/hubble.go index 20f958c533..b92569dbfc 100644 --- a/hubble/hubble.go +++ b/hubble/hubble.go @@ -7,21 +7,28 @@ import ( "context" "fmt" "io" + "os" "strings" "time" + "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/certs" + "github.com/cilium/cilium-cli/internal/helm" + "github.com/cilium/cilium-cli/internal/utils" + "github.com/cilium/cilium-cli/status" + + "github.com/blang/semver/v4" "github.com/cilium/cilium/api/v1/models" ciliumv2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2" + "github.com/cilium/cilium/pkg/versioncheck" + "github.com/spf13/pflag" + "helm.sh/helm/v3/pkg/chartutil" + "helm.sh/helm/v3/pkg/cli/values" + "helm.sh/helm/v3/pkg/strvals" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - - "github.com/cilium/cilium-cli/defaults" - "github.com/cilium/cilium-cli/internal/certs" - "github.com/cilium/cilium-cli/internal/utils" - "github.com/cilium/cilium-cli/status" ) const ( @@ -29,10 +36,6 @@ const ( configNameListenAddress = "hubble-listen-address" ) -var ( - hostPathDirectoryOrCreate = corev1.HostPathDirectoryOrCreate -) - type k8sHubbleImplementation interface { CreateSecret(ctx context.Context, namespace string, secret *corev1.Secret, opts metav1.CreateOptions) (*corev1.Secret, error) DeleteSecret(ctx context.Context, namespace, name string, opts metav1.DeleteOptions) error @@ -46,7 +49,7 @@ type k8sHubbleImplementation interface { CreateConfigMap(ctx context.Context, namespace string, config *corev1.ConfigMap, opts metav1.CreateOptions) (*corev1.ConfigMap, error) DeleteConfigMap(ctx context.Context, namespace, name string, opts metav1.DeleteOptions) error GetConfigMap(ctx context.Context, namespace, name string, opts metav1.GetOptions) (*corev1.ConfigMap, error) - PatchConfigMap(ctx context.Context, namespace, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*corev1.ConfigMap, error) + UpdateConfigMap(ctx context.Context, configMap *corev1.ConfigMap, opts metav1.UpdateOptions) (*corev1.ConfigMap, error) CreateDeployment(ctx context.Context, namespace string, deployment *appsv1.Deployment, opts metav1.CreateOptions) (*appsv1.Deployment, error) GetDeployment(ctx context.Context, namespace, name string, opts metav1.GetOptions) (*appsv1.Deployment, error) DeleteDeployment(ctx context.Context, namespace, name string, opts metav1.DeleteOptions) error @@ -58,15 +61,32 @@ type k8sHubbleImplementation interface { CiliumStatus(ctx context.Context, namespace, pod string) (*models.StatusResponse, error) ListCiliumEndpoints(ctx context.Context, namespace string, opts metav1.ListOptions) (*ciliumv2.CiliumEndpointList, error) GetRunningCiliumVersion(ctx context.Context, namespace string) (string, error) + GetServerVersion() (*semver.Version, error) } type K8sHubble struct { - client k8sHubbleImplementation - params Parameters - certManager *certs.CertManager - ciliumVersion string + client k8sHubbleImplementation + params Parameters + certManager *certs.CertManager + ciliumVersion string + manifests map[string]string + semVerCiliumVersion semver.Version } +var ( + // FlagsToHelmOpts maps the deprecated install flags to the helm + // options + FlagsToHelmOpts = map[string][]string{ + "relay-image": {"hubble.relay.image.override"}, + "relay-version": {"hubble.relay.image.tag"}, + "ui-image": {"hubble.ui.frontend.image.override"}, + "ui-backend-image": {"hubble.ui.backend.image.override"}, + "ui-version": {"hubble.ui.frontend.image.tag", "hubble.ui.backend.image.tag"}, + } + // FlagValues maps all FlagsToHelmOpts keys to their values + FlagValues = map[string]pflag.Value{} +) + type Parameters struct { Namespace string Relay bool @@ -84,6 +104,26 @@ type Parameters struct { Context string // Only for 'kubectl' pass-through commands Wait bool WaitDuration time.Duration + + // BaseVersion is used to explicitly specify Cilium version for generating the config map + // in case it cannot be inferred from the Version field (e.g. commit SHA tags for CI images). + BaseVersion string + + // K8sVersion is the Kubernetes version that will be used to generate the + // kubernetes manifests. If the auto-detection fails, this flag can be used + // as a workaround. + K8sVersion string + // HelmChartDirectory points to the location of a helm chart directory. + // Useful to test from upstream where a helm release is not available yet. + HelmChartDirectory string + + // HelmOpts are all the options the user used to pass into the Cilium cli + // template. + HelmOpts values.Options + + // HelmGenValuesFile points to the file that will store the generated helm + // options. + HelmGenValuesFile string } func (p *Parameters) Log(format string, a ...interface{}) { @@ -158,51 +198,40 @@ func (k *K8sHubble) Validate(ctx context.Context) error { } -var hubbleCfg = map[string]string{ - // Enable Hubble gRPC service. - "enable-hubble": "true", - // UNIX domain socket for Hubble server to listen to. - "hubble-socket-path": defaults.HubbleSocketPath, - // An additional address for Hubble server to listen to (e.g. ":4244"). - "hubble-listen-address": ":4244", - "hubble-disable-tls": "false", - "hubble-tls-cert-file": "/var/lib/cilium/tls/hubble/server.crt", - "hubble-tls-key-file": "/var/lib/cilium/tls/hubble/server.key", - "hubble-tls-client-ca-files": "/var/lib/cilium/tls/hubble/client-ca.crt", +func (k *K8sHubble) disableHubble(ctx context.Context) error { + k.Log("✨ Patching ConfigMap %s to disable Hubble...", defaults.ConfigMapName) + + return k.updateConfigMap(ctx) } -func (k *K8sHubble) disableHubble(ctx context.Context) error { - cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{}) +func (k *K8sHubble) Disable(ctx context.Context) error { + var err error + k.ciliumVersion, err = k.client.GetRunningCiliumVersion(ctx, k.params.Namespace) if err != nil { - return fmt.Errorf("unable to get ConfigMap %s: %w", defaults.ConfigMapName, err) + return err } - var changes []string - for k := range hubbleCfg { - if _, ok := cm.Data[k]; ok { - changes = append(changes, `{"op": "remove", "path": "/data/`+k+`"}`) - } - } - if len(changes) > 0 { - patch := []byte(`[` + strings.Join(changes, ",") + `]`) + k.semVerCiliumVersion = k.getCiliumVersion() - k.Log("✨ Patching ConfigMap %s to disable Hubble...", defaults.ConfigMapName) - _, err := k.client.PatchConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, types.JSONPatchType, patch, metav1.PatchOptions{}) - if err != nil { - return fmt.Errorf("unable to patch ConfigMap %s with patch %q: %w", defaults.ConfigMapName, patch, err) - } + cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("unable to retrieve ConfigMap %q: %w", defaults.ConfigMapName, err) } - if err := k.client.DeletePodCollection(ctx, k.params.Namespace, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: defaults.CiliumPodSelector}); err != nil { - k.Log("⚠️ Unable to restart Clium pods: %s", err) - } else { - k.Log("♻️ Restarted Cilium pods") + value, ok := cm.Data[defaults.ExtraConfigMapUserOptsKey] + if !ok { + return fmt.Errorf("configmap option not found") } - return nil -} + // Generate the manifests has if hubble was being enabled so that we can + // retrieve all UI and Relay's resource names. + k.params.UI = true + k.params.Relay = true + err = k.generateManifestsEnable(ctx, value) + if err != nil { + return err + } -func (k *K8sHubble) Disable(ctx context.Context) error { if err := k.disableUI(ctx); err != nil { return err } @@ -211,6 +240,13 @@ func (k *K8sHubble) Disable(ctx context.Context) error { return err } + // Now that we have delete all UI and Relay's resource names then we can + // generate the manifests with UI and Relay disabled. + err = k.generateManifestsDisable(ctx, value) + if err != nil { + return err + } + if err := k.disableHubble(ctx); err != nil { return err } @@ -220,26 +256,170 @@ func (k *K8sHubble) Disable(ctx context.Context) error { return nil } -func (k *K8sHubble) enableHubble(ctx context.Context) error { - var changes []string - for k, v := range hubbleCfg { - changes = append(changes, `"`+k+`":"`+v+`"`) +func (k *K8sHubble) generateConfigMap() (*corev1.ConfigMap, error) { + var ( + cmFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + cmFilename = "templates/cilium-configmap.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } - patch := []byte(`{"data":{` + strings.Join(changes, ",") + `}}`) + cmFile := k.manifests[cmFilename] + + var cm corev1.ConfigMap + utils.MustUnmarshalYAML([]byte(cmFile), &cm) + k.Log("🚀 Creating ConfigMap for Cilium version %s...", ciliumVer.String()) + return &cm, nil +} + +func (k *K8sHubble) enableHubble(ctx context.Context) error { k.Log("✨ Patching ConfigMap %s to enable Hubble...", defaults.ConfigMapName) - _, err := k.client.PatchConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + + return k.updateConfigMap(ctx) +} + +func (k *K8sHubble) updateConfigMap(ctx context.Context) error { + cm, err := k.generateConfigMap() if err != nil { - return fmt.Errorf("unable to patch ConfigMap %s with patch %q: %w", defaults.ConfigMapName, patch, err) + return err + } + + _, err = k.client.UpdateConfigMap(ctx, cm, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("unable to patch ConfigMap %s with %s: %w", defaults.ConfigMapName, cm, err) } if err := k.client.DeletePodCollection(ctx, k.params.Namespace, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: defaults.CiliumPodSelector}); err != nil { - k.Log("⚠️ Unable to restart Clium pods: %s", err) + k.Log("⚠️ Unable to restart Cilium pods: %s", err) } else { k.Log("♻️ Restarted Cilium pods") } + return nil +} + +func (k *K8sHubble) getCiliumVersion() semver.Version { + v, err := utils.ParseCiliumVersion(k.ciliumVersion, k.params.BaseVersion) + if err != nil { + v = versioncheck.MustVersion(defaults.Version) + k.Log("Unable to parse the provided version %q, assuming %v for ConfigMap compatibility", k.ciliumVersion, defaults.Version) + } + return v +} + +func (k *K8sHubble) generateManifestsEnable(ctx context.Context, helmValues string) error { + ciliumVer := k.semVerCiliumVersion + + helmMapOpts := map[string]string{} + + switch { + // It's likely that certain helm options have changed since 1.9.0 + // These were tested for the >=1.11.0. In case something breaks for versions + // older than 1.11.0 we will fix it afterwards. + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + // case versioncheck.MustCompile(">=1.11.0")(ciliumVer): + + // Pre-define all deprecated flags as helm options + for flagName, helmOpts := range FlagsToHelmOpts { + if v, ok := FlagValues[flagName]; ok { + if val := v.String(); val != "" { + for _, helmOpt := range helmOpts { + helmMapOpts[helmOpt] = val + switch helmOpt { + // If the images or tag are overwritten then we need + // to disable 'useDigest' + case "hubble.relay.image.override", "hubble.relay.image.tag": + helmMapOpts["hubble.relay.image.useDigest"] = "false" + } + } + } + } + } + + helmMapOpts["hubble.enabled"] = "true" + helmMapOpts["hubble.socketPath"] = defaults.HubbleSocketPath + helmMapOpts["hubble.listenAddress"] = ":4244" + helmMapOpts["hubble.listenAddress"] = ":4244" + helmMapOpts["hubble.tls.enabled"] = "true" + + if k.params.UI { + helmMapOpts["hubble.ui.enabled"] = "true" + } + if k.params.Relay { + helmMapOpts["hubble.relay.enabled"] = "true" + helmMapOpts["hubble.relay.tls.server.enabled"] = "true" + } + + default: + return fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) + } + + return k.genManifests(ctx, helmValues, helmMapOpts, ciliumVer) +} + +func (k *K8sHubble) generateManifestsDisable(ctx context.Context, helmValues string) error { + ciliumVer := k.semVerCiliumVersion + + helmMapOpts := map[string]string{} + + switch { + // It's likely that certain helm options have changed since 1.9.0 + // These were tested for the >=1.11.0. In case something breaks for versions + // older than 1.11.0 we will fix it afterwards. + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + // case versioncheck.MustCompile(">=1.11.0")(ciliumVer): + helmMapOpts["hubble.enabled"] = "false" + helmMapOpts["hubble.ui.enabled"] = "false" + helmMapOpts["hubble.relay.enabled"] = "false" + + default: + return fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) + } + + return k.genManifests(ctx, helmValues, helmMapOpts, ciliumVer) +} +func (k *K8sHubble) genManifests(ctx context.Context, helmValues string, helmMapOpts map[string]string, ciliumVer semver.Version) error { + // Store all the options passed by --config into helm extraConfig + prevHelmValues := map[string]interface{}{} + err := strvals.ParseInto(helmValues, prevHelmValues) + if err != nil { + return fmt.Errorf("error parsing helm options %q: %w", helmValues, err) + } + + vals, err := helm.MergeVals(k, k.params.HelmOpts, helmMapOpts, prevHelmValues, nil, k.params.HelmChartDirectory, ciliumVer.String(), k.params.Namespace) + if err != nil { + return err + } + + if k.params.HelmGenValuesFile != "" { + yamlValue, err := chartutil.Values(vals).YAML() + if err != nil { + return err + } + return os.WriteFile(k.params.HelmGenValuesFile, []byte(yamlValue), 0o600) + } + + k8sVersionStr := k.params.K8sVersion + if k8sVersionStr == "" { + k8sVersion, err := k.client.GetServerVersion() + if err != nil { + return fmt.Errorf("error getting Kubernetes version, try --k8s-version: %s", err) + } + k8sVersionStr = k8sVersion.String() + } + + manifests, err := helm.GenManifests(ctx, k.params.HelmChartDirectory, k8sVersionStr, ciliumVer.String(), k.params.Namespace, vals) + if err != nil { + return err + } + + k.manifests = manifests return nil } @@ -254,6 +434,8 @@ func (k *K8sHubble) Enable(ctx context.Context) error { return err } + k.semVerCiliumVersion = k.getCiliumVersion() + caSecret, created, err := k.certManager.GetOrCreateCASecret(ctx, defaults.CASecretName, k.params.CreateCA) if err != nil { k.Log("❌ Unable to get or create the Cilium CA Secret: %s", err) @@ -273,6 +455,21 @@ func (k *K8sHubble) Enable(ctx context.Context) error { } } + cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("unable to retrieve ConfigMap %q: %w", defaults.ConfigMapName, err) + } + + value, ok := cm.Data[defaults.ExtraConfigMapUserOptsKey] + if !ok { + return fmt.Errorf("configmap option not found") + } + + err = k.generateManifestsEnable(ctx, value) + if err != nil { + return err + } + if err := k.enableHubble(ctx); err != nil { return err } @@ -299,34 +496,32 @@ func (k *K8sHubble) Enable(ctx context.Context) error { } } + var warnFreePods []string if k.params.Relay { - if err := k.enableRelay(ctx); err != nil { + podsName, err := k.enableRelay(ctx) + if err != nil { return err } + + warnFreePods = append(warnFreePods, podsName) } if k.params.UI { - if err := k.enableUI(ctx); err != nil { + podsName, err := k.enableUI(ctx) + if err != nil { return err } + + warnFreePods = append(warnFreePods, podsName) } if k.params.Wait { - var pods []string - - if k.params.Relay { - pods = append(pods, defaults.RelayDeploymentName) - } - if k.params.UI { - pods = append(pods, defaults.HubbleUIDeploymentName) - } - k.Log("⌛ Waiting for Hubble to be installed...") collector, err := status.NewK8sStatusCollector(k.client, status.K8sStatusParameters{ Namespace: k.params.Namespace, Wait: true, WaitDuration: k.params.WaitDuration - dur, - WarningFreePods: pods, + WarningFreePods: warnFreePods, }) if err != nil { return err @@ -343,3 +538,93 @@ func (k *K8sHubble) Enable(ctx context.Context) error { return nil } + +func (k *K8sHubble) NewServiceAccount(name string) *corev1.ServiceAccount { + var ( + saFileName string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">1.10.99")(ciliumVer): + switch name { + case defaults.RelayServiceAccountName: + saFileName = "templates/hubble-relay/serviceaccount.yaml" + case defaults.HubbleUIServiceAccountName: + saFileName = "templates/hubble-ui/serviceaccount.yaml" + } + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + switch name { + case defaults.RelayServiceAccountName: + saFileName = "templates/hubble-relay-serviceaccount.yaml" + case defaults.HubbleUIServiceAccountName: + saFileName = "templates/hubble-ui-serviceaccount.yaml" + } + } + + saFile := k.manifests[saFileName] + + var sa corev1.ServiceAccount + utils.MustUnmarshalYAML([]byte(saFile), &sa) + return &sa +} + +func (k *K8sHubble) NewClusterRole(name string) *rbacv1.ClusterRole { + var ( + crFileName string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">1.10.99")(ciliumVer): + switch name { + case defaults.RelayClusterRoleName: + crFileName = "templates/hubble-relay/clusterrole.yaml" + case defaults.HubbleUIClusterRoleName: + crFileName = "templates/hubble-ui/clusterrole.yaml" + } + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + switch name { + case defaults.RelayClusterRoleName: + crFileName = "templates/hubble-relay-clusterrole.yaml" + case defaults.HubbleUIClusterRoleName: + crFileName = "templates/hubble-ui-clusterrole.yaml" + } + } + + crFile := k.manifests[crFileName] + + var cr rbacv1.ClusterRole + utils.MustUnmarshalYAML([]byte(crFile), &cr) + return &cr +} + +func (k *K8sHubble) NewClusterRoleBinding(crbName string) *rbacv1.ClusterRoleBinding { + var ( + crbFileName string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">1.10.99")(ciliumVer): + switch crbName { + case defaults.RelayClusterRoleName: + crbFileName = "templates/hubble-relay/clusterrolebinding.yaml" + case defaults.HubbleUIClusterRoleName: + crbFileName = "templates/hubble-ui/clusterrolebinding.yaml" + } + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + switch crbName { + case defaults.RelayClusterRoleName: + crbFileName = "templates/hubble-relay-clusterrolebinding.yaml" + case defaults.HubbleUIClusterRoleName: + crbFileName = "templates/hubble-ui-clusterrolebinding.yaml" + } + } + + crbFile := k.manifests[crbFileName] + + var crb rbacv1.ClusterRoleBinding + utils.MustUnmarshalYAML([]byte(crbFile), &crb) + return &crb +} diff --git a/hubble/relay.go b/hubble/relay.go index beb4a98534..7f982f6e9d 100644 --- a/hubble/relay.go +++ b/hubble/relay.go @@ -6,300 +6,170 @@ package hubble import ( "context" "fmt" - "time" - "github.com/cloudflare/cfssl/config" - "github.com/cloudflare/cfssl/csr" + "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/utils" + + "github.com/cilium/cilium/pkg/versioncheck" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "github.com/cilium/cilium-cli/defaults" - "github.com/cilium/cilium-cli/internal/utils" - "github.com/cilium/cilium-cli/k8s" ) -const ( - relayPort = int32(defaults.RelayPort) - relayPortName = "grpc" -) +func (k *K8sHubble) generateRelayService() (*corev1.Service, error) { + var ( + svcFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + svcFilename = "templates/hubble-relay/service.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) + } -var ( - secretDefaultMode = int32(0400) - relayReplicas = int32(1) - relayPortIntstr = intstr.FromInt(defaults.RelayPort) - deploymentMaxSurge = intstr.FromInt(1) - deploymentMaxUnavailable = intstr.FromInt(1) -) + svcFile := k.manifests[svcFilename] -var relayClusterRole = &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.RelayClusterRoleName, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"componentstatuses", "endpoints", "namespaces", "nodes", "pods", "services"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, + var svc corev1.Service + utils.MustUnmarshalYAML([]byte(svcFile), &svc) + return &svc, nil } -func (k *K8sHubble) generateRelayService() *corev1.Service { - // NOTE: assuming "disable-server-tls: true", see generateRelayConfigMap(). - s := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.RelayServiceName, - Labels: defaults.RelayDeploymentLabels, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceType(k.params.RelayServiceType), - Ports: []corev1.ServicePort{ - { - Port: int32(defaults.RelayServicePlaintextPort), - TargetPort: relayPortIntstr, - }, - }, - Selector: defaults.RelayDeploymentLabels, - }, +func (k *K8sHubble) generateRelayDeployment() (*appsv1.Deployment, error) { + var ( + deployFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + deployFilename = "templates/hubble-relay/deployment.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } - return s -} -func (k *K8sHubble) generateRelayDeployment() *appsv1.Deployment { - d := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.RelayDeploymentName, - Labels: defaults.RelayDeploymentLabels, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &relayReplicas, - Selector: &metav1.LabelSelector{ - MatchLabels: defaults.RelayDeploymentLabels, - }, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.RollingUpdateDeploymentStrategyType, - RollingUpdate: &appsv1.RollingUpdateDeployment{ - MaxUnavailable: &deploymentMaxUnavailable, - MaxSurge: &deploymentMaxSurge, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.RelayDeploymentName, - Labels: defaults.RelayDeploymentLabels, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyAlways, - ServiceAccountName: defaults.RelayServiceAccountName, - Containers: []corev1.Container{ - { - Name: defaults.RelayContainerName, - Command: []string{"hubble-relay"}, - Args: []string{ - "serve", - }, - Image: k.relayImage(utils.ImagePathIncludeDigest), - ImagePullPolicy: corev1.PullIfNotPresent, - Ports: []corev1.ContainerPort{ - { - Name: relayPortName, - ContainerPort: relayPort, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "hubble-sock-dir", - MountPath: "/var/run/cilium", - ReadOnly: true, - }, - { - Name: "config", - MountPath: "/etc/hubble-relay", - ReadOnly: true, - }, - { - Name: "tls", - MountPath: "/var/lib/hubble-relay/tls", - ReadOnly: true, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: relayPortIntstr, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - TCPSocket: &corev1.TCPSocketAction{ - Port: relayPortIntstr, - }, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: defaults.RelayConfigMapName, - }, - Items: []corev1.KeyToPath{ - {Key: "config.yaml", Path: "config.yaml"}, - }, - }, - }, - }, - { - Name: "hubble-sock-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/run/cilium", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, - { - Name: "tls", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - DefaultMode: &secretDefaultMode, - Sources: []corev1.VolumeProjection{ - { - Secret: &corev1.SecretProjection{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: defaults.RelayClientSecretName, - }, - Items: []corev1.KeyToPath{ - { - Key: corev1.TLSCertKey, - Path: "client.crt", - }, - { - Key: corev1.TLSPrivateKeyKey, - Path: "client.key", - }, - { - Key: defaults.CASecretCertName, - Path: "hubble-server-ca.crt", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - return d + deploymentFile := k.manifests[deployFilename] + + var deploy appsv1.Deployment + utils.MustUnmarshalYAML([]byte(deploymentFile), &deploy) + return &deploy, nil } -func (k *K8sHubble) generateRelayConfigMap() *corev1.ConfigMap { - - var config = ` -peer-service: ` + defaults.HubbleSocketPath + ` -listen-address: ` + fmt.Sprintf("%s:%d", defaults.RelayListenHost, defaults.RelayPort) + ` -dial-timeout: ~ -retry-timeout: ~ -sort-buffer-len-max: ~ -sort-buffer-drain-timeout: ~ -tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt -tls-client-key-file: /var/lib/hubble-relay/tls/client.key -tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt -disable-server-tls: true -` - - //{{- if .Values.hubble.relay.tls.server.enabled }} - //tls-server-cert-file: /var/lib/hubble-relay/tls/server.crt - //tls-server-key-file: /var/lib/hubble-relay/tls/server.key - //{{- else }} - //{{- end }} - //{{- end }} - - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.RelayConfigMapName, - }, - Data: map[string]string{ - "config.yaml": config, - }, +func (k *K8sHubble) generateRelayConfigMap() (*corev1.ConfigMap, error) { + var ( + cmFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + cmFilename = "templates/hubble-relay/configmap.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } -} -func (k *K8sHubble) relayImage(imagePathMode utils.ImagePathMode) string { - return utils.BuildImagePath(k.params.RelayImage, k.params.RelayVersion, defaults.RelayImage, k.ciliumVersion, imagePathMode) + cmFile := k.manifests[cmFilename] + + var cm corev1.ConfigMap + utils.MustUnmarshalYAML([]byte(cmFile), &cm) + return &cm, nil } func (k *K8sHubble) disableRelay(ctx context.Context) error { k.Log("🔥 Deleting Relay...") - k.client.DeleteService(ctx, k.params.Namespace, defaults.RelayServiceName, metav1.DeleteOptions{}) - k.client.DeleteDeployment(ctx, k.params.Namespace, defaults.RelayDeploymentName, metav1.DeleteOptions{}) - k.client.DeleteClusterRoleBinding(ctx, defaults.RelayClusterRoleName, metav1.DeleteOptions{}) - k.client.DeleteClusterRole(ctx, defaults.RelayClusterRoleName, metav1.DeleteOptions{}) - k.client.DeleteServiceAccount(ctx, k.params.Namespace, defaults.RelayServiceAccountName, metav1.DeleteOptions{}) - k.client.DeleteConfigMap(ctx, k.params.Namespace, defaults.RelayConfigMapName, metav1.DeleteOptions{}) - k.deleteRelayCertificates(ctx) + relaySvc, err := k.generateRelayService() + if err != nil { + return err + } + k.client.DeleteService(ctx, relaySvc.GetNamespace(), relaySvc.GetName(), metav1.DeleteOptions{}) - return nil + relayDeployment, err := k.generateRelayDeployment() + if err != nil { + return err + } + k.client.DeleteDeployment(ctx, relayDeployment.GetNamespace(), relayDeployment.GetName(), metav1.DeleteOptions{}) + + crb := k.NewClusterRoleBinding(defaults.RelayClusterRoleName) + k.client.DeleteClusterRoleBinding(ctx, crb.GetName(), metav1.DeleteOptions{}) + + cr := k.NewClusterRole(defaults.RelayClusterRoleName) + k.client.DeleteClusterRole(ctx, cr.GetName(), metav1.DeleteOptions{}) + + sa := k.NewServiceAccount(defaults.RelayServiceAccountName) + k.client.DeleteServiceAccount(ctx, sa.GetNamespace(), sa.GetName(), metav1.DeleteOptions{}) + + relayConfigMap, err := k.generateRelayConfigMap() + if err != nil { + return err + } + k.client.DeleteConfigMap(ctx, relayConfigMap.GetNamespace(), relayConfigMap.GetName(), metav1.DeleteOptions{}) + + return k.deleteRelayCertificates(ctx) } -func (k *K8sHubble) enableRelay(ctx context.Context) error { - _, err := k.client.GetDeployment(ctx, k.params.Namespace, defaults.RelayDeploymentName, metav1.GetOptions{}) +func (k *K8sHubble) enableRelay(ctx context.Context) (string, error) { + relayDeployment, err := k.generateRelayDeployment() + if err != nil { + return "", err + } + + _, err = k.client.GetDeployment(ctx, relayDeployment.GetNamespace(), relayDeployment.GetName(), metav1.GetOptions{}) if err == nil { k.Log("✅ Relay is already deployed") - return nil + return relayDeployment.GetName(), nil } + k.Log("✨ Generating certificates...") + if err := k.createRelayCertificates(ctx); err != nil { - return err + return "", err } - // k.Log("✨ Generating certificates...") - - k.Log("✨ Deploying Relay from %s...", k.relayImage(utils.ImagePathExcludeDigest)) - if _, err := k.client.CreateConfigMap(ctx, k.params.Namespace, k.generateRelayConfigMap(), metav1.CreateOptions{}); err != nil { - return err + relayCm, err := k.generateRelayConfigMap() + if err != nil { + return "", err } - - if _, err := k.client.CreateServiceAccount(ctx, k.params.Namespace, k8s.NewServiceAccount(defaults.RelayServiceAccountName), metav1.CreateOptions{}); err != nil { - return err + k.Log("✨ Deploying Relay...") + if _, err := k.client.CreateConfigMap(ctx, relayCm.GetNamespace(), relayCm, metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateClusterRole(ctx, relayClusterRole, metav1.CreateOptions{}); err != nil { - return err + sa := k.NewServiceAccount(defaults.RelayServiceAccountName) + if _, err := k.client.CreateServiceAccount(ctx, sa.GetNamespace(), sa, metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateClusterRoleBinding(ctx, k8s.NewClusterRoleBinding(defaults.RelayClusterRoleName, k.params.Namespace, defaults.RelayServiceAccountName), metav1.CreateOptions{}); err != nil { - return err + if _, err := k.client.CreateDeployment(ctx, relayDeployment.GetNamespace(), relayDeployment, metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateDeployment(ctx, k.params.Namespace, k.generateRelayDeployment(), metav1.CreateOptions{}); err != nil { - return err + relaySvc, err := k.generateRelayService() + if err != nil { + return "", err } - - //relayService.Spec.Type = corev1.ServiceType(k.params.ServiceType) - if _, err := k.client.CreateService(ctx, k.params.Namespace, k.generateRelayService(), metav1.CreateOptions{}); err != nil { - return err + if _, err := k.client.CreateService(ctx, relaySvc.GetNamespace(), relaySvc, metav1.CreateOptions{}); err != nil { + return "", err } - return nil + return relayDeployment.GetName(), nil } func (k *K8sHubble) deleteRelayCertificates(ctx context.Context) error { k.Log("🔥 Deleting Relay certificates...") - k.client.DeleteSecret(ctx, k.params.Namespace, defaults.RelayServerSecretName, metav1.DeleteOptions{}) - k.client.DeleteSecret(ctx, k.params.Namespace, defaults.RelayClientSecretName, metav1.DeleteOptions{}) + secret, err := k.generateRelayCertificate(defaults.RelayServerSecretName) + if err != nil { + return err + } + + k.client.DeleteSecret(ctx, secret.GetNamespace(), secret.GetName(), metav1.DeleteOptions{}) + + secret, err = k.generateRelayCertificate(defaults.RelayClientSecretName) + if err != nil { + return err + } + k.client.DeleteSecret(ctx, secret.GetNamespace(), secret.GetName(), metav1.DeleteOptions{}) return nil } @@ -313,92 +183,93 @@ func (k *K8sHubble) createRelayCertificates(ctx context.Context) error { } func (k *K8sHubble) createRelayServerCertificate(ctx context.Context) error { - certReq := &csr.CertificateRequest{ - Names: []csr.Name{{C: "US", ST: "San Francisco", L: "CA"}}, - KeyRequest: csr.NewKeyRequest(), - Hosts: []string{"*.hubble-relay.cilium.io"}, - CN: "*.hubble-relay.cilium.io", - } - - signConf := &config.Signing{ - Default: &config.SigningProfile{Expiry: 5 * 365 * 24 * time.Hour}, - Profiles: map[string]*config.SigningProfile{ - defaults.RelayServerSecretName: { - Expiry: 3 * 365 * 24 * time.Hour, - Usage: []string{"signing", "key encipherment", "server auth", "client auth"}, - }, - }, + secret, err := k.generateRelayCertificate(defaults.RelayServerSecretName) + if err != nil { + return err } - cert, key, err := k.certManager.GenerateCertificate(defaults.RelayServerSecretName, certReq, signConf) + _, err = k.client.CreateSecret(ctx, secret.GetNamespace(), &secret, metav1.CreateOptions{}) if err != nil { - return fmt.Errorf("unable to generate certificate %s: %w", defaults.RelayServerSecretName, err) + return fmt.Errorf("unable to create secret %s/%s: %w", secret.GetNamespace(), secret.GetName(), err) } - data := map[string][]byte{ - corev1.TLSCertKey: cert, - corev1.TLSPrivateKeyKey: key, - defaults.CASecretCertName: k.certManager.CACertBytes(), + return nil +} + +func (k *K8sHubble) createRelayClientCertificate(ctx context.Context) error { + secret, err := k.generateRelayCertificate(defaults.RelayClientSecretName) + if err != nil { + return err } - _, err = k.client.CreateSecret(ctx, k.params.Namespace, k8s.NewTLSSecret(defaults.RelayServerSecretName, k.params.Namespace, data), metav1.CreateOptions{}) + _, err = k.client.CreateSecret(ctx, secret.GetNamespace(), &secret, metav1.CreateOptions{}) if err != nil { - return fmt.Errorf("unable to create secret %s/%s: %w", k.params.Namespace, defaults.RelayServerSecretName, err) + return fmt.Errorf("unable to create secret %s/%s: %w", secret.GetNamespace(), secret.GetName(), err) } return nil } -func (k *K8sHubble) createRelayClientCertificate(ctx context.Context) error { - certReq := &csr.CertificateRequest{ - Names: []csr.Name{{C: "US", ST: "San Francisco", L: "CA"}}, - KeyRequest: csr.NewKeyRequest(), - Hosts: []string{"*.hubble-relay.cilium.io"}, - CN: "*.hubble-relay.cilium.io", +func (k *K8sHubble) generateRelayCertificate(name string) (corev1.Secret, error) { + var ( + relaySecretFilename string + ) + + ciliumVer := k.semVerCiliumVersion + + switch { + case versioncheck.MustCompile(">1.10.99")(ciliumVer): + switch name { + case defaults.RelayServerSecretName: + relaySecretFilename = "templates/hubble/tls-helm/relay-server-secret.yaml" + case defaults.RelayClientSecretName: + relaySecretFilename = "templates/hubble/tls-helm/relay-client-secret.yaml" + } } - signConf := &config.Signing{ - Default: &config.SigningProfile{Expiry: 5 * 365 * 24 * time.Hour}, - Profiles: map[string]*config.SigningProfile{ - defaults.RelayClientSecretName: { - Expiry: 3 * 365 * 24 * time.Hour, - Usage: []string{"signing", "key encipherment", "server auth", "client auth"}, - }, - }, - } + relayFile := k.manifests[relaySecretFilename] - cert, key, err := k.certManager.GenerateCertificate(defaults.RelayClientSecretName, certReq, signConf) + var secret corev1.Secret + utils.MustUnmarshalYAML([]byte(relayFile), &secret) + return secret, nil +} + +func (k *K8sHubble) PortForwardCommand(ctx context.Context) error { + cm, err := k.client.GetConfigMap(ctx, k.params.Namespace, defaults.ConfigMapName, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("unable to generate certificate %s: %w", defaults.RelayClientSecretName, err) + return fmt.Errorf("unable to retrieve ConfigMap %q: %w", defaults.ConfigMapName, err) } - data := map[string][]byte{ - corev1.TLSCertKey: cert, - corev1.TLSPrivateKeyKey: key, - defaults.CASecretCertName: k.certManager.CACertBytes(), + value, ok := cm.Data[defaults.ExtraConfigMapUserOptsKey] + if !ok { + return fmt.Errorf("configmap option not found") } - _, err = k.client.CreateSecret(ctx, k.params.Namespace, k8s.NewTLSSecret(defaults.RelayClientSecretName, k.params.Namespace, data), metav1.CreateOptions{}) + // Generate the manifests has if hubble was being enabled so that we can + // retrieve all UI and Relay's resource names. + k.params.UI = true + k.params.Relay = true + err = k.generateManifestsEnable(ctx, value) if err != nil { - return fmt.Errorf("unable to create secret %s/%s: %w", k.params.Namespace, defaults.RelayClientSecretName, err) + return err } - return nil -} - -func (p *Parameters) PortForwardCommand(ctx context.Context) error { + relaySvc, err := k.generateRelayService() + if err != nil { + return err + } args := []string{ "port-forward", - "-n", p.Namespace, + "-n", k.params.Namespace, "svc/hubble-relay", "--address", "0.0.0.0", "--address", "::", - fmt.Sprintf("%d:%d", p.PortForward, defaults.RelayServicePlaintextPort)} + fmt.Sprintf("%d:%d", k.params.PortForward, relaySvc.Spec.Ports[0].Port)} - if p.Context != "" { - args = append([]string{"--context", p.Context}, args...) + if k.params.Context != "" { + args = append([]string{"--context", k.params.Context}, args...) } - _, err := utils.Exec(p, "kubectl", args...) + _, err = utils.Exec(k, "kubectl", args...) return err } diff --git a/hubble/relay_test.go b/hubble/relay_test.go index 2844ef8061..447b204415 100644 --- a/hubble/relay_test.go +++ b/hubble/relay_test.go @@ -4,6 +4,7 @@ import ( "strconv" "testing" + "github.com/cilium/cilium-cli/defaults" "github.com/cilium/cilium-cli/internal/utils" ) @@ -45,3 +46,7 @@ func TestK8sHubbleRelayImage(t *testing.T) { }) } } + +func (k *K8sHubble) relayImage(imagePathMode utils.ImagePathMode) string { + return utils.BuildImagePath(k.params.RelayImage, k.params.RelayVersion, defaults.RelayImage, k.ciliumVersion, imagePathMode) +} diff --git a/hubble/ui.go b/hubble/ui.go index 1d03a37c6a..82ac3018fe 100644 --- a/hubble/ui.go +++ b/hubble/ui.go @@ -9,307 +9,157 @@ import ( "io" "time" + "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/utils" + + "github.com/cilium/cilium/pkg/versioncheck" "github.com/pkg/browser" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - - "github.com/cilium/cilium-cli/defaults" - "github.com/cilium/cilium-cli/internal/utils" - "github.com/cilium/cilium-cli/k8s" ) -var ( - hubbleUIReplicas = int32(1) - hubbleUIPortIntstr = intstr.FromInt(8081) - hubbleUIUser = int64(1001) -) +func (k *K8sHubble) generateHubbleUIService() (*corev1.Service, error) { + var ( + svcFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + svcFilename = "templates/hubble-ui/service.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) + } -var hubbleUIClusterRole = &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.HubbleUIClusterRoleName, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{"networking.k8s.io"}, - Resources: []string{"networkpolicies"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"componentstatuses", "endpoints", "namespaces", "nodes", "pods", "services"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"apiextensions.k8s.io"}, - Resources: []string{"customresourcedefinitions"}, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"cilium.io"}, - Resources: []string{"*"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, -} + svcFile := k.manifests[svcFilename] -func (k *K8sHubble) generateHubbleUIService() *corev1.Service { - s := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.HubbleUIServiceName, - Labels: defaults.HubbleUIDeploymentLabels, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Ports: []corev1.ServicePort{ - { - Port: int32(80), - TargetPort: hubbleUIPortIntstr, - }, - }, - Selector: defaults.HubbleUIDeploymentLabels, - }, - } - return s + var svc corev1.Service + utils.MustUnmarshalYAML([]byte(svcFile), &svc) + return &svc, nil } -func (k *K8sHubble) generateHubbleUIConfigMap() *corev1.ConfigMap { - return &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.HubbleUIConfigMapName, - }, - Data: map[string]string{ - "envoy.yaml": `static_resources: - listeners: - - name: listener_hubble_ui - address: - socket_address: - address: 0.0.0.0 - port_value: 8081 - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - codec_type: auto - stat_prefix: ingress_http - route_config: - name: local_route - virtual_hosts: - - name: local_service - domains: ["*"] - routes: - - match: - prefix: "/api/" - route: - cluster: backend - prefix_rewrite: "/" - timeout: 0s - max_stream_duration: - grpc_timeout_header_max: 0s - - match: - prefix: "/" - route: - cluster: frontend - cors: - allow_origin_string_match: - - prefix: "*" - allow_methods: GET, PUT, DELETE, POST, OPTIONS - allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout - max_age: "1728000" - expose_headers: grpc-status,grpc-message - http_filters: - - name: envoy.filters.http.grpc_web - - name: envoy.filters.http.cors - - name: envoy.filters.http.router - clusters: - - name: frontend - connect_timeout: 0.25s - type: strict_dns - lb_policy: round_robin - load_assignment: - cluster_name: frontend - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 8080 - - name: backend - connect_timeout: 0.25s - type: logical_dns - lb_policy: round_robin - http2_protocol_options: {} - load_assignment: - cluster_name: backend - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: 127.0.0.1 - port_value: 8090 -`, - }, +func (k *K8sHubble) generateHubbleUIConfigMap() (*corev1.ConfigMap, error) { + var ( + cmFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + cmFilename = "templates/hubble-ui/configmap.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } + + cmFile := k.manifests[cmFilename] + + var cm corev1.ConfigMap + utils.MustUnmarshalYAML([]byte(cmFile), &cm) + return &cm, nil } -func (k *K8sHubble) generateHubbleUIDeployment() *appsv1.Deployment { - d := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.HubbleUIDeploymentName, - Labels: defaults.HubbleUIDeploymentLabels, - }, - Spec: appsv1.DeploymentSpec{ - Replicas: &hubbleUIReplicas, - Selector: &metav1.LabelSelector{ - MatchLabels: defaults.HubbleUIDeploymentLabels, - }, - Strategy: appsv1.DeploymentStrategy{ - Type: appsv1.RollingUpdateDeploymentStrategyType, - RollingUpdate: &appsv1.RollingUpdateDeployment{ - MaxUnavailable: &deploymentMaxUnavailable, - MaxSurge: &deploymentMaxSurge, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: defaults.HubbleUIDeploymentName, - Labels: defaults.HubbleUIDeploymentLabels, - }, - Spec: corev1.PodSpec{ - RestartPolicy: corev1.RestartPolicyAlways, - ServiceAccountName: defaults.HubbleUIServiceAccountName, - SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: &hubbleUIUser, - }, - Containers: []corev1.Container{ - { - Name: "frontend", - Image: k.uiImage(utils.ImagePathIncludeDigest), - ImagePullPolicy: corev1.PullIfNotPresent, - Ports: []corev1.ContainerPort{ - { - Name: "http", - ContainerPort: 8080, - }, - }, - }, - { - Name: "backend", - Image: k.uiBackendImage(utils.ImagePathIncludeDigest), - ImagePullPolicy: corev1.PullIfNotPresent, - Env: []corev1.EnvVar{ - {Name: "EVENTS_SERVER_PORT", Value: "8090"}, - {Name: "FLOWS_API_ADDR", Value: "hubble-relay:80"}, - }, - Ports: []corev1.ContainerPort{ - { - Name: "grpc", - ContainerPort: 8090, - }, - }, - }, - { - Name: "proxy", - Image: "docker.io/envoyproxy/envoy:v1.18.2@sha256:e8b37c1d75787dd1e712ff389b0d37337dc8a174a63bed9c34ba73359dc67da7", - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"envoy"}, - Args: []string{"-c", "/etc/envoy.yaml", "-l", "info"}, - Env: []corev1.EnvVar{ - {Name: "EVENTS_SERVER_PORT", Value: "8090"}, - {Name: "FLOWS_API_ADDR", Value: "hubble-relay:80"}, - }, - Ports: []corev1.ContainerPort{ - { - Name: "http", - ContainerPort: 8081, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "hubble-ui-envoy-yaml", - MountPath: "/etc/envoy.yaml", - SubPath: "envoy.yaml", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "hubble-ui-envoy-yaml", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: defaults.HubbleUIConfigMapName, - }, - }, - }, - }, - }, - }, - }, - }, +func (k *K8sHubble) generateHubbleUIDeployment() (*appsv1.Deployment, error) { + var ( + deployFilename string + ) + + ciliumVer := k.semVerCiliumVersion + switch { + case versioncheck.MustCompile(">=1.9.0")(ciliumVer): + deployFilename = "templates/hubble-ui/deployment.yaml" + default: + return nil, fmt.Errorf("cilium version unsupported %s", ciliumVer.String()) } - return d -} -func (k *K8sHubble) uiImage(imagePathMode utils.ImagePathMode) string { - return utils.BuildImagePath(k.params.UIImage, k.params.UIVersion, defaults.HubbleUIImage, defaults.HubbleUIVersion, imagePathMode) -} + deploymentFile := k.manifests[deployFilename] -func (k *K8sHubble) uiBackendImage(imagePathMode utils.ImagePathMode) string { - return utils.BuildImagePath(k.params.UIBackendImage, k.params.UIVersion, defaults.HubbleUIBackendImage, defaults.HubbleUIVersion, imagePathMode) + var deploy appsv1.Deployment + utils.MustUnmarshalYAML([]byte(deploymentFile), &deploy) + return &deploy, nil } func (k *K8sHubble) disableUI(ctx context.Context) error { k.Log("🔥 Deleting Hubble UI...") - k.client.DeleteService(ctx, k.params.Namespace, defaults.HubbleUIServiceName, metav1.DeleteOptions{}) - k.client.DeleteDeployment(ctx, k.params.Namespace, defaults.HubbleUIDeploymentName, metav1.DeleteOptions{}) - k.client.DeleteClusterRoleBinding(ctx, defaults.HubbleUIClusterRoleName, metav1.DeleteOptions{}) - k.client.DeleteClusterRole(ctx, defaults.HubbleUIClusterRoleName, metav1.DeleteOptions{}) - k.client.DeleteServiceAccount(ctx, k.params.Namespace, defaults.HubbleUIServiceAccountName, metav1.DeleteOptions{}) - k.client.DeleteConfigMap(ctx, k.params.Namespace, defaults.HubbleUIConfigMapName, metav1.DeleteOptions{}) + + hubbleUISvc, err := k.generateHubbleUIService() + if err != nil { + return err + } + k.client.DeleteService(ctx, hubbleUISvc.GetNamespace(), hubbleUISvc.GetName(), metav1.DeleteOptions{}) + + hubbleUIDeploy, err := k.generateHubbleUIDeployment() + if err != nil { + return err + } + k.client.DeleteDeployment(ctx, hubbleUIDeploy.GetNamespace(), hubbleUIDeploy.GetName(), metav1.DeleteOptions{}) + + crb := k.NewClusterRoleBinding(defaults.HubbleUIClusterRoleName) + k.client.DeleteClusterRoleBinding(ctx, crb.GetName(), metav1.DeleteOptions{}) + + cr := k.NewClusterRole(defaults.HubbleUIClusterRoleName) + k.client.DeleteClusterRole(ctx, cr.GetName(), metav1.DeleteOptions{}) + + sa := k.NewServiceAccount(defaults.HubbleUIServiceAccountName) + k.client.DeleteServiceAccount(ctx, sa.GetNamespace(), sa.GetName(), metav1.DeleteOptions{}) + + hubbleUICM, err := k.generateHubbleUIConfigMap() + if err != nil { + return err + } + k.client.DeleteConfigMap(ctx, hubbleUICM.GetNamespace(), hubbleUICM.GetName(), metav1.DeleteOptions{}) return nil } -func (k *K8sHubble) enableUI(ctx context.Context) error { - _, err := k.client.GetDeployment(ctx, k.params.Namespace, defaults.HubbleUIDeploymentName, metav1.GetOptions{}) +func (k *K8sHubble) enableUI(ctx context.Context) (string, error) { + hubbleUIDeploy, err := k.generateHubbleUIDeployment() + if err != nil { + return "", err + } + + _, err = k.client.GetDeployment(ctx, hubbleUIDeploy.GetNamespace(), hubbleUIDeploy.GetName(), metav1.GetOptions{}) if err == nil { k.Log("✅ Hubble UI is already deployed") - return nil + return hubbleUIDeploy.GetName(), nil } - k.Log("✨ Deploying Hubble UI from %s and Hubble UI Backend from %s...", k.uiImage(utils.ImagePathExcludeDigest), k.uiBackendImage(utils.ImagePathExcludeDigest)) - if _, err := k.client.CreateConfigMap(ctx, k.params.Namespace, k.generateHubbleUIConfigMap(), metav1.CreateOptions{}); err != nil { - return err + hubbleUICM, err := k.generateHubbleUIConfigMap() + if err != nil { + return "", err } - if _, err := k.client.CreateServiceAccount(ctx, k.params.Namespace, k8s.NewServiceAccount(defaults.HubbleUIServiceAccountName), metav1.CreateOptions{}); err != nil { - return err + k.Log("✨ Deploying Hubble UI from %s and Hubble UI Backend...") + if _, err := k.client.CreateConfigMap(ctx, hubbleUICM.GetNamespace(), hubbleUICM, metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateClusterRole(ctx, hubbleUIClusterRole, metav1.CreateOptions{}); err != nil { - return err + sa := k.NewServiceAccount(defaults.HubbleUIServiceAccountName) + if _, err := k.client.CreateServiceAccount(ctx, sa.GetNamespace(), sa, metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateClusterRoleBinding(ctx, k8s.NewClusterRoleBinding(defaults.HubbleUIClusterRoleName, k.params.Namespace, defaults.HubbleUIServiceAccountName), metav1.CreateOptions{}); err != nil { - return err + if _, err := k.client.CreateClusterRole(ctx, k.NewClusterRole(defaults.HubbleUIClusterRoleName), metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateDeployment(ctx, k.params.Namespace, k.generateHubbleUIDeployment(), metav1.CreateOptions{}); err != nil { - return err + if _, err := k.client.CreateClusterRoleBinding(ctx, k.NewClusterRoleBinding(defaults.HubbleUIClusterRoleName), metav1.CreateOptions{}); err != nil { + return "", err } - if _, err := k.client.CreateService(ctx, k.params.Namespace, k.generateHubbleUIService(), metav1.CreateOptions{}); err != nil { - return err + if _, err := k.client.CreateDeployment(ctx, hubbleUIDeploy.GetNamespace(), hubbleUIDeploy, metav1.CreateOptions{}); err != nil { + return "", err } - return nil + hubbleUISvc, err := k.generateHubbleUIService() + if err != nil { + return "", err + } + if _, err := k.client.CreateService(ctx, hubbleUISvc.GetNamespace(), hubbleUISvc, metav1.CreateOptions{}); err != nil { + return "", err + } + + return hubbleUIDeploy.GetName(), nil } func (p *Parameters) UIPortForwardCommand(ctx context.Context) error { diff --git a/internal/cli/cmd/hubble.go b/internal/cli/cmd/hubble.go index 27b05aba01..1d2c004fb0 100644 --- a/internal/cli/cmd/hubble.go +++ b/internal/cli/cmd/hubble.go @@ -7,10 +7,10 @@ import ( "context" "os" - "github.com/spf13/cobra" - "github.com/cilium/cilium-cli/defaults" "github.com/cilium/cilium-cli/hubble" + + "github.com/spf13/cobra" ) func newCmdHubble() *cobra.Command { @@ -50,18 +50,46 @@ func newCmdHubbleEnable() *cobra.Command { cmd.Flags().StringVarP(¶ms.Namespace, "namespace", "n", "kube-system", "Namespace Cilium is running in") cmd.Flags().BoolVar(¶ms.Relay, "relay", true, "Deploy Hubble Relay") + // It can be deprecated since we have a helm option for it cmd.Flags().StringVar(¶ms.RelayImage, "relay-image", "", "Image path to use for Relay") + // It can be deprecated since we have a helm option for it cmd.Flags().StringVar(¶ms.RelayVersion, "relay-version", "", "Version of Relay to deploy") + // It can be deprecated since there is not a helm option for it and cmd.Flags().StringVar(¶ms.RelayServiceType, "relay-service-type", "ClusterIP", "Type of Kubernetes service to expose Hubble Relay") + cmd.Flags().MarkDeprecated("relay-service-type", "value is no longer used for relay-service") cmd.Flags().BoolVar(¶ms.UI, "ui", false, "Enable Hubble UI") + + // It can be deprecated since we have a helm option for it cmd.Flags().StringVar(¶ms.UIImage, "ui-image", "", "Image path to use for UI") + // It can be deprecated since we have a helm option for it cmd.Flags().StringVar(¶ms.UIBackendImage, "ui-backend-image", "", "Image path to use for UI backend") + // It can be deprecated since we have a helm option for it cmd.Flags().StringVar(¶ms.UIVersion, "ui-version", "", "Version of UI to deploy") cmd.Flags().BoolVar(¶ms.CreateCA, "create-ca", true, "Automatically create CA if needed") cmd.Flags().StringVar(&contextName, "context", "", "Kubernetes configuration context") cmd.Flags().BoolVar(¶ms.Wait, "wait", true, "Wait for status to report success (no errors)") cmd.Flags().DurationVar(¶ms.WaitDuration, "wait-duration", defaults.StatusWaitDuration, "Maximum time to wait for status") + cmd.Flags().StringVar(¶ms.K8sVersion, "k8s-version", "", "Kubernetes server version in case auto-detection fails") + cmd.Flags().StringVar(¶ms.BaseVersion, "base-version", defaults.Version, + "Specify the base Cilium version for configuration purpose in case the --version flag doesn't indicate the actual Cilium version") + cmd.Flags().StringVar(¶ms.HelmChartDirectory, "chart-directory", "", "Helm chart directory") + cmd.Flags().StringSliceVar(¶ms.HelmOpts.ValueFiles, "helm-values", []string{}, "Specify helm values in a YAML file or a URL (can specify multiple)") + cmd.Flags().StringArrayVar(¶ms.HelmOpts.Values, "helm-set", []string{}, "Set helm values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)") + cmd.Flags().StringArrayVar(¶ms.HelmOpts.StringValues, "helm-set-string", []string{}, "Set helm STRING values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)") + cmd.Flags().StringArrayVar(¶ms.HelmOpts.FileValues, "helm-set-file", []string{}, "Set helm values from respective files specified via the command line (can specify multiple or separate values with commas: key1=path1,key2=path2)") + cmd.Flags().StringVar(¶ms.HelmGenValuesFile, "helm-auto-gen-values", "", "Write an auto-generated helm values into this file") + + for flagName := range hubble.FlagsToHelmOpts { + // TODO(aanm) Do not mark the flags has deprecated for now. + // msg := fmt.Sprintf("use --helm-set=%s<=value> instead", helmOpt) + // err := cmd.Flags().MarkDeprecated(flagName, msg) + // if err != nil { + // panic(err) + // } + hubble.FlagValues[flagName] = cmd.Flags().Lookup(flagName).Value + } + return cmd } @@ -99,7 +127,8 @@ func newCmdPortForwardCommand() *cobra.Command { Short: "Forward the relay port to the local machine", Long: ``, RunE: func(cmd *cobra.Command, args []string) error { - if err := params.PortForwardCommand(context.Background()); err != nil { + h := hubble.NewK8sHubble(k8sClient, params) + if err := h.PortForwardCommand(context.Background()); err != nil { fatalf("Unable to port forward: %s", err) } return nil diff --git a/k8s/client.go b/k8s/client.go index c21d8db802..ccca3314e9 100644 --- a/k8s/client.go +++ b/k8s/client.go @@ -163,6 +163,10 @@ func (c *Client) PatchConfigMap(ctx context.Context, namespace, name string, pt return c.Clientset.CoreV1().ConfigMaps(namespace).Patch(ctx, name, pt, data, opts) } +func (c *Client) UpdateConfigMap(ctx context.Context, configMap *corev1.ConfigMap, opts metav1.UpdateOptions) (*corev1.ConfigMap, error) { + return c.Clientset.CoreV1().ConfigMaps(configMap.Namespace).Update(ctx, configMap, opts) +} + func (c *Client) CreateService(ctx context.Context, namespace string, service *corev1.Service, opts metav1.CreateOptions) (*corev1.Service, error) { return c.Clientset.CoreV1().Services(namespace).Create(ctx, service, opts) }