From 33ae0fa073adf6e3f12ee817b274ca742bd5a03b Mon Sep 17 00:00:00 2001 From: Daniel Grimm Date: Wed, 22 Jan 2025 11:15:57 +0100 Subject: [PATCH 1/3] Create IstioRevisionTag documentation (#511) Signed-off-by: Daniel Grimm --- api/v1alpha1/istiorevisiontags_types.go | 6 +- .../sailoperator.clusterserviceversion.yaml | 6 +- .../sailoperator.io_istiorevisiontags.yaml | 10 +- .../sailoperator.io_istiorevisiontags.yaml | 10 +- docs/README.md | 212 +++++++++++++++++- docs/api-reference/sailoperator.io.md | 6 +- enhancements/SEP2-revision-tags.md | 4 +- 7 files changed, 228 insertions(+), 26 deletions(-) diff --git a/api/v1alpha1/istiorevisiontags_types.go b/api/v1alpha1/istiorevisiontags_types.go index 8d2788564..cb1a13ee7 100644 --- a/api/v1alpha1/istiorevisiontags_types.go +++ b/api/v1alpha1/istiorevisiontags_types.go @@ -31,7 +31,7 @@ type IstioRevisionTagSpec struct { TargetRef IstioRevisionTagTargetReference `json:"targetRef"` } -// IstioRevisionTagTargetReference can reference either Istio or IstioRevision objects in the cluster. +// IstioRevisionTagTargetReference can reference either Istio or IstioRevision objects in the cluster. In the case of referencing an Istio object, the Sail Operator will automatically update the reference to the Istio object's Active Revision. type IstioRevisionTagTargetReference struct { // Kind is the kind of the target resource. // @@ -181,7 +181,7 @@ const ( // +kubebuilder:printcolumn:name="Revision",type="string",JSONPath=".status.istioRevision",description="The IstioRevision this object is referencing." // +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp",description="The age of the object" -// IstioRevisionTag references a Istio or IstioRevision object and serves as an alias for sidecar injection. +// IstioRevisionTag references an Istio or IstioRevision object and serves as an alias for sidecar injection. It can be used to manage stable revision tags without having to use istioctl or helm directly. See https://istio.io/latest/docs/setup/upgrade/canary/#stable-revision-labels for more information on the concept. type IstioRevisionTag struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -192,7 +192,7 @@ type IstioRevisionTag struct { // +kubebuilder:object:root=true -// IstioRevisionList contains a list of IstioRevision +// IstioRevisionTagList contains a list of IstioRevisionTags type IstioRevisionTagList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/bundle/manifests/sailoperator.clusterserviceversion.yaml b/bundle/manifests/sailoperator.clusterserviceversion.yaml index 661ed77a7..f30fd586b 100644 --- a/bundle/manifests/sailoperator.clusterserviceversion.yaml +++ b/bundle/manifests/sailoperator.clusterserviceversion.yaml @@ -231,8 +231,10 @@ spec: displayName: Helm Values path: values version: v1alpha1 - - description: IstioRevisionTag references a Istio or IstioRevision object and - serves as an alias for sidecar injection. + - description: IstioRevisionTag references an Istio or IstioRevision object and + serves as an alias for sidecar injection. It can be used to manage stable + revision tags without having to use istioctl or helm directly. See https://istio.io/latest/docs/setup/upgrade/canary/#stable-revision-labels + for more information on the concept. displayName: Istio Revision Tag kind: IstioRevisionTag name: istiorevisiontags.sailoperator.io diff --git a/bundle/manifests/sailoperator.io_istiorevisiontags.yaml b/bundle/manifests/sailoperator.io_istiorevisiontags.yaml index 001026086..1ad29e8bd 100644 --- a/bundle/manifests/sailoperator.io_istiorevisiontags.yaml +++ b/bundle/manifests/sailoperator.io_istiorevisiontags.yaml @@ -38,8 +38,10 @@ spec: name: v1alpha1 schema: openAPIV3Schema: - description: IstioRevisionTag references a Istio or IstioRevision object and - serves as an alias for sidecar injection. + description: IstioRevisionTag references an Istio or IstioRevision object + and serves as an alias for sidecar injection. It can be used to manage stable + revision tags without having to use istioctl or helm directly. See https://istio.io/latest/docs/setup/upgrade/canary/#stable-revision-labels + for more information on the concept. properties: apiVersion: description: |- @@ -63,7 +65,9 @@ spec: properties: targetRef: description: IstioRevisionTagTargetReference can reference either - Istio or IstioRevision objects in the cluster. + Istio or IstioRevision objects in the cluster. In the case of referencing + an Istio object, the Sail Operator will automatically update the + reference to the Istio object's Active Revision. properties: kind: description: Kind is the kind of the target resource. diff --git a/chart/crds/sailoperator.io_istiorevisiontags.yaml b/chart/crds/sailoperator.io_istiorevisiontags.yaml index 716291003..6f4ed6565 100644 --- a/chart/crds/sailoperator.io_istiorevisiontags.yaml +++ b/chart/crds/sailoperator.io_istiorevisiontags.yaml @@ -38,8 +38,10 @@ spec: name: v1alpha1 schema: openAPIV3Schema: - description: IstioRevisionTag references a Istio or IstioRevision object and - serves as an alias for sidecar injection. + description: IstioRevisionTag references an Istio or IstioRevision object + and serves as an alias for sidecar injection. It can be used to manage stable + revision tags without having to use istioctl or helm directly. See https://istio.io/latest/docs/setup/upgrade/canary/#stable-revision-labels + for more information on the concept. properties: apiVersion: description: |- @@ -63,7 +65,9 @@ spec: properties: targetRef: description: IstioRevisionTagTargetReference can reference either - Istio or IstioRevision objects in the cluster. + Istio or IstioRevision objects in the cluster. In the case of referencing + an Istio object, the Sail Operator will automatically update the + reference to the Istio object's Active Revision. properties: kind: description: Kind is the kind of the target resource. diff --git a/docs/README.md b/docs/README.md index c507d7f9d..6fd1ab593 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,7 +6,10 @@ - [Concepts](#concepts) - [Istio resource](#istio-resource) - [IstioRevision resource](#istiorevision-resource) + - [IstioRevisionTag resource](#istiorevisiontag-resource) - [IstioCNI resource](#istiocni-resource) + - [Resource Status](#resource-status) + - [InUse Detection](#inuse-detection) - [API Reference documentation](#api-reference-documentation) - [Getting Started](#getting-started) - [Installation on OpenShift](#installation-on-openshift) @@ -96,6 +99,24 @@ The `IstioRevision` is the lowest-level API the Sail Operator provides, and it i You can think of the relationship between the `Istio` and `IstioRevision` resource as similar to the one between Kubernetes' `ReplicaSet` and `Pod`: a `ReplicaSet` can be created by users and results in the automatic creation of `Pods`, which will trigger the instantiation of your containers. Similarly, users create an `Istio` resource which instructs the operator to create a matching `IstioRevision`, which then in turn triggers the creation of the Istio control plane. To do that, the Sail Operator will copy all of your relevant configuration from the `Istio` resource to the `IstioRevision` resource. +### IstioRevisionTag resource +The `IstioRevisionTag` resource represents a *Stable Revision Tag*, which functions as an alias for Istio control plane revisions. With a stable tag `prod`, you can e.g. use the label `istio.io/rev=prod` to inject proxies into your workloads. When you perform an upgrade to a control plane with a new revision name, you can simply update your tag to point to the new revision, instead of having to re-label your workloads and namespaces. Also see the [Stable Revision Tags](https://istio.io/latest/docs/setup/upgrade/canary/#stable-revision-labels) section of Istio's [Canary Upgrades documentation](https://istio.io/latest/docs/setup/upgrade/canary/) for more details. + +In Istio, stable revision tags are usually created using `istioctl`, but if you're using the Sail Operator, you can use the `IstioRevisionTag` resource, which comes with an additional feature: instead of just being able to reference an `IstioRevision`, you can also reference an `Istio` resource. When you now update your control plane and the underlying `IstioRevision` changes, the Sail Operator will update your revision tag for you. You only need to restart your deployments to re-inject the new proxies. + +```yaml +apiVersion: sailoperator.io/v1alpha1 +kind: IstioRevisionTag +metadata: + name: default +spec: + targetRef: + kind: Istio # can be either Istio or IstioRevision + name: prod # the name of the Istio/IstioRevision resource +``` + +As you can see in the YAML above, `IstioRevisionTag` really only has one field in its spec: `targetRef`. With this field, you can reference an `Istio` or `IstioRevision` resource. So after deploying this, you will be able to use both the `istio.io/rev=default` and also `istio-injection=enabled` labels to inject proxies into your workloads. The `istio-injection` label can only be used for revisions and revision tags named `default`, like the `IstioRevisionTag` in the above example. + ### IstioCNI resource The lifecycle of Istio's CNI plugin is managed separately when using Sail Operator. To install it, you can create an `IstioCNI` resource. The `IstioCNI` resource is a cluster-wide resource as it will install a `DaemonSet` that will be operating on all nodes of your cluster. You can select a version by setting the `spec.version` field, as you can see in the sample below. To update the CNI plugin, just change the `version` field to the version you want to install. Just like the `Istio` resource, it also has a `values` field that exposes all of the options provided in the `istio-cni` chart: @@ -115,7 +136,22 @@ spec: ``` > [!NOTE] -> The CNI plugin at version `1.x` is compatible with `Istio` at version `1.x-1`, `1.x` and `1.x+1`. +> The CNI plugin at version `1.x` is compatible with `Istio` at version `1.x-1`, `1.x` and `1.x+1`. + +### Resource Status +All of the Sail Operator API resources have a `status` subresource that contains information about their current state in the Kubernetes cluster. + +#### Conditions +All resources have a `Ready` condition which is set to `true` as soon as all child resource have been created and are deemed Ready by their respective controllers. To see additional conditions for each of the resources, check the [API reference documentation](https://github.com/istio-ecosystem/sail-operator/tree/main/docs/api-reference/sailoperator.io.md). + +#### InUse Detection +The Sail Operator uses InUse detection to determine whether an object is referenced. This is currently present on all resources apart from `IstioCNI`. On the `Istio` resource, it is a counter as it only aggregates the `InUse` conditions on its child `IstioRevisions`. + +|API |Type |Name|Description +|------------------|------------|----|------------------------------------------- +|Istio |Counter |Status.Revisions.InUse|Aggregates across all child `IstioRevisions`. +|IstioRevision |Condition |Status.Conditions[type="InUse']|Set to `true` if the `IstioRevision` is referenced by a namespace, workload or `IstioRevisionTag`. +|IstioRevisionTag |Condition |Status.Conditions[type="InUse']|Set to `true` if the `IstioRevisionTag` is referenced by a namespace or workload. ## API Reference documentation The Sail Operator API reference documentation can be found [here](https://github.com/istio-ecosystem/sail-operator/tree/main/docs/api-reference/sailoperator.io.md). @@ -321,7 +357,7 @@ Steps: NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE default 1 1 0 default Healthy v1.22.5 23s ``` - Note: `IN USE` field shows as 0, as `Istio` is yet installed. + Note: `IN USE` field shows as 0, as `Istio` has just been installed and there are no workloads using it. 4. Create namespace `bookinfo` and deploy bookinfo application. @@ -339,7 +375,7 @@ Steps: NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE default 1 1 1 default Healthy v1.22.5 115s ``` - Note: `IN USE` field shows as 1, after application being deployed. + Note: `IN USE` field shows as 1, as the namespace label and the injected proxies reference the IstioRevision. 6. Perform the update of the control plane by changing the version in the Istio resource. @@ -409,7 +445,7 @@ Steps: NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE default 1 1 0 default-v1-22-5 Healthy v1.22.5 52s ``` - Note: `IN USE` field shows as 0, as `Istio` is yet installed. + Note: `IN USE` field shows as 0, as the control plane has just been installed and there are no workloads using it. 4. Get the `IstioRevision` name. @@ -458,11 +494,11 @@ Steps: 10. Verify the `Istio` and `IstioRevision` resources. There will be a new revision created with the new version. ```console - $ kubectl get istio -n istio-system + $ kubectl get istio NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE default 2 2 1 default-v1-23-2 Healthy v1.23.2 9m23s - $ kubectl get istiorevision -n istio-system + $ kubectl get istiorevision NAME TYPE READY STATUS IN USE VERSION AGE default-v1-22-5 Local True Healthy True v1.22.5 10m default-v1-23-2 Local True Healthy False v1.23.2 66s @@ -491,7 +527,7 @@ Steps: ``` The existing workload sidecars will continue to run and will remain connected to the old control plane instance. They will not be replaced with a new version until the pods are deleted and recreated. -14. Delete all the pods in the `bookinfo` namespace. +14. Restart all Deplyments in the `bookinfo` namespace. ```bash kubectl rollout restart deployment -n bookinfo @@ -504,23 +540,179 @@ Steps: ``` The column `VERSION` should match the updated control plane version. -16. Confirm the old control plane and revision deletion. +16. Confirm the deletion of the old control plane and IstioRevision. ```console $ kubectl get pods -n istio-system NAME READY STATUS RESTARTS AGE istiod-default-v1-23-2-7495cdc7bf-v8t4g 1/1 Running 0 4m40s - $ kubectl get istio -n istio-system + $ kubectl get istio NAME REVISIONS READY IN USE ACTIVE REVISION STATUS VERSION AGE default 1 1 1 default-v1-23-2 Healthy v1.23.2 5m - $ kubectl get istiorevision -n istio-system + $ kubectl get istiorevision NAME TYPE READY STATUS IN USE VERSION AGE default-v1-23-2 Local True Healthy True v1.23.2 5m31s ``` The old `IstioRevision` resource and the old control plane will be deleted when the grace period specified in the `Istio` resource field `spec.updateStrategy.inactiveRevisionDeletionGracePeriodSeconds` expires. +#### Example using the RevisionBased strategy and an IstioRevisionTag + +Prerequisites: +* Sail Operator is installed. +* `istioctl` is [installed](common/install-istioctl-tool.md). + +Steps: + +1. Create the `istio-system` namespace. + + ```bash + kubectl create namespace istio-system + ``` + +2. Create the `Istio` and `IstioRevisionTag` resources. + + ```bash + cat < Date: Wed, 22 Jan 2025 12:03:56 +0100 Subject: [PATCH 2/3] Remove automatic channel prefix detection (#574) It causes problems as we usually don't work on branches with a `release-` prefix. Signed-off-by: Daniel Grimm --- Makefile.core.mk | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index 37f53ad46..515df20e6 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -80,9 +80,6 @@ GINKGO_FLAGS := $(if $(VERBOSE),-v) $(if $(CI),--no-color) $(if $(COVERAGE),-cov # - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) # - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") CHANNEL_PREFIX := dev -ifneq (,$(findstring release-,$(shell git rev-parse --abbrev-ref HEAD))) -CHANNEL_PREFIX = stable -endif CHANNELS ?= $(CHANNEL_PREFIX)-$(MINOR_VERSION) ifneq ($(origin CHANNELS), undefined) From 7c545463c010cc91e8a9a708ef021bc0c561fd69 Mon Sep 17 00:00:00 2001 From: Daniel Grimm Date: Wed, 22 Jan 2025 15:22:57 +0100 Subject: [PATCH 3/3] Remove kube-rbac-proxy (#556) Replaces proxy with controller-runtime functionality. Fixes #502. Signed-off-by: Daniel Grimm --- ...l-operator-metrics-service_v1_service.yaml | 10 +- .../sailoperator.clusterserviceversion.yaml | 25 +-- chart/templates/auth_proxy_service.yaml | 10 +- chart/templates/deployment.yaml | 30 +--- .../rbac/auth_proxy_role_binding.yaml | 2 +- .../rbac/leader_election_role_binding.yaml | 2 +- chart/values.yaml | 11 -- cmd/main.go | 29 +++- go.mod | 12 ++ go.sum | 24 +++ tests/e2e/operator/operator_install_test.go | 146 +++++++++++++++++- tests/e2e/operator/operator_suite_test.go | 12 +- 12 files changed, 228 insertions(+), 85 deletions(-) diff --git a/bundle/manifests/sail-operator-metrics-service_v1_service.yaml b/bundle/manifests/sail-operator-metrics-service_v1_service.yaml index 48a94205d..301787350 100644 --- a/bundle/manifests/sail-operator-metrics-service_v1_service.yaml +++ b/bundle/manifests/sail-operator-metrics-service_v1_service.yaml @@ -3,11 +3,11 @@ kind: Service metadata: creationTimestamp: null labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: sail-operator app.kubernetes.io/created-by: sailoperator - app.kubernetes.io/instance: sail-operator-metrics-service + app.kubernetes.io/instance: sail-operator app.kubernetes.io/managed-by: helm - app.kubernetes.io/name: service + app.kubernetes.io/name: deployment app.kubernetes.io/part-of: sailoperator control-plane: sail-operator name: sail-operator-metrics-service @@ -17,10 +17,8 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: - app.kubernetes.io/created-by: sailoperator - app.kubernetes.io/part-of: sailoperator control-plane: sail-operator status: loadBalancer: {} diff --git a/bundle/manifests/sailoperator.clusterserviceversion.yaml b/bundle/manifests/sailoperator.clusterserviceversion.yaml index f30fd586b..d3f3c0e93 100644 --- a/bundle/manifests/sailoperator.clusterserviceversion.yaml +++ b/bundle/manifests/sailoperator.clusterserviceversion.yaml @@ -716,32 +716,9 @@ spec: values: - linux containers: - - args: - - --secure-listen-address=:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8443 - --zap-log-level=info command: - /sail-operator diff --git a/chart/templates/auth_proxy_service.yaml b/chart/templates/auth_proxy_service.yaml index 0d298466f..e49a412e5 100644 --- a/chart/templates/auth_proxy_service.yaml +++ b/chart/templates/auth_proxy_service.yaml @@ -2,11 +2,11 @@ apiVersion: v1 kind: Service metadata: labels: - app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/component: sail-operator app.kubernetes.io/created-by: {{ .Values.name }} - app.kubernetes.io/instance: {{ .Values.deployment.name }}-metrics-service + app.kubernetes.io/instance: {{ .Values.deployment.name }} app.kubernetes.io/managed-by: helm - app.kubernetes.io/name: service + app.kubernetes.io/name: deployment app.kubernetes.io/part-of: {{ .Values.name }} control-plane: {{ .Values.deployment.name }} name: {{ .Values.deployment.name }}-metrics-service @@ -17,8 +17,6 @@ spec: - name: https port: 8443 protocol: TCP - targetPort: https + targetPort: 8443 selector: - app.kubernetes.io/created-by: {{ .Values.name }} - app.kubernetes.io/part-of: {{ .Values.name }} control-plane: {{ .Values.deployment.name }} diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml index 1d54dde78..aa9502b79 100644 --- a/chart/templates/deployment.yaml +++ b/chart/templates/deployment.yaml @@ -48,40 +48,14 @@ spec: values: - linux containers: - - args: - - --secure-listen-address=:8443 - - --upstream=http://127.0.0.1:8080/ - - --logtostderr=true - - --v=0 - image: {{ .Values.proxy.image }} -{{- if .Values.proxy.imagePullPolicy }} - imagePullPolicy: {{ .Values.proxy.imagePullPolicy }} -{{- end }} - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - protocol: TCP - resources: - limits: - cpu: {{ .Values.proxy.resources.limits.cpu }} - memory: {{ .Values.proxy.resources.limits.memory }} - requests: - cpu: {{ .Values.proxy.resources.requests.cpu }} - memory: {{ .Values.proxy.resources.requests.memory }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - args: - --health-probe-bind-address=:8081 - - --metrics-bind-address=127.0.0.1:8080 + - --metrics-bind-address=:8443 - --zap-log-level={{ .Values.operatorLogLevel }} command: - /sail-operator image: {{ .Values.image }} -{{- if .Values.proxy.imagePullPolicy }} +{{- if .Values.imagePullPolicy }} imagePullPolicy: {{ .Values.imagePullPolicy }} {{- end }} livenessProbe: diff --git a/chart/templates/rbac/auth_proxy_role_binding.yaml b/chart/templates/rbac/auth_proxy_role_binding.yaml index 7f5155305..96d619d1f 100644 --- a/chart/templates/rbac/auth_proxy_role_binding.yaml +++ b/chart/templates/rbac/auth_proxy_role_binding.yaml @@ -13,5 +13,5 @@ roleRef: name: {{ .Values.name }}-proxy-role subjects: - kind: ServiceAccount - name: {{ .Values.deployment.name }} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace }} diff --git a/chart/templates/rbac/leader_election_role_binding.yaml b/chart/templates/rbac/leader_election_role_binding.yaml index 7b59f4a79..1ee75b2e0 100644 --- a/chart/templates/rbac/leader_election_role_binding.yaml +++ b/chart/templates/rbac/leader_election_role_binding.yaml @@ -14,5 +14,5 @@ roleRef: name: leader-election-role subjects: - kind: ServiceAccount - name: {{ .Values.deployment.name }} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace }} diff --git a/chart/values.yaml b/chart/values.yaml index bbd5a83d8..312bb449e 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -54,17 +54,6 @@ csv: image: quay.io/sail-dev/sail-operator:0.3-latest # We're commenting out the imagePullPolicy to use k8s defaults # imagePullPolicy: Always -proxy: - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.16.0 - # We're commenting out the imagePullPolicy to use k8s defaults - # imagePullPolicy: IfNotPresent - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi operator: resources: limits: diff --git a/cmd/main.go b/cmd/main.go index 740faecbb..bce7c9a4b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -15,6 +15,7 @@ package main import ( + "crypto/tls" "flag" "fmt" "net/http" @@ -36,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) @@ -50,7 +52,7 @@ func main() { var leaderElectionEnabled bool var reconcilerCfg config.ReconcilerConfig - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8443", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.StringVar(&configFile, "config-file", "/etc/sail-operator/config.properties", "Location of the config file, propagated by k8s downward APIs") flag.StringVar(&reconcilerCfg.ResourceDirectory, "resource-directory", "/var/lib/sail-operator/resources", "Where to find resources (e.g. charts)") @@ -100,9 +102,32 @@ func main() { }) } + // if the enable-http2 flag is false (the default), http/2 should be disabled + // due to its vulnerabilities. More specifically, disabling http/2 will + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + + tlsOpts := []func(*tls.Config){ + // disable http/2 because of https://github.com/kubernetes/kubernetes/issues/121197 + disableHTTP2, + } + + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: true, + FilterProvider: filters.WithAuthenticationAndAuthorization, + TLSOpts: tlsOpts, + } + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme.Scheme, - Metrics: metricsserver.Options{BindAddress: metricsAddr}, + Metrics: metricsServerOptions, HealthProbeBindAddress: probeAddr, LeaderElection: leaderElectionEnabled, LeaderElectionID: "sail-operator-lock", diff --git a/go.mod b/go.mod index 06ea8cdae..e9fe5a1de 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( ) require ( + cel.dev/expr v0.19.1 // indirect dario.cat/mergo v1.0.1 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect @@ -46,9 +47,11 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible // indirect github.com/Masterminds/sprig/v3 v3.3.0 // indirect github.com/Masterminds/squirrel v1.5.4 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chai2010/gettext-go v1.0.3 // indirect github.com/containerd/containerd v1.7.23 // indirect @@ -70,6 +73,7 @@ require ( github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect @@ -85,6 +89,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.22.1 // indirect github.com/google/gnostic-models v0.6.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect @@ -94,6 +99,7 @@ require ( github.com/gorilla/websocket v1.5.3 // indirect github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/huandu/xstrings v1.5.0 // indirect @@ -138,6 +144,7 @@ require ( github.com/spf13/cast v1.7.0 // indirect github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect @@ -146,8 +153,12 @@ require ( go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect go.opentelemetry.io/otel v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/sdk v1.33.0 // indirect go.opentelemetry.io/otel/trace v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.31.0 // indirect @@ -175,6 +186,7 @@ require ( k8s.io/kubectl v0.32.0 // indirect k8s.io/utils v0.0.0-20241210054802-24370beab758 // indirect oras.land/oras-go v1.2.5 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.1 // indirect sigs.k8s.io/controller-tools v0.15.0 // indirect sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/kustomize/api v0.18.0 // indirect diff --git a/go.sum b/go.sum index 12fde30fd..c13d3d8b4 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= +cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= @@ -32,6 +34,8 @@ github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/O github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -50,6 +54,8 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembj github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v1.0.3 h1:9liNh8t+u26xl5ddmWLmsOsdNLwkdRTg5AG+JnTiM80= @@ -168,6 +174,8 @@ github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k github.com/gomodule/redigo v1.8.2/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.22.1 h1:AfVXx3chM2qwoSbM7Da8g8hX8OVSkBFwX+rz2+PcK40= +github.com/google/cel-go v0.22.1/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -193,6 +201,8 @@ github.com/gosuri/uitable v0.0.4 h1:IG2xLKRvErL3uhY6e1BylFzG+aJiwQviDDTfOKeKTpY= github.com/gosuri/uitable v0.0.4/go.mod h1:tKR86bXuXPZazfOTG1FIzvjIdXzd0mo4Vtn16vt0PJo= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -349,6 +359,8 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -391,10 +403,20 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEj go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/sdk/metric v1.33.0 h1:Gs5VK9/WUJhNXZgn8MR6ITatvAmKeIuCtNbsP3JkNqU= +go.opentelemetry.io/otel/sdk/metric v1.33.0/go.mod h1:dL5ykHZmm1B1nVRk9dDjChwDmt81MjVp3gLkQRwKf/Q= go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -524,6 +546,8 @@ k8s.io/utils v0.0.0-20241210054802-24370beab758 h1:sdbE21q2nlQtFh65saZY+rRM6x6aJ k8s.io/utils v0.0.0-20241210054802-24370beab758/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= oras.land/oras-go v1.2.5/go.mod h1:PuAwRShRZCsZb7g8Ar3jKKQR/2A/qN+pkYxIOd/FAoo= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.1 h1:uOuSLOMBWkJH0TWa9X6l+mj5nZdm6Ay6Bli8HL8rNfk= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.1/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= sigs.k8s.io/controller-tools v0.15.0 h1:4dxdABXGDhIa68Fiwaif0vcu32xfwmgQ+w8p+5CxoAI= diff --git a/tests/e2e/operator/operator_install_test.go b/tests/e2e/operator/operator_install_test.go index aba3f264f..d726e82e3 100644 --- a/tests/e2e/operator/operator_install_test.go +++ b/tests/e2e/operator/operator_install_test.go @@ -17,6 +17,11 @@ package operator import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" "time" "github.com/istio-ecosystem/sail-operator/pkg/kube" @@ -90,10 +95,91 @@ var _ = Describe("Operator", Ordered, func() { It("starts successfully", func(ctx SpecContext) { Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). - Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") + Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Deployment status") + }) + + It("serves metrics securely", func(ctx SpecContext) { + metricsReaderRoleName := "metrics-reader" + metricsServiceName := deploymentName + "-metrics-service" + + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + err := k.CreateFromString(fmt.Sprintf(` +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-reader-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: %s +subjects: +- kind: ServiceAccount + name: %s + namespace: %s +`, metricsReaderRoleName, deploymentName, namespace)) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd := exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + err = cmd.Run() + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g Gomega) { + output, err := k.WithNamespace(namespace).GetYAML("endpoints", metricsServiceName) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") + } + Eventually(verifyMetricsEndpointReady).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + output, err := k.WithNamespace(namespace).Logs("deployment/"+deploymentName, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted).Should(Succeed()) + + By("creating the curl-metrics namespace") + Expect(k.CreateNamespace(curlNamespace)).To(Succeed(), "Namespace failed to be created") + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", curlNamespace, + "--image=quay.io/curl/curl:8.11.1", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics", + token, metricsServiceName, namespace)) + err = cmd.Run() + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", curlNamespace) + output, err := cmd.Output() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(string(output)).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + + By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput() + Expect(metricsOutput).To(ContainSubstring( + "controller_runtime_reconcile_total", + )) }) AfterAll(func() { + Expect(k.DeleteNamespace(curlNamespace)).To(Succeed(), "failed to delete curl namespace") + if CurrentSpecReport().Failed() { common.LogDebugInfo(k) } @@ -128,3 +214,61 @@ func extractCRDNames(crdList *apiextensionsv1.CustomResourceDefinitionList) []st } return names } + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() string { + By("getting the curl-metrics logs") + metricsOutput, err := k.WithNamespace(curlNamespace).Logs("curl-metrics", nil) + Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutput +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/tests/e2e/operator/operator_suite_test.go b/tests/e2e/operator/operator_suite_test.go index 5e5c938a6..11063b486 100644 --- a/tests/e2e/operator/operator_suite_test.go +++ b/tests/e2e/operator/operator_suite_test.go @@ -29,11 +29,13 @@ import ( ) var ( - cl client.Client - skipDeploy = env.GetBool("SKIP_DEPLOY", false) - namespace = common.OperatorNamespace - deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") - multicluster = env.GetBool("MULTICLUSTER", false) + cl client.Client + skipDeploy = env.GetBool("SKIP_DEPLOY", false) + namespace = common.OperatorNamespace + deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") + serviceAccountName = deploymentName + multicluster = env.GetBool("MULTICLUSTER", false) + curlNamespace = "curl-metrics" k kubectl.Kubectl )