diff --git a/.env.example b/.env.example index bdd7e5d318..abd5094a9f 100644 --- a/.env.example +++ b/.env.example @@ -106,10 +106,6 @@ export BOOKWAREHOUSE_NAMESPACE=bookwarehouse # Default: false # export DEPLOY_PROMETHEUS=true -# optional: ENABLE_PROMETHEUS_SCRAPING (true/false) -# Default: true -# export ENABLE_PROMETHEUS_SCRAPING=true - # optional: Maximum of iterations to test for expected return codes. 0 means unlimited. # export CI_MAX_ITERATIONS_THRESHOLD=0 diff --git a/.github/workflows/openshift-nightly.yml b/.github/workflows/openshift-nightly.yml new file mode 100644 index 0000000000..c10788a185 --- /dev/null +++ b/.github/workflows/openshift-nightly.yml @@ -0,0 +1,35 @@ +name: OpenShift Nightly Job +on: + schedule: + - cron: "0 0 * * *" + pull_request: + branches: + - main + - release-* + +jobs: + test: + name: OpenShift Nightly Job + runs-on: ubuntu-latest + steps: + - name: Checkout v2 + uses: actions/checkout@v2 + - name: Authenticate and set context + uses: redhat-actions/oc-login@v1 + with: + openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }} + openshift_token: ${{ secrets.OPENSHIFT_TOKEN }} + insecure_skip_tls_verify: true + - name: Test oc + run: oc version --client + - name: Setup Go 1.16 + uses: actions/setup-go@v1 + with: + go-version: 1.16 + - name: Run e2es + run: | + make build-osm + go test ./tests/e2e -test.v -ginkgo.v -ginkgo.progress -ginkgo.skip="\bHTTP ingress\b" -ginkgo.skip="\bUpgrade\b" -test.timeout 180m -deployOnOpenShift=true + env: + CTR_REGISTRY: openservicemesh + CTR_TAG: ed16e2b4a3b6c3ccb36e6e764804a8d18e242fb6 diff --git a/charts/osm/Chart.yaml b/charts/osm/Chart.yaml index 46bc85710a..2a47d88a48 100644 --- a/charts/osm/Chart.yaml +++ b/charts/osm/Chart.yaml @@ -14,11 +14,11 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 0.9.0-rc.2 +version: 0.9.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: v0.9.0-rc.2 +appVersion: v0.9.1 # This specifies a particular range of k8s versions that the application is compatible with. kubeVersion: ">= 1.18.0" diff --git a/charts/osm/README.md b/charts/osm/README.md index 709de2c67c..9b36f23736 100644 --- a/charts/osm/README.md +++ b/charts/osm/README.md @@ -75,7 +75,6 @@ The following table lists the configurable parameters of the osm chart and their | OpenServiceMesh.enableFluentbit | bool | `false` | Enable Fluent Bit sidecar deployment on OSM controller's pod | | OpenServiceMesh.enablePermissiveTrafficPolicy | bool | `false` | Enable permissive traffic policy mode | | OpenServiceMesh.enablePrivilegedInitContainer | bool | `false` | Run init container in privileged mode | -| OpenServiceMesh.enablePrometheusScraping | bool | `true` | Enable Prometheus metrics scraping on sidecar proxies | | OpenServiceMesh.enforceSingleMesh | bool | `false` | Enforce only deploying one mesh in the cluster | | OpenServiceMesh.envoyLogLevel | string | `"error"` | Log level for the Envoy proxy sidecar | | OpenServiceMesh.featureFlags.enableEgressPolicy | bool | `true` | Enable OSM's Egress policy API. If specified, fine grained control over Egress (external) traffic is enforced | @@ -95,13 +94,26 @@ The following table lists the configurable parameters of the osm chart and their | OpenServiceMesh.grafana.port | int | `3000` | Grafana service's port | | OpenServiceMesh.image.pullPolicy | string | `"IfNotPresent"` | Container image pull policy | | OpenServiceMesh.image.registry | string | `"openservicemesh"` | Container image registry | -| OpenServiceMesh.image.tag | string | `"v0.9.0-rc.2"` | Container image tag | +| OpenServiceMesh.image.tag | string | `"v0.9.1"` | Container image tag | | OpenServiceMesh.imagePullSecrets | list | `[]` | `osm-controller` image pull secret | -| OpenServiceMesh.injector.podLabels | object | `{}` | | +| OpenServiceMesh.inboundPortExclusionList | list | `[]` | Specifies a global list of ports to exclude from inbound traffic interception by the sidecar proxy. If specified, must be a list of positive integers. | +| OpenServiceMesh.injector.autoScale | object | `{"enable":false,"maxReplicas":5,"minReplicas":1,"targetAverageUtilization":80}` | Auto scale configuration | +| OpenServiceMesh.injector.autoScale.enable | bool | `false` | Enable Autoscale | +| OpenServiceMesh.injector.autoScale.maxReplicas | int | `5` | Maximum replicas for autoscale | +| OpenServiceMesh.injector.autoScale.minReplicas | int | `1` | Minimum replicas for autoscale | +| OpenServiceMesh.injector.autoScale.targetAverageUtilization | int | `80` | Average target CPU utilization (%) | +| OpenServiceMesh.injector.enablePodDisruptionBudget | bool | `false` | Enable Pod Disruption Budget | +| OpenServiceMesh.injector.podLabels | object | `{}` | Sidecar injector's pod labels | | OpenServiceMesh.injector.replicaCount | int | `1` | Sidecar injector's replica count | | OpenServiceMesh.injector.resource | object | `{"limits":{"cpu":"0.5","memory":"64M"},"requests":{"cpu":"0.3","memory":"64M"}}` | Sidecar injector's container resource parameters | | OpenServiceMesh.maxDataPlaneConnections | int | `0` | Sets the max data plane connections allowed for an instance of osm-controller, set to 0 to not enforce limits | | OpenServiceMesh.meshName | string | `"osm"` | Identifier for the instance of a service mesh within a cluster | +| OpenServiceMesh.osmController.autoScale | object | `{"enable":false,"maxReplicas":5,"minReplicas":1,"targetAverageUtilization":80}` | Auto scale configuration | +| OpenServiceMesh.osmController.autoScale.enable | bool | `false` | Enable Autoscale | +| OpenServiceMesh.osmController.autoScale.maxReplicas | int | `5` | Maximum replicas for autoscale | +| OpenServiceMesh.osmController.autoScale.minReplicas | int | `1` | Minimum replicas for autoscale | +| OpenServiceMesh.osmController.autoScale.targetAverageUtilization | int | `80` | Average target CPU utilization (%) | +| OpenServiceMesh.osmController.enablePodDisruptionBudget | bool | `false` | Enable Pod Disruption Budget | | OpenServiceMesh.osmController.podLabels | object | `{}` | OSM controller's pod labels | | OpenServiceMesh.osmController.replicaCount | int | `1` | OSM controller's replica count | | OpenServiceMesh.osmController.resource | object | `{"limits":{"cpu":"1.5","memory":"512M"},"requests":{"cpu":"0.5","memory":"128M"}}` | OSM controller's container resource parameters | diff --git a/charts/osm/crds/meshconfig.yaml b/charts/osm/crds/meshconfig.yaml index eef150181c..1af242113b 100644 --- a/charts/osm/crds/meshconfig.yaml +++ b/charts/osm/crds/meshconfig.yaml @@ -68,11 +68,10 @@ spec: description: Image for the Envoy sidecar type: string default: "envoyproxy/envoy-alpine:v1.18.3" - pattern: envoyproxy\/envoy-alpine:v\d+\.\d+\.\d+$ initContainerImage: description: Image for the init container type: string - default: "openservicemesh/init:v0.9.0-rc.2" + default: "openservicemesh/init:v0.9.1" resources: type: object properties: @@ -109,6 +108,13 @@ spec: type: integer minimum: 1 maximum: 65535 + inboundPortExclusionList: + description: Global list of ports to exclude from inbound traffic interception by the sidecar proxy. + type: array + items: + type: integer + minimum: 1 + maximum: 65535 useHTTPSIngress: description: Enable HTTPS ingress on the mesh type: boolean @@ -149,14 +155,14 @@ spec: description: Configuration for observing the service mesh, including metrics, logs, tracing etc,. type: object properties: + osmLogLevel: + description: Allows setting OSM control plane log level at runtime + type: string + default: "info" enableDebugServer: description: Enables a debug endpoint on the osm-controller pod to list information regarding the mesh such as proxy connections, certificates, and SMI policies. type: boolean default: false - prometheusScraping: - description: Enables Prometheus metrics scraping on sidecar proxies. - type: boolean - default: true tracing: description: Configuration for distributed tracing type: object @@ -185,3 +191,16 @@ spec: description: Sets the service certificate validity duration, represented as a sequence of decimal numbers each with optional fraction and a unit suffix. type: string default: "24h" + featureFlags: + description: OSM feature flags + type: object + properties: + enableWASMStats: + type: boolean + default: true + enableEgressPolicy: + type: boolean + default: true + enableMulticlusterMode: + type: boolean + default: false diff --git a/charts/osm/templates/cleanup-hook.yaml b/charts/osm/templates/cleanup-hook.yaml index 54753ff5cb..5fe8153178 100644 --- a/charts/osm/templates/cleanup-hook.yaml +++ b/charts/osm/templates/cleanup-hook.yaml @@ -17,6 +17,9 @@ spec: spec: serviceAccountName: {{ .Release.Name }}-cleanup restartPolicy: Never + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: garbage-collector image: bitnami/kubectl diff --git a/charts/osm/templates/grafana-deployment.yaml b/charts/osm/templates/grafana-deployment.yaml index 341903d1bd..36c2400d14 100644 --- a/charts/osm/templates/grafana-deployment.yaml +++ b/charts/osm/templates/grafana-deployment.yaml @@ -24,6 +24,9 @@ spec: {{- include "restricted.securityContext" . | nindent 6 }} {{- end }} serviceAccountName: osm-grafana + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: grafana image: "grafana/grafana:7.0.1" diff --git a/charts/osm/templates/jaeger-deployment.yaml b/charts/osm/templates/jaeger-deployment.yaml index ebc19795b3..d6fd621d56 100644 --- a/charts/osm/templates/jaeger-deployment.yaml +++ b/charts/osm/templates/jaeger-deployment.yaml @@ -22,6 +22,9 @@ spec: {{- include "restricted.securityContext" . | nindent 6 }} {{- end }} serviceAccountName: jaeger + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: jaeger image: jaegertracing/all-in-one diff --git a/charts/osm/templates/osm-controller-hpa.yaml b/charts/osm/templates/osm-controller-hpa.yaml new file mode 100644 index 0000000000..588a33461b --- /dev/null +++ b/charts/osm/templates/osm-controller-hpa.yaml @@ -0,0 +1,21 @@ +{{- if .Values.OpenServiceMesh.osmController.autoScale.enable }} +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: osm-controller-hpa + namespace: {{ include "osm.namespace" . }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: osm-controller + minReplicas: {{.Values.OpenServiceMesh.osmController.autoScale.minReplicas}} + maxReplicas: {{.Values.OpenServiceMesh.osmController.autoScale.maxReplicas}} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{.Values.OpenServiceMesh.osmController.autoScale.targetAverageUtilization}} +{{- end }} diff --git a/charts/osm/templates/osm-controller-pod-disruption-budget.yaml b/charts/osm/templates/osm-controller-pod-disruption-budget.yaml new file mode 100644 index 0000000000..13d8f4bd1f --- /dev/null +++ b/charts/osm/templates/osm-controller-pod-disruption-budget.yaml @@ -0,0 +1,14 @@ +{{- if .Values.OpenServiceMesh.osmController.enablePodDisruptionBudget }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: osm-controller-pdb + namespace: {{ include "osm.namespace" . }} + labels: + app: osm-controller +spec: + minAvailable: 1 + selector: + matchLabels: + app: osm-controller +{{- end }} diff --git a/charts/osm/templates/osm-deployment.yaml b/charts/osm/templates/osm-deployment.yaml index c34e939c13..72e41d1dd8 100644 --- a/charts/osm/templates/osm-deployment.yaml +++ b/charts/osm/templates/osm-deployment.yaml @@ -67,15 +67,6 @@ spec: "--cert-manager-issuer-name", "{{.Values.OpenServiceMesh.certmanager.issuerName}}", "--cert-manager-issuer-kind", "{{.Values.OpenServiceMesh.certmanager.issuerKind}}", "--cert-manager-issuer-group", "{{.Values.OpenServiceMesh.certmanager.issuerGroup}}", - {{- if .Values.OpenServiceMesh.featureFlags.enableWASMStats }} - "--stats-wasm-experimental", - {{- end }} - {{- if .Values.OpenServiceMesh.featureFlags.enableEgressPolicy }} - "--enable-egress-policy", - {{- end }} - {{- if .Values.OpenServiceMesh.featureFlags.enableMulticlusterMode }} - "--enable-multicluster", - {{- end }} ] resources: limits: diff --git a/charts/osm/templates/osm-injector-hpa.yaml b/charts/osm/templates/osm-injector-hpa.yaml new file mode 100644 index 0000000000..d9650abbdd --- /dev/null +++ b/charts/osm/templates/osm-injector-hpa.yaml @@ -0,0 +1,21 @@ +{{- if .Values.OpenServiceMesh.injector.autoScale.enable }} +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: osm-injector-hpa + namespace: {{ include "osm.namespace" . }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: osm-injector + minReplicas: {{.Values.OpenServiceMesh.injector.autoScale.minReplicas}} + maxReplicas: {{.Values.OpenServiceMesh.injector.autoScale.maxReplicas}} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{.Values.OpenServiceMesh.injector.autoScale.targetAverageUtilization}} +{{- end }} diff --git a/charts/osm/templates/osm-injector-pod-disruption-budget.yaml b/charts/osm/templates/osm-injector-pod-disruption-budget.yaml new file mode 100644 index 0000000000..1c0e236bf4 --- /dev/null +++ b/charts/osm/templates/osm-injector-pod-disruption-budget.yaml @@ -0,0 +1,14 @@ +{{- if .Values.OpenServiceMesh.injector.enablePodDisruptionBudget }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: osm-injector-pdb + namespace: {{ include "osm.namespace" . }} + labels: + app: osm-injector +spec: + minAvailable: 1 + selector: + matchLabels: + app: osm-injector +{{- end }} diff --git a/charts/osm/templates/osm-rbac.yaml b/charts/osm/templates/osm-rbac.yaml index 62c4ed1e3a..6bcaa1f0f8 100644 --- a/charts/osm/templates/osm-rbac.yaml +++ b/charts/osm/templates/osm-rbac.yaml @@ -80,7 +80,10 @@ rules: resources: ["events"] verbs: ["create", "watch"] - apiGroups: [""] - resources: ["secrets", "configmaps"] + resources: ["secrets"] + verbs: ["create", "update", "delete"] + - apiGroups: [""] + resources: ["configmaps"] verbs: ["create", "update"] - apiGroups: ["admissionregistration.k8s.io"] resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] diff --git a/charts/osm/templates/preset-mesh-config.yaml b/charts/osm/templates/preset-mesh-config.yaml index 2f1761dba5..e59d6f9f0d 100644 --- a/charts/osm/templates/preset-mesh-config.yaml +++ b/charts/osm/templates/preset-mesh-config.yaml @@ -15,10 +15,10 @@ spec: useHTTPSIngress: {{.Values.OpenServiceMesh.useHTTPSIngress}} enablePermissiveTrafficPolicyMode: {{.Values.OpenServiceMesh.enablePermissiveTrafficPolicy}} outboundPortExclusionList: {{.Values.OpenServiceMesh.outboundPortExclusionList}} + inboundPortExclusionList: {{.Values.OpenServiceMesh.inboundPortExclusionList}} outboundIPRangeExclusionList: {{.Values.OpenServiceMesh.outboundIPRangeExclusionList}} observability: enableDebugServer: {{.Values.OpenServiceMesh.enableDebugServer}} - prometheusScraping: {{.Values.OpenServiceMesh.enablePrometheusScraping}} tracing: enable: {{.Values.OpenServiceMesh.tracing.enable}} {{- if .Values.OpenServiceMesh.tracing.enable }} @@ -27,4 +27,8 @@ spec: endpoint: {{.Values.OpenServiceMesh.tracing.endpoint | quote}} {{- end }} certificate: - serviceCertValidityDuration: {{.Values.OpenServiceMesh.serviceCertValidityDuration}} \ No newline at end of file + serviceCertValidityDuration: {{.Values.OpenServiceMesh.serviceCertValidityDuration}} + featureFlags: + enableWASMStats: {{.Values.OpenServiceMesh.featureFlags.enableWASMStats}} + enableEgressPolicy: {{.Values.OpenServiceMesh.featureFlags.enableEgressPolicy}} + enableMulticlusterMode: {{.Values.OpenServiceMesh.featureFlags.enableMulticlusterMode}} diff --git a/charts/osm/templates/prometheus-deployment.yaml b/charts/osm/templates/prometheus-deployment.yaml index 3603e703ed..cb758b8b01 100644 --- a/charts/osm/templates/prometheus-deployment.yaml +++ b/charts/osm/templates/prometheus-deployment.yaml @@ -22,6 +22,9 @@ spec: {{- if not (.Capabilities.APIVersions.Has "security.openshift.io/v1") }} {{- include "restricted.securityContext" . | nindent 6 }} {{- end }} + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: prometheus ports: diff --git a/charts/osm/values.schema.json b/charts/osm/values.schema.json index a1fb091f0a..989e492cff 100644 --- a/charts/osm/values.schema.json +++ b/charts/osm/values.schema.json @@ -67,6 +67,59 @@ "additionalProperties": false } } + }, + "autoScale": { + "$id": "#/properties/definitions/properties/autoScale", + "type": "object", + "title": "The autoScale schema", + "description": "Autoscale configuration parameters", + "required": [ + "enable" + ], + "properties": { + "enable": { + "$id": "#/properties/definitions/properties/autoScale/properties/enable", + "type": "boolean", + "title": "Autoscale enable", + "description": "Indicates whether autoscale should be enabled or not.", + "examples": [ + false + ] + }, + "minReplicas": { + "$id": "#/properties/definitions/properties/autoScale/properties/minReplicas", + "type": "integer", + "title": "Autoscale minimum replicas", + "description": "Indicates the minimum replicas for autoscale.", + "minimum": 1, + "maximum": 10, + "examples": [ + 1 + ] + }, + "maxReplicas": { + "$id": "#/properties/definitions/properties/autoScale/properties/maxReplicas", + "type": "integer", + "title": "Autoscale maximum replicase", + "description": "Indicates the maximum replicas for autoscale.", + "minimum": 1, + "maximum": 10, + "examples": [ + 5 + ] + }, + "targetAverageUtilization": { + "$id": "#/properties/definitions/properties/autoScale/properties/targetAverageUtilization", + "type": "integer", + "title": "Autoscale targetAverageUtilization", + "description": "Indicates average target CPU utilization (percentage) for autoscale.", + "minimum": 0, + "maximum": 100, + "examples": [ + 80 + ] + } + } } }, "properties": { @@ -85,7 +138,6 @@ "enablePermissiveTrafficPolicy", "enableEgress", "deployPrometheus", - "enablePrometheusScraping", "deployGrafana", "enableFluentbit", "fluentBit", @@ -130,6 +182,18 @@ "title": "The podLabels schema", "description": "Labels for the osmController pod.", "default": {} + }, + "enablePodDisruptionBudget": { + "$id": "#/properties/OpenServiceMesh/properties/osmController/properties/enablePodDisruptionBudget", + "type": "boolean", + "title": "The enablePodDisruptionBudget schema", + "description": "Indicates whether Pod Disruption Budget should be enabled or not.", + "examples": [ + false + ] + }, + "autoScale": { + "$ref": "#/definitions/autoScale" } }, "additionalProperties": false @@ -259,8 +323,8 @@ "enablePrometheusScraping": { "$id": "#/properties/OpenServiceMesh/properties/enablePrometheusScraping", "type": "boolean", - "title": "The enablePrometheusScraping schema", - "description": "Indicates whether Prometheus scraping should be enabled.", + "title": "Deprecated: The enablePrometheusScraping schema", + "description": "Deprecated: Indicates whether Prometheus scraping should be enabled.", "examples": [ true ] @@ -588,6 +652,18 @@ "title": "The podLabels schema", "description": "Labels for the osm-injector pod.", "default": {} + }, + "enablePodDisruptionBudget": { + "$id": "#/properties/OpenServiceMesh/properties/injector/properties/enablePodDisruptionBudget", + "type": "boolean", + "title": "The enablePodDisruptionBudget schema", + "description": "Indicates whether Pod Disruption Budget should be enabled or not.", + "examples": [ + false + ] + }, + "autoScale": { + "$ref": "#/definitions/autoScale" } }, "additionalProperties": false @@ -700,6 +776,23 @@ ] ] }, + "inboundPortExclusionList": { + "$id": "#/properties/OpenServiceMesh/properties/inboundPortExclusionList", + "type": "array", + "title": "The inboundPortExclusionList schema", + "description": "Inbound port exluclusion list for sidecar traffic interception", + "items": { + "type": "integer", + "minimum": 1, + "maximum": 65535 + }, + "examples": [ + [ + 6379, + 3315 + ] + ] + }, "grafana": { "$id": "#/properties/OpenServiceMesh/properties/grafana", "type": "object", diff --git a/charts/osm/values.yaml b/charts/osm/values.yaml index 57205962eb..3dc0977087 100644 --- a/charts/osm/values.yaml +++ b/charts/osm/values.yaml @@ -12,7 +12,7 @@ OpenServiceMesh: # -- Container image pull policy pullPolicy: IfNotPresent # -- Container image tag - tag: v0.9.0-rc.2 + tag: v0.9.1 # -- `osm-controller` image pull secret imagePullSecrets: [] @@ -34,6 +34,18 @@ OpenServiceMesh: memory: "128M" # -- OSM controller's pod labels podLabels: {} + # -- Enable Pod Disruption Budget + enablePodDisruptionBudget: false + # -- Auto scale configuration + autoScale: + # -- Enable Autoscale + enable: false + # -- Minimum replicas for autoscale + minReplicas: 1 + # -- Maximum replicas for autoscale + maxReplicas: 5 + # -- Average target CPU utilization (%) + targetAverageUtilization: 80 # # -- Prometheus parameters @@ -104,9 +116,6 @@ OpenServiceMesh: # -- Deploy Prometheus with OSM installation deployPrometheus: false - # -- Enable Prometheus metrics scraping on sidecar proxies - enablePrometheusScraping: true - # -- Deploy Grafana with OSM installation deployGrafana: false @@ -191,6 +200,10 @@ OpenServiceMesh: # If specified, must be a list of positive integers. outboundPortExclusionList: [] + # -- Specifies a global list of ports to exclude from inbound traffic interception by the sidecar proxy. + # If specified, must be a list of positive integers. + inboundPortExclusionList: [] + # # -- OSM's sidecar injector parameters injector: @@ -204,7 +217,20 @@ OpenServiceMesh: requests: cpu: "0.3" memory: "64M" + # -- Sidecar injector's pod labels podLabels: {} + # -- Enable Pod Disruption Budget + enablePodDisruptionBudget: false + # -- Auto scale configuration + autoScale: + # -- Enable Autoscale + enable: false + # -- Minimum replicas for autoscale + minReplicas: 1 + # -- Maximum replicas for autoscale + maxReplicas: 5 + # -- Average target CPU utilization (%) + targetAverageUtilization: 80 # -- Run init container in privileged mode enablePrivilegedInitContainer: false diff --git a/cmd/cli/install.go b/cmd/cli/install.go index ea1257cd86..2245e09f46 100644 --- a/cmd/cli/install.go +++ b/cmd/cli/install.go @@ -233,13 +233,6 @@ func (i *installCmd) validateOptions() error { } if setOptions, ok := s["OpenServiceMesh"].(map[string]interface{}); ok { - // if deployPrometheus is true, make sure enablePrometheusScraping is not disabled - if setOptions["deployPrometheus"] == true { - if setOptions["enablePrometheusScraping"] == false { - _, _ = fmt.Fprintf(i.out, "Prometheus scraping is disabled. To enable it, set prometheus_scraping in %s/%s to true.\n", settings.Namespace(), constants.OSMMeshConfig) - } - } - // if certificateManager is vault, ensure all relevant information (vault-host, vault-token) is available if setOptions["certificateManager"] == "vault" { var missingFields []string diff --git a/cmd/cli/install_test.go b/cmd/cli/install_test.go index 6f61a34dd6..79aa231f1c 100644 --- a/cmd/cli/install_test.go +++ b/cmd/cli/install_test.go @@ -534,7 +534,6 @@ var _ = Describe("deployPrometheus is true", func() { installCmd := getDefaultInstallCmd(out) installCmd.setOptions = []string{ "OpenServiceMesh.deployPrometheus=true", - "OpenServiceMesh.enablePrometheusScraping=false", } err = installCmd.run(config) @@ -542,7 +541,6 @@ var _ = Describe("deployPrometheus is true", func() { It("should not error", func() { Expect(err).NotTo(HaveOccurred()) - Expect(out.String()).To(Equal("Prometheus scraping is disabled. To enable it, set prometheus_scraping in osm-system/osm-mesh-config to true.\nOSM installed successfully in namespace [osm-system] with mesh name [osm]\n")) }) }) diff --git a/cmd/cli/mesh_upgrade.go b/cmd/cli/mesh_upgrade.go index 8f14c56630..f6cc9ad7e2 100644 --- a/cmd/cli/mesh_upgrade.go +++ b/cmd/cli/mesh_upgrade.go @@ -16,7 +16,7 @@ import ( const ( defaultContainerRegistry = "openservicemesh" - defaultOsmImageTag = "v0.9.0-rc.2" + defaultOsmImageTag = "v0.9.1" ) const upgradeDesc = ` diff --git a/cmd/cli/namespace_ignore.go b/cmd/cli/namespace_ignore.go index 00365b238b..3f6f688324 100644 --- a/cmd/cli/namespace_ignore.go +++ b/cmd/cli/namespace_ignore.go @@ -11,6 +11,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + + "github.com/openservicemesh/osm/pkg/constants" ) const namespaceIgnoreDescription = ` @@ -21,8 +23,6 @@ The command will not remove previously injected sidecars on pods belonging to the given namespaces. ` -const ignoreLabel = "openservicemesh.io/ignore" - type namespaceIgnoreCmd struct { out io.Writer namespaces []string @@ -77,7 +77,7 @@ func (cmd *namespaceIgnoreCmd) run() error { "%s": "true" } } -}`, ignoreLabel) +}`, constants.IgnoreLabel) _, err := cmd.clientSet.CoreV1().Namespaces().Patch(ctx, ns, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}, "") if err != nil { diff --git a/cmd/cli/namespace_list.go b/cmd/cli/namespace_list.go index b525c7df9e..14415c2db7 100644 --- a/cmd/cli/namespace_list.go +++ b/cmd/cli/namespace_list.go @@ -84,7 +84,7 @@ func (l *namespaceListCmd) run() error { if !ok { sidecarInjectionEnabled = "-" // not set } - if _, ignored := ns.Labels[ignoreLabel]; ignored { + if _, ignored := ns.Labels[constants.IgnoreLabel]; ignored { sidecarInjectionEnabled = "disabled (ignored)" } diff --git a/cmd/cli/namespace_list_test.go b/cmd/cli/namespace_list_test.go index 740ddc7901..829dadd4f1 100644 --- a/cmd/cli/namespace_list_test.go +++ b/cmd/cli/namespace_list_test.go @@ -68,7 +68,7 @@ func TestNamespaceList(t *testing.T) { Name: "ns", Labels: map[string]string{ constants.OSMKubeResourceMonitorAnnotation: "my-mesh", - ignoreLabel: "any value", + constants.IgnoreLabel: "any value", }, Annotations: map[string]string{ constants.SidecarInjectionAnnotation: "enabled", diff --git a/cmd/cli/namespace_test.go b/cmd/cli/namespace_test.go index 392fea6141..a36293e94c 100644 --- a/cmd/cli/namespace_test.go +++ b/cmd/cli/namespace_test.go @@ -703,7 +703,7 @@ var _ = Describe("Running the namespace ignore command", func() { It("should correctly add an ignore label to the namespace", func() { ns, err := fakeClientSet.CoreV1().Namespaces().Get(context.TODO(), testNamespace, metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) - Expect(ns.Labels[ignoreLabel]).To(Equal("true")) + Expect(ns.Labels[constants.IgnoreLabel]).To(Equal("true")) }) }) @@ -738,11 +738,11 @@ var _ = Describe("Running the namespace ignore command", func() { It("should correctly add an ignore label to the namespaces", func() { ns, err := fakeClientSet.CoreV1().Namespaces().Get(context.TODO(), testNamespace, metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) - Expect(ns.Labels[ignoreLabel]).To(Equal("true")) + Expect(ns.Labels[constants.IgnoreLabel]).To(Equal("true")) ns2, err := fakeClientSet.CoreV1().Namespaces().Get(context.TODO(), testNamespace2, metav1.GetOptions{}) Expect(err).ToNot(HaveOccurred()) - Expect(ns2.Labels[ignoreLabel]).To(Equal("true")) + Expect(ns2.Labels[constants.IgnoreLabel]).To(Equal("true")) }) }) }) diff --git a/cmd/cli/testdata/test-chart/templates/deployment.yaml b/cmd/cli/testdata/test-chart/templates/deployment.yaml index 0a72443a3a..52d8abab71 100644 --- a/cmd/cli/testdata/test-chart/templates/deployment.yaml +++ b/cmd/cli/testdata/test-chart/templates/deployment.yaml @@ -15,6 +15,9 @@ spec: testing: test spec: {{- with .Values.OpenServiceMesh.imagePullSecrets }} + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end}} diff --git a/cmd/init-osm-controller/init-osm-controller_test.go b/cmd/init-osm-controller/init-osm-controller_test.go index 7b6163aea8..a3837b5704 100644 --- a/cmd/init-osm-controller/init-osm-controller_test.go +++ b/cmd/init-osm-controller/init-osm-controller_test.go @@ -24,7 +24,7 @@ func TestCreateDefaultMeshConfig(t *testing.T) { Sidecar: v1alpha1.SidecarSpec{ LogLevel: "error", EnvoyImage: "envoyproxy/envoy-alpine:v1.18.3", - InitContainerImage: "openservicemesh/init:v0.9.0-rc.2", + InitContainerImage: "openservicemesh/init:v0.9.1", EnablePrivilegedInitContainer: false, MaxDataPlaneConnections: 0, ConfigResyncInterval: "2s", @@ -35,8 +35,7 @@ func TestCreateDefaultMeshConfig(t *testing.T) { EnablePermissiveTrafficPolicyMode: true, }, Observability: v1alpha1.ObservabilitySpec{ - EnableDebugServer: false, - PrometheusScraping: true, + EnableDebugServer: false, Tracing: v1alpha1.TracingSpec{ Enable: false, }, @@ -55,7 +54,6 @@ func TestCreateDefaultMeshConfig(t *testing.T) { assert.Equal(meshConfig.Spec.Traffic.EnablePermissiveTrafficPolicyMode, true) assert.Equal(meshConfig.Spec.Traffic.EnableEgress, true) assert.Equal(meshConfig.Spec.Traffic.UseHTTPSIngress, false) - assert.Equal(meshConfig.Spec.Observability.PrometheusScraping, true) assert.Equal(meshConfig.Spec.Observability.EnableDebugServer, false) assert.Equal(meshConfig.Spec.Certificate.ServiceCertValidityDuration, "24h") } diff --git a/cmd/osm-controller/log_handler.go b/cmd/osm-controller/log_handler.go new file mode 100644 index 0000000000..a640fa5bf3 --- /dev/null +++ b/cmd/osm-controller/log_handler.go @@ -0,0 +1,52 @@ +package main + +import ( + _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + + "github.com/openservicemesh/osm/pkg/announcements" + "github.com/openservicemesh/osm/pkg/configurator" + "github.com/openservicemesh/osm/pkg/constants" + "github.com/openservicemesh/osm/pkg/kubernetes/events" + "github.com/openservicemesh/osm/pkg/logger" +) + +// StartGlobalLogLevelHandler registers a listener to meshconfig events and log level changes, +// and applies new log level at global scope +func StartGlobalLogLevelHandler(cfg configurator.Configurator, stop <-chan struct{}) { + meshConfigChannel := events.GetPubSubInstance().Subscribe( + announcements.MeshConfigAdded, + announcements.MeshConfigDeleted, + announcements.MeshConfigUpdated) + + // Run config listener + // Bootstrap after subscribing + currentLogLevel := constants.DefaultOSMLogLevel + logLevel := cfg.GetOSMLogLevel() + log.Info().Msgf("Setting initial log level from meshconfig: %s", logLevel) + err := logger.SetLogLevel(logLevel) + if err != nil { + log.Error().Msgf("Error setting initial log level from meshconfig: %v", err) + } else { + currentLogLevel = logLevel + } + + go func() { + for { + select { + case <-meshConfigChannel: + logLevel := cfg.GetOSMLogLevel() + if logLevel != currentLogLevel { + err := logger.SetLogLevel(logLevel) + if err != nil { + log.Error().Msgf("Error setting log level from meshconfig: %v", err) + } else { + log.Info().Msgf("Global log level changed to: %s", logLevel) + currentLogLevel = logLevel + } + } + case <-stop: + return + } + } + }() +} diff --git a/cmd/osm-controller/log_handler_test.go b/cmd/osm-controller/log_handler_test.go new file mode 100644 index 0000000000..6ff741c087 --- /dev/null +++ b/cmd/osm-controller/log_handler_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/rs/zerolog" + tassert "github.com/stretchr/testify/assert" + + "github.com/openservicemesh/osm/pkg/announcements" + "github.com/openservicemesh/osm/pkg/configurator" + "github.com/openservicemesh/osm/pkg/kubernetes/events" +) + +func TestGlobalLogLevelHandler(t *testing.T) { + assert := tassert.New(t) + mockCtrl := gomock.NewController(t) + mockConfigurator := configurator.NewMockConfigurator(mockCtrl) + + stop := make(chan struct{}) + defer close(stop) + + mockConfigurator.EXPECT().GetOSMLogLevel().Return("trace").Times(1) + StartGlobalLogLevelHandler(mockConfigurator, stop) + + // Set log level through a meshconfig event + mockConfigurator.EXPECT().GetOSMLogLevel().Return("warn").Times(1) + events.GetPubSubInstance().Publish(events.PubSubMessage{ + AnnouncementType: announcements.MeshConfigUpdated, + }) + + assert.Eventually(func() bool { + return zerolog.GlobalLevel() == zerolog.WarnLevel + }, 2*time.Second, 25*time.Millisecond, "Global log level did not change in specified time") + + // Reset back + mockConfigurator.EXPECT().GetOSMLogLevel().Return("trace").Times(1) + events.GetPubSubInstance().Publish(events.PubSubMessage{ + AnnouncementType: announcements.MeshConfigUpdated, + }) + + assert.Eventually(func() bool { + return zerolog.GlobalLevel() == zerolog.TraceLevel + }, 2*time.Second, 25*time.Millisecond, "Global log level did not reset to trace") +} diff --git a/cmd/osm-controller/osm-controller.go b/cmd/osm-controller/osm-controller.go index 42d57361a3..102a43c790 100644 --- a/cmd/osm-controller/osm-controller.go +++ b/cmd/osm-controller/osm-controller.go @@ -5,7 +5,6 @@ package main import ( "context" - "encoding/json" "flag" "fmt" "net/http" @@ -34,7 +33,6 @@ import ( "github.com/openservicemesh/osm/pkg/endpoint/providers/kube" "github.com/openservicemesh/osm/pkg/envoy/ads" "github.com/openservicemesh/osm/pkg/envoy/registry" - "github.com/openservicemesh/osm/pkg/featureflags" "github.com/openservicemesh/osm/pkg/gen/client/config/clientset/versioned" "github.com/openservicemesh/osm/pkg/health" "github.com/openservicemesh/osm/pkg/httpserver" @@ -68,9 +66,6 @@ var ( vaultOptions providers.VaultOptions certManagerOptions providers.CertManagerOptions - // feature flag options - optionalFeatures featureflags.OptionalFeatures - scheme = runtime.NewScheme() ) @@ -80,7 +75,7 @@ var ( ) func init() { - flags.StringVarP(&verbosity, "verbosity", "v", "info", "Set log verbosity level") + flags.StringVarP(&verbosity, "verbosity", "v", constants.DefaultOSMLogLevel, "Set boot log verbosity level") flags.StringVar(&meshName, "mesh-name", "", "OSM mesh name") flags.StringVar(&kubeConfigFile, "kubeconfig", "", "Path to Kubernetes config file.") flags.StringVar(&osmNamespace, "osm-namespace", "", "Namespace to which OSM belongs to.") @@ -103,11 +98,6 @@ func init() { flags.StringVar(&certManagerOptions.IssuerKind, "cert-manager-issuer-kind", "Issuer", "cert-manager issuer kind") flags.StringVar(&certManagerOptions.IssuerGroup, "cert-manager-issuer-group", "cert-manager.io", "cert-manager issuer group") - // feature flags - flags.BoolVar(&optionalFeatures.WASMStats, "stats-wasm-experimental", false, "Enable a WebAssembly module that generates additional Envoy statistics") - flags.BoolVar(&optionalFeatures.EgressPolicy, "enable-egress-policy", false, "Enable OSM's Egress policy API") - flags.BoolVar(&optionalFeatures.MulticlusterMode, "enable-multicluster", false, "Enable multicluster mode in OSM") - _ = clientgoscheme.AddToScheme(scheme) _ = admissionv1.AddToScheme(scheme) } @@ -122,13 +112,6 @@ func main() { log.Fatal().Err(err).Msg("Error setting log level") } - if featureFlagsJSON, err := json.Marshal(featureflags.Features); err != nil { - log.Error().Err(err).Msgf("Error marshaling feature flags struct: %+v", featureflags.Features) - } else { - log.Info().Msgf("Feature flags: %s", string(featureFlagsJSON)) - } - - featureflags.Initialize(optionalFeatures) events.GetPubSubInstance() // Just to generate the interface, single routine context // Initialize kube config and client @@ -169,6 +152,9 @@ func main() { } log.Info().Msgf("Initial MeshConfig %s: %s", osmMeshConfigName, meshConfig) + // Start Global log level handler, reads from configurator (meshconfig) + StartGlobalLogLevelHandler(cfg, stop) + kubernetesClient, err := k8s.NewKubernetesController(kubeClient, meshName, stop) if err != nil { events.GenericEventRecorder().FatalEvent(err, events.InitializationError, "Error creating Kubernetes Controller") diff --git a/demo/deploy-bookbuyer.sh b/demo/deploy-bookbuyer.sh index b7231f266d..6421d784f4 100755 --- a/demo/deploy-bookbuyer.sh +++ b/demo/deploy-bookbuyer.sh @@ -47,7 +47,9 @@ spec: version: v1 spec: serviceAccountName: bookbuyer - + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: # Main container with APP - name: bookbuyer diff --git a/demo/deploy-bookstore-with-same-sa.sh b/demo/deploy-bookstore-with-same-sa.sh index 20d023774f..5428f3a37f 100755 --- a/demo/deploy-bookstore-with-same-sa.sh +++ b/demo/deploy-bookstore-with-same-sa.sh @@ -74,6 +74,9 @@ spec: version: $VERSION spec: serviceAccountName: bookstore + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - image: "${CTR_REGISTRY}/bookstore:${CTR_TAG}" imagePullPolicy: Always diff --git a/demo/deploy-bookstore.sh b/demo/deploy-bookstore.sh index ad24859dd9..38c4c67e0b 100755 --- a/demo/deploy-bookstore.sh +++ b/demo/deploy-bookstore.sh @@ -80,6 +80,9 @@ spec: version: $VERSION spec: serviceAccountName: "$SVC" + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - image: "${CTR_REGISTRY}/bookstore:${CTR_TAG}" imagePullPolicy: Always diff --git a/demo/deploy-bookthief.sh b/demo/deploy-bookthief.sh index 1693f0e74f..0318de7191 100755 --- a/demo/deploy-bookthief.sh +++ b/demo/deploy-bookthief.sh @@ -65,7 +65,9 @@ spec: version: v1 spec: serviceAccountName: bookthief - + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: # Main container with APP - name: bookthief diff --git a/demo/deploy-bookwarehouse.sh b/demo/deploy-bookwarehouse.sh index eebf7c37be..38ae079988 100755 --- a/demo/deploy-bookwarehouse.sh +++ b/demo/deploy-bookwarehouse.sh @@ -59,7 +59,9 @@ spec: version: v1 spec: serviceAccountName: bookwarehouse - + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: # Main container with APP - name: bookwarehouse diff --git a/demo/deploy-tcp-client.sh b/demo/deploy-tcp-client.sh index cd218548fd..8a190d968b 100755 --- a/demo/deploy-tcp-client.sh +++ b/demo/deploy-tcp-client.sh @@ -41,6 +41,9 @@ spec: version: v1 spec: serviceAccountName: tcp-client + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: tcp-client image: "${CTR_REGISTRY}/tcp-client:${CTR_TAG}" diff --git a/demo/deploy-tcp-echo-service.sh b/demo/deploy-tcp-echo-service.sh index 855282105b..ec2b827baf 100755 --- a/demo/deploy-tcp-echo-service.sh +++ b/demo/deploy-tcp-echo-service.sh @@ -59,6 +59,9 @@ spec: version: v1 spec: serviceAccountName: tcp-echo + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: tcp-echo-server image: "${CTR_REGISTRY}/tcp-echo-server:${CTR_TAG}" diff --git a/demo/deploy-traffic-specs.sh b/demo/deploy-traffic-specs.sh index e13cf10178..ed213dfbc7 100755 --- a/demo/deploy-traffic-specs.sh +++ b/demo/deploy-traffic-specs.sh @@ -44,6 +44,4 @@ spec: - name: restock-books methods: - POST - headers: - - host: "bookwarehouse.$BOOKWAREHOUSE_NAMESPACE" EOF diff --git a/demo/deploy-vault.sh b/demo/deploy-vault.sh index 669d0a8e5b..d06e30897b 100755 --- a/demo/deploy-vault.sh +++ b/demo/deploy-vault.sh @@ -27,6 +27,9 @@ spec: labels: app: vault spec: + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux terminationGracePeriodSeconds: 10 containers: - name: vault diff --git a/demo/run-osm-demo.sh b/demo/run-osm-demo.sh index e810d04fb0..2e43c78581 100755 --- a/demo/run-osm-demo.sh +++ b/demo/run-osm-demo.sh @@ -33,7 +33,6 @@ DEPLOY_GRAFANA="${DEPLOY_GRAFANA:-false}" DEPLOY_JAEGER="${DEPLOY_JAEGER:-false}" ENABLE_FLUENTBIT="${ENABLE_FLUENTBIT:-false}" DEPLOY_PROMETHEUS="${DEPLOY_PROMETHEUS:-false}" -ENABLE_PROMETHEUS_SCRAPING="${ENABLE_PROMETHEUS_SCRAPING:-true}" DEPLOY_WITH_SAME_SA="${DEPLOY_WITH_SAME_SA:-false}" ENVOY_LOG_LEVEL="${ENVOY_LOG_LEVEL:-debug}" DEPLOY_ON_OPENSHIFT="${DEPLOY_ON_OPENSHIFT:-false}" @@ -107,7 +106,6 @@ if [ "$CERT_MANAGER" = "vault" ]; then --set=OpenServiceMesh.deployJaeger="$DEPLOY_JAEGER" \ --set=OpenServiceMesh.enableFluentbit="$ENABLE_FLUENTBIT" \ --set=OpenServiceMesh.deployPrometheus="$DEPLOY_PROMETHEUS" \ - --set=OpenServiceMesh.enablePrometheusScraping="$ENABLE_PROMETHEUS_SCRAPING" \ --set=OpenServiceMesh.envoyLogLevel="$ENVOY_LOG_LEVEL" \ --set=OpenServiceMesh.controllerLogLevel="trace" \ --timeout=90s \ @@ -128,7 +126,6 @@ else --set=OpenServiceMesh.deployJaeger="$DEPLOY_JAEGER" \ --set=OpenServiceMesh.enableFluentbit="$ENABLE_FLUENTBIT" \ --set=OpenServiceMesh.deployPrometheus="$DEPLOY_PROMETHEUS" \ - --set=OpenServiceMesh.enablePrometheusScraping="$ENABLE_PROMETHEUS_SCRAPING" \ --set=OpenServiceMesh.envoyLogLevel="$ENVOY_LOG_LEVEL" \ --set=OpenServiceMesh.controllerLogLevel="trace" \ --timeout=90s \ diff --git a/docs/example/manifests/access/traffic-access-v1-allow-bookthief.yaml b/docs/example/manifests/access/traffic-access-v1-allow-bookthief.yaml index 240a9fd96c..19453f4f4d 100644 --- a/docs/example/manifests/access/traffic-access-v1-allow-bookthief.yaml +++ b/docs/example/manifests/access/traffic-access-v1-allow-bookthief.yaml @@ -34,12 +34,9 @@ spec: methods: - GET headers: - - host: "bookstore.bookstore" - "user-agent": ".*-http-client/*.*" - "client-app": "bookbuyer" - name: buy-a-book pathRegex: ".*a-book.*new" methods: - GET - headers: - - host: "bookstore.bookstore" diff --git a/docs/example/manifests/access/traffic-access-v1.yaml b/docs/example/manifests/access/traffic-access-v1.yaml index 8349790533..500e75dbe9 100644 --- a/docs/example/manifests/access/traffic-access-v1.yaml +++ b/docs/example/manifests/access/traffic-access-v1.yaml @@ -34,12 +34,9 @@ spec: methods: - GET headers: - - host: "bookstore.bookstore" - "user-agent": ".*-http-client/*.*" - "client-app": "bookbuyer" - name: buy-a-book pathRegex: ".*a-book.*new" methods: - GET - headers: - - host: "bookstore.bookstore" diff --git a/docs/example/manifests/apps/bookbuyer.yaml b/docs/example/manifests/apps/bookbuyer.yaml index 5831915b24..d1213cbc85 100644 --- a/docs/example/manifests/apps/bookbuyer.yaml +++ b/docs/example/manifests/apps/bookbuyer.yaml @@ -26,9 +26,12 @@ spec: version: v1 spec: serviceAccountName: bookbuyer + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: bookbuyer - image: openservicemesh/bookbuyer:v0.9.0-rc.2 + image: openservicemesh/bookbuyer:v0.9.1 imagePullPolicy: Always command: ["/bookbuyer"] env: diff --git a/docs/example/manifests/apps/bookstore-v2.yaml b/docs/example/manifests/apps/bookstore-v2.yaml index a0cd3a276d..1f0d30c77f 100644 --- a/docs/example/manifests/apps/bookstore-v2.yaml +++ b/docs/example/manifests/apps/bookstore-v2.yaml @@ -41,9 +41,12 @@ spec: app: bookstore-v2 spec: serviceAccountName: bookstore-v2 + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: bookstore - image: openservicemesh/bookstore:v0.9.0-rc.2 + image: openservicemesh/bookstore:v0.9.1 imagePullPolicy: Always ports: - containerPort: 14001 diff --git a/docs/example/manifests/apps/bookstore.yaml b/docs/example/manifests/apps/bookstore.yaml index df463bacb1..7c75d7df32 100644 --- a/docs/example/manifests/apps/bookstore.yaml +++ b/docs/example/manifests/apps/bookstore.yaml @@ -41,9 +41,12 @@ spec: app: bookstore spec: serviceAccountName: bookstore + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: bookstore - image: openservicemesh/bookstore:v0.9.0-rc.2 + image: openservicemesh/bookstore:v0.9.1 imagePullPolicy: Always ports: - containerPort: 14001 diff --git a/docs/example/manifests/apps/bookthief.yaml b/docs/example/manifests/apps/bookthief.yaml index 2833de6d8e..949a8452b6 100644 --- a/docs/example/manifests/apps/bookthief.yaml +++ b/docs/example/manifests/apps/bookthief.yaml @@ -25,9 +25,12 @@ spec: version: v1 spec: serviceAccountName: bookthief + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: bookthief - image: openservicemesh/bookthief:v0.9.0-rc.2 + image: openservicemesh/bookthief:v0.9.1 imagePullPolicy: Always command: ["/bookthief"] env: diff --git a/docs/example/manifests/apps/bookwarehouse.yaml b/docs/example/manifests/apps/bookwarehouse.yaml index 26f9174e17..712ff00046 100644 --- a/docs/example/manifests/apps/bookwarehouse.yaml +++ b/docs/example/manifests/apps/bookwarehouse.yaml @@ -42,8 +42,11 @@ spec: version: v1 spec: serviceAccountName: bookwarehouse + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: bookwarehouse - image: openservicemesh/bookwarehouse:v0.9.0-rc.2 + image: openservicemesh/bookwarehouse:v0.9.1 imagePullPolicy: Always command: ["/bookwarehouse"] diff --git a/docs/example/manifests/meshconfig/mesh-config.yaml b/docs/example/manifests/meshconfig/mesh-config.yaml index 3f2274718c..d0b83c3b95 100644 --- a/docs/example/manifests/meshconfig/mesh-config.yaml +++ b/docs/example/manifests/meshconfig/mesh-config.yaml @@ -8,7 +8,7 @@ spec: logLevel: error maxDataPlaneConnections: 0 envoyImage: "envoyproxy/envoy-alpine:v1.18.3" - initContainerImage: "openservicemesh/init:v0.9.0-rc.2" + initContainerImage: "openservicemesh/init:v0.9.1" configResyncInterval: "0s" traffic: enableEgress: false @@ -16,8 +16,8 @@ spec: enablePermissiveTrafficPolicyMode: true observability: enableDebugServer: true - prometheusScraping: true outboundPortExclusionList: [] + inboundPortExclusionList: [] outboundIPRangeExclusionList: [] tracing: enable: false diff --git a/docs/example/manifests/opa/deploy-opa-envoy.yaml b/docs/example/manifests/opa/deploy-opa-envoy.yaml index a67cc4ef42..e797c97d5a 100644 --- a/docs/example/manifests/opa/deploy-opa-envoy.yaml +++ b/docs/example/manifests/opa/deploy-opa-envoy.yaml @@ -18,6 +18,9 @@ spec: labels: app: opa spec: + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - name: opa-envoy image: openpolicyagent/opa:0.28.0-envoy diff --git a/docs/example/manifests/samples/curl/curl.yaml b/docs/example/manifests/samples/curl/curl.yaml index 22fe0a4d49..726d7d9b13 100644 --- a/docs/example/manifests/samples/curl/curl.yaml +++ b/docs/example/manifests/samples/curl/curl.yaml @@ -18,6 +18,9 @@ spec: app: curl spec: serviceAccountName: curl + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - image: curlimages/curl imagePullPolicy: IfNotPresent diff --git a/docs/example/manifests/samples/httpbin/httpbin.yaml b/docs/example/manifests/samples/httpbin/httpbin.yaml index 3d4bc7300c..85b96632a9 100644 --- a/docs/example/manifests/samples/httpbin/httpbin.yaml +++ b/docs/example/manifests/samples/httpbin/httpbin.yaml @@ -33,6 +33,9 @@ spec: app: httpbin spec: serviceAccountName: httpbin + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux containers: - image: kennethreitz/httpbin imagePullPolicy: IfNotPresent diff --git a/go.mod b/go.mod index 4a5b059b7c..1a6e4c56f5 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,6 @@ require ( google.golang.org/grpc v1.36.0 google.golang.org/protobuf v1.25.0 gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect helm.sh/helm/v3 v3.5.3 honnef.co/go/tools v0.1.1 // indirect k8s.io/api v0.20.5 @@ -59,7 +58,7 @@ require ( k8s.io/utils v0.0.0-20201110183641-67b214c5f920 mvdan.cc/gofumpt v0.1.0 // indirect sigs.k8s.io/controller-runtime v0.6.3 - sigs.k8s.io/kind v0.9.0 + sigs.k8s.io/kind v0.11.1 ) replace ( diff --git a/go.sum b/go.sum index a39b273bd5..0025a64695 100644 --- a/go.sum +++ b/go.sum @@ -113,8 +113,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alessio/shellescape v1.2.2 h1:8LnL+ncxhWT2TR00dfJRT25JWWrhkMZXneHVWnetDZg= -github.com/alessio/shellescape v1.2.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -176,7 +176,6 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.8.5/go.mod h1:8KhU6K+zHUEWOSU++mEQYf7D9UZOcQcibUoSm6vCUz4= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed h1:OZmjad4L3H8ncOIR8rnb5MREYqG8ixi5+WbeUsquF0c= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -281,8 +280,8 @@ github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.1.0 h1:B0aXl1o/1cP8NbviYiBMkcHBtUjIJ1/Ccg6b+SwCLQg= -github.com/evanphx/json-patch/v5 v5.1.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch/v5 v5.2.0 h1:8ozOH5xxoMYDt5/u+yMTsVXydVCbTORFnOOoq2lumco= +github.com/evanphx/json-patch/v5 v5.2.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= @@ -424,7 +423,6 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -848,8 +846,8 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible/go.mod h1:xlUlxe/2ItGlQyMTstqeDv9r3U4obH7xYd26TbDQutY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -1271,7 +1269,6 @@ golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1445,7 +1442,6 @@ google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= @@ -1617,8 +1613,8 @@ sigs.k8s.io/controller-runtime v0.5.1-0.20200416234307-5377effd4043/go.mod h1:j4 sigs.k8s.io/controller-runtime v0.6.3 h1:SBbr+inLPEKhvlJtrvDcwIpm+uhDvp63Bl72xYJtoOE= sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY= sigs.k8s.io/controller-tools v0.2.9-0.20200414181213-645d44dca7c0/go.mod h1:YKE/iHvcKITCljdnlqHYe+kAt7ZldvtAwUzQff0k1T0= -sigs.k8s.io/kind v0.9.0 h1:SoDlXq6pEc7dGagHULNRCCBYrLH6xOi7lqXTRXeAlg4= -sigs.k8s.io/kind v0.9.0/go.mod h1:cxKQWwmbtRDzQ+RNKnR6gZG6fjbeTtItp5cGf+ww+1Y= +sigs.k8s.io/kind v0.11.1 h1:pVzOkhUwMBrCB0Q/WllQDO3v14Y+o2V0tFgjTqIUjwA= +sigs.k8s.io/kind v0.11.1/go.mod h1:fRpgVhtqAWrtLB9ED7zQahUimpUXuG/iHT88xYqEGIA= sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/pkg/apis/config/v1alpha1/mesh_config.go b/pkg/apis/config/v1alpha1/mesh_config.go index 3ff917795b..cf94e1501d 100644 --- a/pkg/apis/config/v1alpha1/mesh_config.go +++ b/pkg/apis/config/v1alpha1/mesh_config.go @@ -35,6 +35,9 @@ type MeshConfigSpec struct { // Certificate defines the certificate management configurations for a mesh instance. Certificate CertificateSpec `json:"certificate,omitempty"` + + // FeatureFlags defines the feature flags for a mesh instance. + FeatureFlags FeatureFlags `json:"featureFlags,omitempty"` } // SidecarSpec is the type used to represent the specifications for the proxy sidecar. @@ -72,6 +75,9 @@ type TrafficSpec struct { // OutboundPortExclusionList defines a global list of ports to exclude from outbound traffic interception by the sidecar proxy. OutboundPortExclusionList []int `json:"outboundPortExclusionList,omitempty"` + // InboundPortExclusionList defines a global list of ports to exclude from inbound traffic interception by the sidecar proxy. + InboundPortExclusionList []int `json:"inboundPortExclusionList,omitempty"` + // UseHTTPSIngress defines a boolean indicating if HTTPS Ingress is enabled globally in the mesh. UseHTTPSIngress bool `json:"useHTTPSIngress,omitempty"` @@ -85,12 +91,12 @@ type TrafficSpec struct { // ObservabilitySpec is the type to represent OSM's observability configurations. type ObservabilitySpec struct { + // OSMLogLevel defines the log level for OSM control plane logs. + OSMLogLevel string `json:"osmLogLevel,omitempty"` + // EnableDebugServer defines if the debug endpoint on the OSM controller pod is enabled. EnableDebugServer bool `json:"enableDebugServer,omitempty"` - // PrometheusScraping defines a boolean indicating if sidecars should be configured for Prometheus metrics scraping. - PrometheusScraping bool `json:"prometheusScraping,omitempty"` - // Tracing defines OSM's tracing configuration. Tracing TracingSpec `json:"tracing,omitempty"` } @@ -147,3 +153,15 @@ type MeshConfigList struct { Items []MeshConfig `json:"items"` } + +// FeatureFlags is a type to represent OSM's feature flags. +type FeatureFlags struct { + // EnableWASMStats defines if WASM Stats are enabled. + EnableWASMStats bool `json:"enableWASMStats,omitempty"` + + // EnableEgressPolicy defines if OSM's Egress policy is enabled. + EnableEgressPolicy bool `json:"enableEgressPolicy,omitempty"` + + // EnableMulticlusterMode defines if Multicluster mode is enabled. + EnableMulticlusterMode bool `json:"enableMulticlusterMode,omitempty"` +} diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index 59739906cc..3a1d870eee 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -196,6 +196,11 @@ func (in *TrafficSpec) DeepCopyInto(out *TrafficSpec) { *out = make([]int, len(*in)) copy(*out, *in) } + if in.InboundPortExclusionList != nil { + in, out := &in.InboundPortExclusionList, &out.InboundPortExclusionList + *out = make([]int, len(*in)) + copy(*out, *in) + } out.InboundExternalAuthorization = in.InboundExternalAuthorization return } diff --git a/pkg/catalog/catalog.go b/pkg/catalog/catalog.go index 55336386ef..fc71c43130 100644 --- a/pkg/catalog/catalog.go +++ b/pkg/catalog/catalog.go @@ -36,3 +36,8 @@ func NewMeshCatalog(kubeController k8s.Controller, kubeClient kubernetes.Interfa return &mc } + +// GetKubeController returns the kube controller instance handling the current cluster +func (mc *MeshCatalog) GetKubeController() k8s.Controller { + return mc.kubeController +} diff --git a/pkg/catalog/egress.go b/pkg/catalog/egress.go index e648fd75b4..20531026c6 100644 --- a/pkg/catalog/egress.go +++ b/pkg/catalog/egress.go @@ -11,7 +11,6 @@ import ( policyV1alpha1 "github.com/openservicemesh/osm/pkg/apis/policy/v1alpha1" "github.com/openservicemesh/osm/pkg/constants" - "github.com/openservicemesh/osm/pkg/featureflags" "github.com/openservicemesh/osm/pkg/identity" "github.com/openservicemesh/osm/pkg/service" "github.com/openservicemesh/osm/pkg/trafficpolicy" @@ -19,7 +18,7 @@ import ( // GetEgressTrafficPolicy returns the Egress traffic policy associated with the given service identity func (mc *MeshCatalog) GetEgressTrafficPolicy(serviceIdentity identity.ServiceIdentity) (*trafficpolicy.EgressTrafficPolicy, error) { - if !featureflags.IsEgressPolicyEnabled() { + if !mc.configurator.GetFeatureFlags().EnableEgressPolicy { return nil, nil } diff --git a/pkg/catalog/egress_test.go b/pkg/catalog/egress_test.go index a12f057e98..f02defec7a 100644 --- a/pkg/catalog/egress_test.go +++ b/pkg/catalog/egress_test.go @@ -13,26 +13,22 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" policyV1alpha1 "github.com/openservicemesh/osm/pkg/apis/policy/v1alpha1" + "github.com/openservicemesh/osm/pkg/configurator" "github.com/openservicemesh/osm/pkg/identity" "github.com/openservicemesh/osm/pkg/policy" - "github.com/openservicemesh/osm/pkg/featureflags" "github.com/openservicemesh/osm/pkg/service" "github.com/openservicemesh/osm/pkg/smi" "github.com/openservicemesh/osm/pkg/trafficpolicy" ) -func init() { - optionalFeatures := featureflags.OptionalFeatures{ - EgressPolicy: true, - } - featureflags.Initialize(optionalFeatures) -} - func TestGetEgressTrafficPolicy(t *testing.T) { assert := tassert.New(t) mockCtrl := gomock.NewController(t) + mockCfg := configurator.NewMockConfigurator(mockCtrl) + defer mockCtrl.Finish() testCases := []struct { @@ -354,9 +350,12 @@ func TestGetEgressTrafficPolicy(t *testing.T) { mc := &MeshCatalog{ meshSpec: mockMeshSpec, + configurator: mockCfg, policyController: mockPolicyController, } + mockCfg.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{EnableEgressPolicy: true}).Times(1) + actual, err := mc.GetEgressTrafficPolicy(testSourceIdentity) assert.Equal(tc.expectError, err != nil) assert.ElementsMatch(tc.expectedEgressPolicy.TrafficMatches, actual.TrafficMatches) diff --git a/pkg/catalog/fake.go b/pkg/catalog/fake.go index 98f5c70999..17e461ed33 100644 --- a/pkg/catalog/fake.go +++ b/pkg/catalog/fake.go @@ -109,6 +109,7 @@ func NewFakeMeshCatalog(kubeClient kubernetes.Interface, meshConfigClient versio mockKubeController.EXPECT().ListServiceIdentitiesForService(tests.BookstoreV1Service).Return([]identity.K8sServiceAccount{tests.BookstoreServiceAccount}, nil).AnyTimes() mockKubeController.EXPECT().ListServiceIdentitiesForService(tests.BookstoreV2Service).Return([]identity.K8sServiceAccount{tests.BookstoreV2ServiceAccount}, nil).AnyTimes() mockKubeController.EXPECT().ListServiceIdentitiesForService(tests.BookbuyerService).Return([]identity.K8sServiceAccount{tests.BookbuyerServiceAccount}, nil).AnyTimes() + mockKubeController.EXPECT().IsMetricsEnabled(gomock.Any()).Return(true).AnyTimes() mockPolicyController.EXPECT().ListEgressPoliciesForSourceIdentity(gomock.Any()).Return(nil).AnyTimes() diff --git a/pkg/catalog/mock_catalog_generated.go b/pkg/catalog/mock_catalog_generated.go index 3f5b49f246..e61bf7713f 100644 --- a/pkg/catalog/mock_catalog_generated.go +++ b/pkg/catalog/mock_catalog_generated.go @@ -10,6 +10,7 @@ import ( gomock "github.com/golang/mock/gomock" endpoint "github.com/openservicemesh/osm/pkg/endpoint" identity "github.com/openservicemesh/osm/pkg/identity" + kubernetes "github.com/openservicemesh/osm/pkg/kubernetes" service "github.com/openservicemesh/osm/pkg/service" trafficpolicy "github.com/openservicemesh/osm/pkg/trafficpolicy" ) @@ -67,6 +68,20 @@ func (mr *MockMeshCatalogerMockRecorder) GetIngressPoliciesForService(arg0 inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIngressPoliciesForService", reflect.TypeOf((*MockMeshCataloger)(nil).GetIngressPoliciesForService), arg0) } +// GetKubeController mocks base method +func (m *MockMeshCataloger) GetKubeController() kubernetes.Controller { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetKubeController") + ret0, _ := ret[0].(kubernetes.Controller) + return ret0 +} + +// GetKubeController indicates an expected call of GetKubeController +func (mr *MockMeshCatalogerMockRecorder) GetKubeController() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetKubeController", reflect.TypeOf((*MockMeshCataloger)(nil).GetKubeController)) +} + // GetPortToProtocolMappingForService mocks base method func (m *MockMeshCataloger) GetPortToProtocolMappingForService(arg0 service.MeshService) (map[uint32]string, error) { m.ctrl.T.Helper() diff --git a/pkg/catalog/types.go b/pkg/catalog/types.go index a70d74935f..5936d0c7b7 100644 --- a/pkg/catalog/types.go +++ b/pkg/catalog/types.go @@ -97,8 +97,11 @@ type MeshCataloger interface { // ListMeshServicesForIdentity lists the services for a given service identity. ListMeshServicesForIdentity(identity.ServiceIdentity) []service.MeshService - // GetEgressTrafficPolicy returns the Egress traffic policy associated with the given service identity + // GetEgressTrafficPolicy returns the Egress traffic policy associated with the given service identity. GetEgressTrafficPolicy(identity.ServiceIdentity) (*trafficpolicy.EgressTrafficPolicy, error) + + // GetKubeController returns the kube controller instance handling the current cluster + GetKubeController() k8s.Controller } type trafficDirection string diff --git a/pkg/configurator/client_test.go b/pkg/configurator/client_test.go index a850b81861..b73142e16a 100644 --- a/pkg/configurator/client_test.go +++ b/pkg/configurator/client_test.go @@ -165,6 +165,13 @@ func TestMeshConfigEventTriggers(t *testing.T) { }, expectProxyBroadcast: true, }, + { + caseName: "osmLogLevel", + updateMeshConfigSpec: func(spec *v1alpha1.MeshConfigSpec) { + spec.Observability.OSMLogLevel = "warn" + }, + expectProxyBroadcast: false, + }, } for _, tc := range tests { diff --git a/pkg/configurator/methods.go b/pkg/configurator/methods.go index 8213d8ea28..7c02582b06 100644 --- a/pkg/configurator/methods.go +++ b/pkg/configurator/methods.go @@ -60,11 +60,6 @@ func (c *Client) IsDebugServerEnabled() bool { return c.getMeshConfig().Spec.Observability.EnableDebugServer } -// IsPrometheusScrapingEnabled determines whether Prometheus is enabled for scraping metrics -func (c *Client) IsPrometheusScrapingEnabled() bool { - return c.getMeshConfig().Spec.Observability.PrometheusScraping -} - // IsTracingEnabled returns whether tracing is enabled func (c *Client) IsTracingEnabled() bool { return c.getMeshConfig().Spec.Observability.Tracing.Enable @@ -156,6 +151,11 @@ func (c *Client) GetOutboundPortExclusionList() []int { return c.getMeshConfig().Spec.Traffic.OutboundPortExclusionList } +// GetInboundPortExclusionList returns the list of ports (positive integers) to exclude from inbound sidecar interception +func (c *Client) GetInboundPortExclusionList() []int { + return c.getMeshConfig().Spec.Traffic.InboundPortExclusionList +} + // IsPrivilegedInitContainer returns whether init containers should be privileged func (c *Client) IsPrivilegedInitContainer() bool { return c.getMeshConfig().Spec.Sidecar.EnablePrivilegedInitContainer @@ -198,3 +198,13 @@ func (c *Client) GetInboundExternalAuthConfig() auth.ExtAuthConfig { return extAuthConfig } + +// GetFeatureFlags returns OSM's feature flags +func (c *Client) GetFeatureFlags() v1alpha1.FeatureFlags { + return c.getMeshConfig().Spec.FeatureFlags +} + +// GetOSMLogLevel returns the configured OSM log level +func (c *Client) GetOSMLogLevel() string { + return c.getMeshConfig().Spec.Observability.OSMLogLevel +} diff --git a/pkg/configurator/methods_test.go b/pkg/configurator/methods_test.go index ab07bce09e..802b6f4153 100644 --- a/pkg/configurator/methods_test.go +++ b/pkg/configurator/methods_test.go @@ -15,6 +15,7 @@ import ( testclient "github.com/openservicemesh/osm/pkg/gen/client/config/clientset/versioned/fake" "github.com/openservicemesh/osm/pkg/announcements" + "github.com/openservicemesh/osm/pkg/constants" "github.com/openservicemesh/osm/pkg/kubernetes/events" ) @@ -62,8 +63,8 @@ func TestCreateUpdateConfig(t *testing.T) { UseHTTPSIngress: true, }, Observability: v1alpha1.ObservabilitySpec{ - EnableDebugServer: true, - PrometheusScraping: true, + OSMLogLevel: constants.DefaultOSMLogLevel, + EnableDebugServer: true, Tracing: v1alpha1.TracingSpec{ Enable: true, }, @@ -88,8 +89,8 @@ func TestCreateUpdateConfig(t *testing.T) { UseHTTPSIngress: true, }, Observability: v1alpha1.ObservabilitySpec{ - EnableDebugServer: true, - PrometheusScraping: true, + OSMLogLevel: constants.DefaultOSMLogLevel, + EnableDebugServer: true, Tracing: v1alpha1.TracingSpec{ Enable: true, }, @@ -163,25 +164,6 @@ func TestCreateUpdateConfig(t *testing.T) { assert.False(cfg.IsDebugServerEnabled()) }, }, - { - name: "IsPrometheusScrapingEnabled", - initialMeshConfigData: &v1alpha1.MeshConfigSpec{ - Observability: v1alpha1.ObservabilitySpec{ - PrometheusScraping: true, - }, - }, - checkCreate: func(assert *tassert.Assertions, cfg Configurator) { - assert.True(cfg.IsPrometheusScrapingEnabled()) - }, - updatedMeshConfigData: &v1alpha1.MeshConfigSpec{ - Observability: v1alpha1.ObservabilitySpec{ - PrometheusScraping: false, - }, - }, - checkUpdate: func(assert *tassert.Assertions, cfg Configurator) { - assert.False(cfg.IsPrometheusScrapingEnabled()) - }, - }, { name: "IsTracingEnabled", initialMeshConfigData: &v1alpha1.MeshConfigSpec{ @@ -267,15 +249,15 @@ func TestCreateUpdateConfig(t *testing.T) { name: "GetInitContainerImage", initialMeshConfigData: &v1alpha1.MeshConfigSpec{}, checkCreate: func(assert *tassert.Assertions, cfg Configurator) { - assert.Equal("openservicemesh/init:v0.9.0-rc.2", cfg.GetInitContainerImage()) + assert.Equal("openservicemesh/init:v0.9.1", cfg.GetInitContainerImage()) }, updatedMeshConfigData: &v1alpha1.MeshConfigSpec{ Sidecar: v1alpha1.SidecarSpec{ - InitContainerImage: "openservicemesh/init:v0.8.2", + InitContainerImage: "openservicemesh/init:v0.9.0", }, }, checkUpdate: func(assert *tassert.Assertions, cfg Configurator) { - assert.Equal("openservicemesh/init:v0.8.2", cfg.GetInitContainerImage()) + assert.Equal("openservicemesh/init:v0.9.0", cfg.GetInitContainerImage()) }, }, { @@ -327,6 +309,21 @@ func TestCreateUpdateConfig(t *testing.T) { assert.Equal([]int{7070, 6080}, cfg.GetOutboundPortExclusionList()) }, }, + { + name: "GetIboundPortExclusionList", + initialMeshConfigData: &v1alpha1.MeshConfigSpec{}, + checkCreate: func(assert *tassert.Assertions, cfg Configurator) { + assert.Nil(cfg.GetInboundPortExclusionList()) + }, + updatedMeshConfigData: &v1alpha1.MeshConfigSpec{ + Traffic: v1alpha1.TrafficSpec{ + InboundPortExclusionList: []int{7070, 6080}, + }, + }, + checkUpdate: func(assert *tassert.Assertions, cfg Configurator) { + assert.Equal([]int{7070, 6080}, cfg.GetInboundPortExclusionList()) + }, + }, { name: "IsPrivilegedInitContainer", initialMeshConfigData: &v1alpha1.MeshConfigSpec{ @@ -425,6 +422,70 @@ func TestCreateUpdateConfig(t *testing.T) { assert.Equal(resource.MustParse("512M"), res.Limits[v1.ResourceMemory]) }, }, + { + name: "IsWASMStatsEnabled", + initialMeshConfigData: &v1alpha1.MeshConfigSpec{}, + checkCreate: func(assert *tassert.Assertions, cfg Configurator) { + assert.Equal(false, cfg.GetFeatureFlags().EnableWASMStats) + }, + updatedMeshConfigData: &v1alpha1.MeshConfigSpec{ + FeatureFlags: v1alpha1.FeatureFlags{ + EnableWASMStats: true, + }, + }, + checkUpdate: func(assert *tassert.Assertions, cfg Configurator) { + assert.Equal(true, cfg.GetFeatureFlags().EnableWASMStats) + }, + }, + { + name: "IsEgressPolicyEnabled", + initialMeshConfigData: &v1alpha1.MeshConfigSpec{}, + checkCreate: func(assert *tassert.Assertions, cfg Configurator) { + assert.Equal(false, cfg.GetFeatureFlags().EnableEgressPolicy) + }, + updatedMeshConfigData: &v1alpha1.MeshConfigSpec{ + FeatureFlags: v1alpha1.FeatureFlags{ + EnableEgressPolicy: true, + }, + }, + checkUpdate: func(assert *tassert.Assertions, cfg Configurator) { + assert.Equal(true, cfg.GetFeatureFlags().EnableEgressPolicy) + }, + }, + { + name: "IsMulticlusterModeEnabled", + initialMeshConfigData: &v1alpha1.MeshConfigSpec{}, + checkCreate: func(assert *tassert.Assertions, cfg Configurator) { + assert.Equal(false, cfg.GetFeatureFlags().EnableMulticlusterMode) + }, + updatedMeshConfigData: &v1alpha1.MeshConfigSpec{ + FeatureFlags: v1alpha1.FeatureFlags{ + EnableMulticlusterMode: true, + }, + }, + checkUpdate: func(assert *tassert.Assertions, cfg Configurator) { + assert.Equal(true, cfg.GetFeatureFlags().EnableMulticlusterMode) + }, + }, + { + name: "OSMLogLevel", + initialMeshConfigData: &v1alpha1.MeshConfigSpec{ + Observability: v1alpha1.ObservabilitySpec{ + OSMLogLevel: constants.DefaultOSMLogLevel, + }, + }, + checkCreate: func(assert *tassert.Assertions, cfg Configurator) { + assert.Equal(constants.DefaultOSMLogLevel, cfg.GetOSMLogLevel()) + }, + updatedMeshConfigData: &v1alpha1.MeshConfigSpec{ + Observability: v1alpha1.ObservabilitySpec{ + OSMLogLevel: "warn", + }, + }, + checkUpdate: func(assert *tassert.Assertions, cfg Configurator) { + assert.Equal("warn", cfg.GetOSMLogLevel()) + }, + }, } for _, test := range tests { diff --git a/pkg/configurator/mock_client_generated.go b/pkg/configurator/mock_client_generated.go index 9d6374fe63..b71fabef02 100644 --- a/pkg/configurator/mock_client_generated.go +++ b/pkg/configurator/mock_client_generated.go @@ -9,6 +9,7 @@ import ( time "time" gomock "github.com/golang/mock/gomock" + v1alpha1 "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" auth "github.com/openservicemesh/osm/pkg/auth" v1 "k8s.io/api/core/v1" ) @@ -78,6 +79,20 @@ func (mr *MockConfiguratorMockRecorder) GetEnvoyLogLevel() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEnvoyLogLevel", reflect.TypeOf((*MockConfigurator)(nil).GetEnvoyLogLevel)) } +// GetFeatureFlags mocks base method +func (m *MockConfigurator) GetFeatureFlags() v1alpha1.FeatureFlags { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFeatureFlags") + ret0, _ := ret[0].(v1alpha1.FeatureFlags) + return ret0 +} + +// GetFeatureFlags indicates an expected call of GetFeatureFlags +func (mr *MockConfiguratorMockRecorder) GetFeatureFlags() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeatureFlags", reflect.TypeOf((*MockConfigurator)(nil).GetFeatureFlags)) +} + // GetInboundExternalAuthConfig mocks base method func (m *MockConfigurator) GetInboundExternalAuthConfig() auth.ExtAuthConfig { m.ctrl.T.Helper() @@ -92,6 +107,20 @@ func (mr *MockConfiguratorMockRecorder) GetInboundExternalAuthConfig() *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInboundExternalAuthConfig", reflect.TypeOf((*MockConfigurator)(nil).GetInboundExternalAuthConfig)) } +// GetInboundPortExclusionList mocks base method +func (m *MockConfigurator) GetInboundPortExclusionList() []int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetInboundPortExclusionList") + ret0, _ := ret[0].([]int) + return ret0 +} + +// GetInboundPortExclusionList indicates an expected call of GetInboundPortExclusionList +func (mr *MockConfiguratorMockRecorder) GetInboundPortExclusionList() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInboundPortExclusionList", reflect.TypeOf((*MockConfigurator)(nil).GetInboundPortExclusionList)) +} + // GetInitContainerImage mocks base method func (m *MockConfigurator) GetInitContainerImage() string { m.ctrl.T.Helper() @@ -135,6 +164,20 @@ func (mr *MockConfiguratorMockRecorder) GetMeshConfigJSON() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMeshConfigJSON", reflect.TypeOf((*MockConfigurator)(nil).GetMeshConfigJSON)) } +// GetOSMLogLevel mocks base method +func (m *MockConfigurator) GetOSMLogLevel() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetOSMLogLevel") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetOSMLogLevel indicates an expected call of GetOSMLogLevel +func (mr *MockConfiguratorMockRecorder) GetOSMLogLevel() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOSMLogLevel", reflect.TypeOf((*MockConfigurator)(nil).GetOSMLogLevel)) +} + // GetOSMNamespace mocks base method func (m *MockConfigurator) GetOSMNamespace() string { m.ctrl.T.Helper() @@ -303,20 +346,6 @@ func (mr *MockConfiguratorMockRecorder) IsPrivilegedInitContainer() *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPrivilegedInitContainer", reflect.TypeOf((*MockConfigurator)(nil).IsPrivilegedInitContainer)) } -// IsPrometheusScrapingEnabled mocks base method -func (m *MockConfigurator) IsPrometheusScrapingEnabled() bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsPrometheusScrapingEnabled") - ret0, _ := ret[0].(bool) - return ret0 -} - -// IsPrometheusScrapingEnabled indicates an expected call of IsPrometheusScrapingEnabled -func (mr *MockConfiguratorMockRecorder) IsPrometheusScrapingEnabled() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPrometheusScrapingEnabled", reflect.TypeOf((*MockConfigurator)(nil).IsPrometheusScrapingEnabled)) -} - // IsTracingEnabled mocks base method func (m *MockConfigurator) IsTracingEnabled() bool { m.ctrl.T.Helper() diff --git a/pkg/configurator/types.go b/pkg/configurator/types.go index 64f8975e0e..04a99ce28a 100644 --- a/pkg/configurator/types.go +++ b/pkg/configurator/types.go @@ -7,6 +7,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" "github.com/openservicemesh/osm/pkg/auth" "github.com/openservicemesh/osm/pkg/logger" ) @@ -42,9 +43,6 @@ type Configurator interface { // IsDebugServerEnabled determines whether osm debug HTTP server is enabled IsDebugServerEnabled() bool - // IsPrometheusScrapingEnabled determines whether Prometheus is enabled for scraping metrics - IsPrometheusScrapingEnabled() bool - // IsTracingEnabled returns whether tracing is enabled IsTracingEnabled() bool @@ -63,6 +61,9 @@ type Configurator interface { // GetMaxDataPlaneConnections returns the max data plane connections allowed, 0 if disabled GetMaxDataPlaneConnections() int + // GetOsmLogLevel returns the configured OSM log level + GetOSMLogLevel() string + // GetEnvoyLogLevel returns the envoy log level GetEnvoyLogLevel() string @@ -81,6 +82,9 @@ type Configurator interface { // GetOutboundPortExclusionList returns the list of ports to exclude from outbound sidecar interception GetOutboundPortExclusionList() []int + // GetInboundPortExclusionList returns the list of ports to exclude from inbound sidecar interception + GetInboundPortExclusionList() []int + // IsPrivilegedInitContainer determines whether init containers should be privileged IsPrivilegedInitContainer() bool @@ -93,4 +97,7 @@ type Configurator interface { // GetInboundExternalAuthConfig returns the External Authentication configuration for incoming traffic, if any GetInboundExternalAuthConfig() auth.ExtAuthConfig + + // GetFeatureFlags returns OSM's feature flags + GetFeatureFlags() v1alpha1.FeatureFlags } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 9d4d25665d..cd96dd6935 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -55,11 +55,14 @@ const ( // DefaultEnvoyLogLevel is the default envoy log level if not defined in the osm MeshConfig DefaultEnvoyLogLevel = "error" + // DefaultOSMLogLevel is the default OSM log level if none is specified + DefaultOSMLogLevel = "info" + // DefaultEnvoyImage is the default envoy proxy sidecar image if not defined in the osm MeshConfig DefaultEnvoyImage = "envoyproxy/envoy-alpine:v1.18.3" // DefaultInitContainerImage is the default init container image if not defined in the osm MeshConfig - DefaultInitContainerImage = "openservicemesh/init:v0.9.0-rc.2" + DefaultInitContainerImage = "openservicemesh/init:v0.9.1" // EnvoyPrometheusInboundListenerPort is Envoy's inbound listener port number for prometheus EnvoyPrometheusInboundListenerPort = 15010 @@ -149,7 +152,7 @@ const ( OSMMeshConfig = "osm-mesh-config" ) -// Annotations used by the controller +// Annotations used by the control plane const ( // SidecarInjectionAnnotation is the annotation used for sidecar injection SidecarInjectionAnnotation = "openservicemesh.io/sidecar-injection" @@ -158,6 +161,12 @@ const ( MetricsAnnotation = "openservicemesh.io/metrics" ) +// Labels used by the control plane +const ( + // IgnoreLabel is the label used to ignore a resource + IgnoreLabel = "openservicemesh.io/ignore" +) + // Annotations used for Metrics const ( // PrometheusScrapeAnnotation is the annotation used to configure prometheus scraping diff --git a/pkg/debugger/feature_flags.go b/pkg/debugger/feature_flags.go index 649c4fed57..e69cb570b4 100644 --- a/pkg/debugger/feature_flags.go +++ b/pkg/debugger/feature_flags.go @@ -4,14 +4,13 @@ import ( "encoding/json" "fmt" "net/http" - - "github.com/openservicemesh/osm/pkg/featureflags" ) func (ds DebugConfig) getFeatureFlags() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if featureFlagsJSON, err := json.Marshal(featureflags.Features); err != nil { - log.Error().Err(err).Msgf("Error marshaling feature flags struct: %+v", featureflags.Features) + featureFlags := ds.configurator.GetFeatureFlags() + if featureFlagsJSON, err := json.Marshal(featureFlags); err != nil { + log.Error().Err(err).Msgf("Error marshaling feature flags struct: %+v", featureFlags) } else { _, _ = fmt.Fprint(w, string(featureFlagsJSON)) } diff --git a/pkg/envoy/ads/response_test.go b/pkg/envoy/ads/response_test.go index 8010d82242..34b1736180 100644 --- a/pkg/envoy/ads/response_test.go +++ b/pkg/envoy/ads/response_test.go @@ -17,6 +17,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" testclient "k8s.io/client-go/kubernetes/fake" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" "github.com/openservicemesh/osm/pkg/auth" configFake "github.com/openservicemesh/osm/pkg/gen/client/config/clientset/versioned/fake" @@ -124,11 +125,14 @@ var _ = Describe("Test ADS response functions", func() { server, actualResponses := tests.NewFakeXDSServer(cert, nil, nil) mockConfigurator.EXPECT().IsEgressEnabled().Return(false).AnyTimes() - mockConfigurator.EXPECT().IsPrometheusScrapingEnabled().Return(false).AnyTimes() mockConfigurator.EXPECT().IsTracingEnabled().Return(false).AnyTimes() mockConfigurator.EXPECT().IsPermissiveTrafficPolicyMode().Return(false).AnyTimes() mockConfigurator.EXPECT().GetServiceCertValidityPeriod().Return(certDuration).AnyTimes() mockConfigurator.EXPECT().IsDebugServerEnabled().Return(true).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + EnableEgressPolicy: false, + }).AnyTimes() It("returns Aggregated Discovery Service response", func() { s := NewADSServer(mc, proxyRegistry, true, tests.Namespace, mockConfigurator, mockCertManager) @@ -204,7 +208,6 @@ var _ = Describe("Test ADS response functions", func() { server, actualResponses := tests.NewFakeXDSServer(cert, nil, nil) mockConfigurator.EXPECT().IsEgressEnabled().Return(false).AnyTimes() - mockConfigurator.EXPECT().IsPrometheusScrapingEnabled().Return(false).AnyTimes() mockConfigurator.EXPECT().IsTracingEnabled().Return(false).AnyTimes() mockConfigurator.EXPECT().IsPermissiveTrafficPolicyMode().Return(false).AnyTimes() mockConfigurator.EXPECT().GetServiceCertValidityPeriod().Return(certDuration).AnyTimes() diff --git a/pkg/envoy/cds/response.go b/pkg/envoy/cds/response.go index e3f1d20e23..afaf210f84 100644 --- a/pkg/envoy/cds/response.go +++ b/pkg/envoy/cds/response.go @@ -75,7 +75,9 @@ func NewResponse(meshCatalog catalog.MeshCataloger, proxy *envoy.Proxy, _ *xds_d } // Add an inbound prometheus cluster (from Prometheus to localhost) - if cfg.IsPrometheusScrapingEnabled() { + if pod, err := envoy.GetPodFromCertificate(proxy.GetCertificateCommonName(), meshCatalog.GetKubeController()); err != nil { + log.Warn().Msgf("Could not find pod for connecting proxy %s. No metadata was recorded.", proxy.GetCertificateSerialNumber()) + } else if meshCatalog.GetKubeController().IsMetricsEnabled(pod) { clusters = append(clusters, getPrometheusCluster()) } diff --git a/pkg/envoy/cds/response_test.go b/pkg/envoy/cds/response_test.go index 37d48d27b9..a26de1d5de 100644 --- a/pkg/envoy/cds/response_test.go +++ b/pkg/envoy/cds/response_test.go @@ -1,6 +1,7 @@ package cds import ( + "context" "fmt" "testing" "time" @@ -19,6 +20,9 @@ import ( tassert "github.com/stretchr/testify/assert" trequire "github.com/stretchr/testify/require" "google.golang.org/protobuf/testing/protocmp" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + testclient "k8s.io/client-go/kubernetes/fake" "github.com/openservicemesh/osm/pkg/catalog" "github.com/openservicemesh/osm/pkg/certificate" @@ -27,6 +31,7 @@ import ( "github.com/openservicemesh/osm/pkg/envoy" "github.com/openservicemesh/osm/pkg/envoy/registry" "github.com/openservicemesh/osm/pkg/envoy/secrets" + k8s "github.com/openservicemesh/osm/pkg/kubernetes" "github.com/openservicemesh/osm/pkg/service" "github.com/openservicemesh/osm/pkg/tests" ) @@ -36,8 +41,10 @@ func TestNewResponse(t *testing.T) { require := trequire.New(t) mockCtrl := gomock.NewController(t) + kubeClient := testclient.NewSimpleClientset() mockConfigurator := configurator.NewMockConfigurator(mockCtrl) mockCatalog := catalog.NewMockMeshCataloger(mockCtrl) + mockKubeController := k8s.NewMockController(mockCtrl) proxyUUID := uuid.New() // The format of the CN matters @@ -53,10 +60,25 @@ func TestNewResponse(t *testing.T) { mockCatalog.EXPECT().GetEgressTrafficPolicy(tests.BookbuyerServiceIdentity).Return(nil, nil).AnyTimes() mockConfigurator.EXPECT().IsPermissiveTrafficPolicyMode().Return(false).AnyTimes() mockConfigurator.EXPECT().IsEgressEnabled().Return(true).AnyTimes() - mockConfigurator.EXPECT().IsPrometheusScrapingEnabled().Return(true).AnyTimes() mockConfigurator.EXPECT().IsTracingEnabled().Return(true).AnyTimes() mockConfigurator.EXPECT().GetTracingHost().Return(constants.DefaultTracingHost).AnyTimes() mockConfigurator.EXPECT().GetTracingPort().Return(constants.DefaultTracingPort).AnyTimes() + mockCatalog.EXPECT().GetKubeController().Return(mockKubeController).AnyTimes() + + podlabels := map[string]string{ + tests.SelectorKey: tests.BookbuyerServiceName, + constants.EnvoyUniqueIDLabelName: proxyUUID.String(), + } + + newPod1 := tests.NewPodFixture(tests.Namespace, fmt.Sprintf("pod-1-%s", proxyUUID), tests.BookbuyerServiceAccountName, podlabels) + newPod1.Annotations = map[string]string{ + constants.PrometheusScrapeAnnotation: "true", + } + _, err := kubeClient.CoreV1().Pods(tests.Namespace).Create(context.TODO(), &newPod1, metav1.CreateOptions{}) + assert.Nil(err) + + mockKubeController.EXPECT().ListPods().Return([]*v1.Pod{&newPod1}) + mockKubeController.EXPECT().IsMetricsEnabled(&newPod1).Return(true) resp, err := NewResponse(mockCatalog, proxy, nil, mockConfigurator, nil, proxyRegistry) assert.Nil(err) diff --git a/pkg/envoy/lds/connection_manager.go b/pkg/envoy/lds/connection_manager.go index 7fee1e36a9..738119ad20 100644 --- a/pkg/envoy/lds/connection_manager.go +++ b/pkg/envoy/lds/connection_manager.go @@ -14,7 +14,6 @@ import ( "github.com/openservicemesh/osm/pkg/configurator" "github.com/openservicemesh/osm/pkg/constants" "github.com/openservicemesh/osm/pkg/envoy" - "github.com/openservicemesh/osm/pkg/featureflags" ) // trafficDirection defines, for filter terms, the direction of the traffic from an application @@ -77,7 +76,7 @@ func getHTTPConnectionManager(routeName string, cfg configurator.Configurator, h connManager.Tracing = tracing } - if featureflags.IsWASMStatsEnabled() { + if cfg.GetFeatureFlags().EnableWASMStats { statsFilter, err := getStatsWASMFilter() if err != nil { log.Error().Err(err).Msg("failed to get stats WASM filter") diff --git a/pkg/envoy/lds/egress_test.go b/pkg/envoy/lds/egress_test.go index 94145d6981..0d9e543ddf 100644 --- a/pkg/envoy/lds/egress_test.go +++ b/pkg/envoy/lds/egress_test.go @@ -10,24 +10,19 @@ import ( tassert "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/wrapperspb" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" "github.com/openservicemesh/osm/pkg/configurator" - "github.com/openservicemesh/osm/pkg/featureflags" "github.com/openservicemesh/osm/pkg/trafficpolicy" ) -func init() { - // Initialize the Egress policy feature - featureflags.Initialize(featureflags.OptionalFeatures{ - EgressPolicy: true, - }) -} - func TestGetEgressHTTPFilterChain(t *testing.T) { assert := tassert.New(t) mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() + mockConfigurator := configurator.NewMockConfigurator(mockCtrl) + testCases := []struct { name string destinationPort int @@ -56,12 +51,13 @@ func TestGetEgressHTTPFilterChain(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - mockConfigurator := configurator.NewMockConfigurator(mockCtrl) lb := &listenerBuilder{ cfg: mockConfigurator, } mockConfigurator.EXPECT().IsTracingEnabled().Return(false).AnyTimes() - + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableEgressPolicy: true, + EnableWASMStats: false}).AnyTimes() actual, err := lb.getEgressHTTPFilterChain(tc.destinationPort) assert.Equal(tc.expectError, err != nil) @@ -78,6 +74,8 @@ func TestGetEgressTCPFilterChain(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() + mockConfigurator := configurator.NewMockConfigurator(mockCtrl) + testCases := []struct { name string trafficMatch trafficpolicy.TrafficMatch @@ -153,6 +151,11 @@ func TestGetEgressTCPFilterChain(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableEgressPolicy: true, + EnableWASMStats: false, + }).AnyTimes() + lb := &listenerBuilder{} actual, err := lb.getEgressTCPFilterChain(tc.trafficMatch) @@ -171,6 +174,8 @@ func TestGetEgressFilterChainsForMatches(t *testing.T) { mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() + mockConfigurator := configurator.NewMockConfigurator(mockCtrl) + testCases := []struct { name string trafficMatches []*trafficpolicy.TrafficMatch @@ -217,11 +222,14 @@ func TestGetEgressFilterChainsForMatches(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - mockConfigurator := configurator.NewMockConfigurator(mockCtrl) lb := &listenerBuilder{ cfg: mockConfigurator, } mockConfigurator.EXPECT().IsTracingEnabled().Return(false).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableEgressPolicy: true, + EnableWASMStats: false, + }).AnyTimes() actual := lb.getEgressFilterChainsForMatches(tc.trafficMatches) diff --git a/pkg/envoy/lds/ingress_test.go b/pkg/envoy/lds/ingress_test.go index 2d19cabee9..a98a08ca59 100644 --- a/pkg/envoy/lds/ingress_test.go +++ b/pkg/envoy/lds/ingress_test.go @@ -11,6 +11,7 @@ import ( "github.com/envoyproxy/go-control-plane/pkg/wellknown" "google.golang.org/protobuf/types/known/wrapperspb" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" "github.com/openservicemesh/osm/pkg/auth" "github.com/openservicemesh/osm/pkg/catalog" "github.com/openservicemesh/osm/pkg/configurator" @@ -108,6 +109,8 @@ func TestGetIngressFilterChains(t *testing.T) { mockConfigurator.EXPECT().GetInboundExternalAuthConfig().Return(auth.ExtAuthConfig{ Enable: false, }).AnyTimes() + // Mock configurator call to determine if WASMStats are enabled + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{EnableWASMStats: false}).AnyTimes() filterChains := lb.getIngressFilterChains(proxyService) diff --git a/pkg/envoy/lds/inmesh_test.go b/pkg/envoy/lds/inmesh_test.go index 29629edd5a..84d14d8554 100644 --- a/pkg/envoy/lds/inmesh_test.go +++ b/pkg/envoy/lds/inmesh_test.go @@ -14,6 +14,7 @@ import ( tassert "github.com/stretchr/testify/assert" "google.golang.org/protobuf/types/known/wrapperspb" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" "github.com/openservicemesh/osm/pkg/auth" "github.com/openservicemesh/osm/pkg/catalog" "github.com/openservicemesh/osm/pkg/configurator" @@ -38,6 +39,9 @@ func TestGetOutboundHTTPFilterChainForService(t *testing.T) { mockConfigurator.EXPECT().GetInboundExternalAuthConfig().Return(auth.ExtAuthConfig{ Enable: false, }).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).AnyTimes() lb := &listenerBuilder{ meshCatalog: mockCatalog, @@ -203,6 +207,9 @@ func TestGetInboundMeshHTTPFilterChain(t *testing.T) { mockConfigurator.EXPECT().GetInboundExternalAuthConfig().Return(auth.ExtAuthConfig{ Enable: false, }).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).AnyTimes() lb := &listenerBuilder{ meshCatalog: mockCatalog, diff --git a/pkg/envoy/lds/listener.go b/pkg/envoy/lds/listener.go index 0a4eb7674b..5bd61a080a 100644 --- a/pkg/envoy/lds/listener.go +++ b/pkg/envoy/lds/listener.go @@ -14,7 +14,6 @@ import ( "github.com/openservicemesh/osm/pkg/constants" "github.com/openservicemesh/osm/pkg/envoy" - "github.com/openservicemesh/osm/pkg/featureflags" "github.com/openservicemesh/osm/pkg/trafficpolicy" ) @@ -57,7 +56,7 @@ func (lb *listenerBuilder) newOutboundListener() (*xds_listener.Listener, error) listener.DefaultFilterChain = egressFilterChain } - if featureflags.IsEgressPolicyEnabled() { + if featureflags := lb.cfg.GetFeatureFlags(); featureflags.EnableEgressPolicy { var filterDisableMatchPredicate *xds_listener.ListenerFilterChainMatchPredicate // Create filter chains for egress based on policies if egressTrafficPolicy, err := lb.meshCatalog.GetEgressTrafficPolicy(lb.serviceIdentity); err != nil { diff --git a/pkg/envoy/lds/listener_test.go b/pkg/envoy/lds/listener_test.go index a0f24c5bc3..863fe8e3de 100644 --- a/pkg/envoy/lds/listener_test.go +++ b/pkg/envoy/lds/listener_test.go @@ -14,12 +14,12 @@ import ( . "github.com/onsi/gomega" tassert "github.com/stretchr/testify/assert" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" "github.com/openservicemesh/osm/pkg/auth" "github.com/openservicemesh/osm/pkg/configurator" "github.com/openservicemesh/osm/pkg/constants" "github.com/openservicemesh/osm/pkg/envoy" "github.com/openservicemesh/osm/pkg/envoy/rds/route" - "github.com/openservicemesh/osm/pkg/featureflags" "github.com/openservicemesh/osm/pkg/trafficpolicy" ) @@ -42,6 +42,9 @@ func TestGetFilterForService(t *testing.T) { mockConfigurator.EXPECT().GetInboundExternalAuthConfig().Return(auth.ExtAuthConfig{ Enable: false, }).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).AnyTimes() // Check we get HTTP connection manager filter without Permissive mode filter, err := lb.getOutboundHTTPFilter(route.OutboundRouteConfigName) @@ -108,10 +111,16 @@ var _ = Describe("Test getHTTPConnectionManager", func() { It("Should have the correct StatPrefix", func() { mockConfigurator.EXPECT().IsTracingEnabled().Return(false).Times(1) + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).Times(1) connManager := getHTTPConnectionManager("foo", mockConfigurator, nil, outbound) Expect(connManager.StatPrefix).To(Equal("mesh-http-conn-manager.foo")) mockConfigurator.EXPECT().IsTracingEnabled().Return(false).Times(1) + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).Times(1) connManager = getHTTPConnectionManager("bar", mockConfigurator, nil, outbound) Expect(connManager.StatPrefix).To(Equal("mesh-http-conn-manager.bar")) }) @@ -121,6 +130,9 @@ var _ = Describe("Test getHTTPConnectionManager", func() { mockConfigurator.EXPECT().GetTracingPort().Return(constants.DefaultTracingPort).Times(1) mockConfigurator.EXPECT().GetTracingEndpoint().Return(constants.DefaultTracingEndpoint).Times(1) mockConfigurator.EXPECT().IsTracingEnabled().Return(true).Times(1) + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).Times(1) connManager := getHTTPConnectionManager(route.InboundRouteConfigName, mockConfigurator, nil, outbound) @@ -130,6 +142,9 @@ var _ = Describe("Test getHTTPConnectionManager", func() { It("Returns proper Zipkin config given when tracing is disabled", func() { mockConfigurator.EXPECT().IsTracingEnabled().Return(false).Times(1) + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).Times(1) connManager := getHTTPConnectionManager(route.InboundRouteConfigName, mockConfigurator, nil, outbound) var nilHcmTrace *xds_hcm.HttpConnectionManager_Tracing = nil @@ -139,9 +154,9 @@ var _ = Describe("Test getHTTPConnectionManager", func() { It("Returns no stats config when WASM is disabled", func() { mockConfigurator.EXPECT().IsTracingEnabled().AnyTimes() - - oldWASMflag := featureflags.Features.WASMStats - featureflags.Features.WASMStats = false + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).Times(1) oldStatsWASMBytes := statsWASMBytes statsWASMBytes = testWASM @@ -155,14 +170,13 @@ var _ = Describe("Test getHTTPConnectionManager", func() { // reset global state statsWASMBytes = oldStatsWASMBytes - featureflags.Features.WASMStats = oldWASMflag }) It("Returns no stats config when WASM is disabled and no WASM is defined", func() { mockConfigurator.EXPECT().IsTracingEnabled().AnyTimes() - - oldWASMflag := featureflags.Features.WASMStats - featureflags.Features.WASMStats = true + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: true, + }).Times(1) oldStatsWASMBytes := statsWASMBytes statsWASMBytes = "" @@ -176,14 +190,13 @@ var _ = Describe("Test getHTTPConnectionManager", func() { // reset global state statsWASMBytes = oldStatsWASMBytes - featureflags.Features.WASMStats = oldWASMflag }) It("Returns no Lua headers filter config when there are no headers to add", func() { mockConfigurator.EXPECT().IsTracingEnabled().AnyTimes() - - oldWASMflag := featureflags.Features.WASMStats - featureflags.Features.WASMStats = true + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: true, + }).Times(1) oldStatsWASMBytes := statsWASMBytes statsWASMBytes = testWASM @@ -198,14 +211,13 @@ var _ = Describe("Test getHTTPConnectionManager", func() { // reset global state statsWASMBytes = oldStatsWASMBytes - featureflags.Features.WASMStats = oldWASMflag }) It("Returns proper stats config when WASM is enabled", func() { mockConfigurator.EXPECT().IsTracingEnabled().AnyTimes() - - oldWASMflag := featureflags.Features.WASMStats - featureflags.Features.WASMStats = true + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: true, + }).Times(1) oldStatsWASMBytes := statsWASMBytes statsWASMBytes = testWASM @@ -222,7 +234,6 @@ var _ = Describe("Test getHTTPConnectionManager", func() { // reset global state statsWASMBytes = oldStatsWASMBytes - featureflags.Features.WASMStats = oldWASMflag }) It("Returns inbound external authorization enabled connection manager when enabled by config", func() { @@ -235,6 +246,9 @@ var _ = Describe("Test getHTTPConnectionManager", func() { AuthzTimeout: 3 * time.Second, FailureModeAllow: false, }).Times(1) + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).Times(1) connManager := getHTTPConnectionManager(route.InboundRouteConfigName, mockConfigurator, nil, inbound) diff --git a/pkg/envoy/lds/response.go b/pkg/envoy/lds/response.go index 2333d5cad0..cac1e08ee5 100644 --- a/pkg/envoy/lds/response.go +++ b/pkg/envoy/lds/response.go @@ -9,7 +9,6 @@ import ( "github.com/openservicemesh/osm/pkg/configurator" "github.com/openservicemesh/osm/pkg/envoy" "github.com/openservicemesh/osm/pkg/envoy/registry" - "github.com/openservicemesh/osm/pkg/featureflags" "github.com/openservicemesh/osm/pkg/identity" ) @@ -34,7 +33,7 @@ func NewResponse(meshCatalog catalog.MeshCataloger, proxy *envoy.Proxy, _ *xds_d var ldsResources []types.Resource var statsHeaders map[string]string - if featureflags.IsWASMStatsEnabled() { + if featureflags := cfg.GetFeatureFlags(); featureflags.EnableWASMStats { statsHeaders = proxy.StatsHeaders() } @@ -86,7 +85,9 @@ func NewResponse(meshCatalog catalog.MeshCataloger, proxy *envoy.Proxy, _ *xds_d ldsResources = append(ldsResources, inboundListener) } - if cfg.IsPrometheusScrapingEnabled() { + if pod, err := envoy.GetPodFromCertificate(proxy.GetCertificateCommonName(), meshCatalog.GetKubeController()); err != nil { + log.Warn().Msgf("Could not find pod for connecting proxy %s. No metadata was recorded.", proxy.GetCertificateSerialNumber()) + } else if meshCatalog.GetKubeController().IsMetricsEnabled(pod) { // Build Prometheus listener config prometheusConnManager := getPrometheusConnectionManager() if prometheusListener, err := buildPrometheusListener(prometheusConnManager); err != nil { diff --git a/pkg/envoy/lds/response_test.go b/pkg/envoy/lds/response_test.go index 36148a83b0..98da39b430 100644 --- a/pkg/envoy/lds/response_test.go +++ b/pkg/envoy/lds/response_test.go @@ -1,6 +1,7 @@ package lds import ( + "context" "fmt" "testing" @@ -9,9 +10,11 @@ import ( "github.com/envoyproxy/go-control-plane/pkg/wellknown" "github.com/golang/mock/gomock" tassert "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" testclient "k8s.io/client-go/kubernetes/fake" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" "github.com/openservicemesh/osm/pkg/auth" "github.com/openservicemesh/osm/pkg/envoy/registry" configFake "github.com/openservicemesh/osm/pkg/gen/client/config/clientset/versioned/fake" @@ -30,7 +33,12 @@ func getProxy(kubeClient kubernetes.Interface) (*envoy.Proxy, error) { tests.SelectorKey: tests.BookbuyerService.Name, constants.EnvoyUniqueIDLabelName: tests.ProxyUUID, } - if _, err := tests.MakePod(kubeClient, tests.Namespace, tests.BookbuyerServiceName, tests.BookbuyerServiceAccountName, podLabels); err != nil { + + newPod1 := tests.NewPodFixture(tests.Namespace, tests.BookbuyerServiceName, tests.BookbuyerServiceAccountName, podLabels) + newPod1.Annotations = map[string]string{ + constants.PrometheusScrapeAnnotation: "true", + } + if _, err := kubeClient.CoreV1().Pods(tests.Namespace).Create(context.TODO(), &newPod1, metav1.CreateOptions{}); err != nil { return nil, err } @@ -66,7 +74,6 @@ func TestNewResponse(t *testing.T) { proxy, err := getProxy(kubeClient) assert.Empty(err) - assert.NotNil(meshCatalog) assert.NotNil(proxy) proxyRegistry := registry.NewProxyRegistry(registry.ExplicitProxyServiceMapper(func(*envoy.Proxy) ([]service.MeshService, error) { @@ -74,12 +81,15 @@ func TestNewResponse(t *testing.T) { })) mockConfigurator.EXPECT().IsPermissiveTrafficPolicyMode().Return(false).AnyTimes() - mockConfigurator.EXPECT().IsPrometheusScrapingEnabled().Return(true).AnyTimes() mockConfigurator.EXPECT().IsTracingEnabled().Return(false).AnyTimes() mockConfigurator.EXPECT().IsEgressEnabled().Return(true).AnyTimes() mockConfigurator.EXPECT().GetInboundExternalAuthConfig().Return(auth.ExtAuthConfig{ Enable: false, }).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + EnableEgressPolicy: true, + }).AnyTimes() resources, err := NewResponse(meshCatalog, proxy, nil, mockConfigurator, nil, proxyRegistry) assert.Empty(err) diff --git a/pkg/envoy/rds/response.go b/pkg/envoy/rds/response.go index 6dd2aff088..d60d068854 100644 --- a/pkg/envoy/rds/response.go +++ b/pkg/envoy/rds/response.go @@ -38,7 +38,7 @@ func NewResponse(cataloger catalog.MeshCataloger, proxy *envoy.Proxy, discoveryR inboundTrafficPolicies = cataloger.ListInboundTrafficPolicies(proxyIdentity.ToServiceIdentity(), services) outboundTrafficPolicies = cataloger.ListOutboundTrafficPolicies(proxyIdentity.ToServiceIdentity()) - routeConfiguration := route.BuildRouteConfiguration(inboundTrafficPolicies, outboundTrafficPolicies, proxy) + routeConfiguration := route.BuildRouteConfiguration(inboundTrafficPolicies, outboundTrafficPolicies, proxy, cfg) var rdsResources []types.Resource for _, config := range routeConfiguration { @@ -55,7 +55,7 @@ func NewResponse(cataloger catalog.MeshCataloger, proxy *envoy.Proxy, discoveryR ingressTrafficPolicies = trafficpolicy.MergeInboundPolicies(catalog.AllowPartialHostnamesMatch, ingressTrafficPolicies, ingressInboundPolicies...) } if len(ingressTrafficPolicies) > 0 { - ingressRouteConfig := route.BuildIngressConfiguration(ingressTrafficPolicies, proxy) + ingressRouteConfig := route.BuildIngressConfiguration(ingressTrafficPolicies, proxy, cfg) rdsResources = append(rdsResources, ingressRouteConfig) } diff --git a/pkg/envoy/rds/response_test.go b/pkg/envoy/rds/response_test.go index 750fdaaa87..2466610430 100644 --- a/pkg/envoy/rds/response_test.go +++ b/pkg/envoy/rds/response_test.go @@ -19,6 +19,7 @@ import ( "k8s.io/client-go/kubernetes" testclient "k8s.io/client-go/kubernetes/fake" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" "github.com/openservicemesh/osm/pkg/catalog" "github.com/openservicemesh/osm/pkg/certificate" "github.com/openservicemesh/osm/pkg/configurator" @@ -276,6 +277,10 @@ func TestNewResponse(t *testing.T) { mockConfigurator.EXPECT().IsPermissiveTrafficPolicyMode().Return(false).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).AnyTimes() + mockCatalog.EXPECT().ListInboundTrafficPolicies(gomock.Any(), gomock.Any()).Return(tc.expectedInboundPolicies).AnyTimes() mockCatalog.EXPECT().ListOutboundTrafficPolicies(gomock.Any()).Return(tc.expectedOutboundPolicies).AnyTimes() mockCatalog.EXPECT().GetIngressPoliciesForService(gomock.Any()).Return(tc.ingressInboundPolicies, nil).AnyTimes() @@ -530,6 +535,10 @@ func TestNewResponseWithPermissiveMode(t *testing.T) { mockConfigurator.EXPECT().IsPermissiveTrafficPolicyMode().Return(true).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).AnyTimes() + resources, err := NewResponse(mockCatalog, testProxy, &discoveryRequest, mockConfigurator, nil, proxyRegistry) assert.Nil(err) @@ -593,6 +602,9 @@ func TestResponseRequestCompletion(t *testing.T) { mockCatalog.EXPECT().GetIngressPoliciesForService(gomock.Any()).Return([]*trafficpolicy.InboundTrafficPolicy{}, nil).AnyTimes() mockCatalog.EXPECT().GetEgressTrafficPolicy(gomock.Any()).Return(nil, nil).AnyTimes() mockConfigurator.EXPECT().IsPermissiveTrafficPolicyMode().Return(false).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).AnyTimes() testCases := []struct { request *xds_discovery.DiscoveryRequest diff --git a/pkg/envoy/rds/route/route_config.go b/pkg/envoy/rds/route/route_config.go index d19c3725a9..f008c63044 100644 --- a/pkg/envoy/rds/route/route_config.go +++ b/pkg/envoy/rds/route/route_config.go @@ -10,9 +10,9 @@ import ( xds_matcher "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" "github.com/golang/protobuf/ptypes/wrappers" + "github.com/openservicemesh/osm/pkg/configurator" "github.com/openservicemesh/osm/pkg/constants" "github.com/openservicemesh/osm/pkg/envoy" - "github.com/openservicemesh/osm/pkg/featureflags" "github.com/openservicemesh/osm/pkg/service" "github.com/openservicemesh/osm/pkg/trafficpolicy" ) @@ -64,7 +64,7 @@ const ( ) // BuildRouteConfiguration constructs the Envoy constructs ([]*xds_route.RouteConfiguration) for implementing inbound and outbound routes -func BuildRouteConfiguration(inbound []*trafficpolicy.InboundTrafficPolicy, outbound []*trafficpolicy.OutboundTrafficPolicy, proxy *envoy.Proxy) []*xds_route.RouteConfiguration { +func BuildRouteConfiguration(inbound []*trafficpolicy.InboundTrafficPolicy, outbound []*trafficpolicy.OutboundTrafficPolicy, proxy *envoy.Proxy, cfg configurator.Configurator) []*xds_route.RouteConfiguration { var routeConfiguration []*xds_route.RouteConfiguration // For both Inbound and Outbound routes, we will always generate the route resource stubs and send them even when empty, @@ -77,7 +77,7 @@ func BuildRouteConfiguration(inbound []*trafficpolicy.InboundTrafficPolicy, outb inboundRouteConfig.VirtualHosts = append(inboundRouteConfig.VirtualHosts, virtualHost) } - if featureflags.IsWASMStatsEnabled() { + if featureFlags := cfg.GetFeatureFlags(); featureFlags.EnableWASMStats { for k, v := range proxy.StatsHeaders() { inboundRouteConfig.ResponseHeadersToAdd = append(inboundRouteConfig.ResponseHeadersToAdd, &core.HeaderValueOption{ Header: &core.HeaderValue{ @@ -102,7 +102,7 @@ func BuildRouteConfiguration(inbound []*trafficpolicy.InboundTrafficPolicy, outb } // BuildIngressConfiguration constructs the Envoy constructs ([]*xds_route.RouteConfiguration) for implementing ingress routes -func BuildIngressConfiguration(ingress []*trafficpolicy.InboundTrafficPolicy, proxy *envoy.Proxy) *xds_route.RouteConfiguration { +func BuildIngressConfiguration(ingress []*trafficpolicy.InboundTrafficPolicy, proxy *envoy.Proxy, cfg configurator.Configurator) *xds_route.RouteConfiguration { if len(ingress) == 0 { return nil } @@ -114,7 +114,7 @@ func BuildIngressConfiguration(ingress []*trafficpolicy.InboundTrafficPolicy, pr ingressRouteConfig.VirtualHosts = append(ingressRouteConfig.VirtualHosts, virtualHost) } - if featureflags.IsWASMStatsEnabled() { + if featureFlags := cfg.GetFeatureFlags(); featureFlags.EnableWASMStats { for k, v := range proxy.StatsHeaders() { ingressRouteConfig.ResponseHeadersToAdd = append(ingressRouteConfig.ResponseHeadersToAdd, &core.HeaderValueOption{ Header: &core.HeaderValue{ diff --git a/pkg/envoy/rds/route/route_config_test.go b/pkg/envoy/rds/route/route_config_test.go index 3eac65f49e..5411c5e6ed 100644 --- a/pkg/envoy/rds/route/route_config_test.go +++ b/pkg/envoy/rds/route/route_config_test.go @@ -7,12 +7,14 @@ import ( mapset "github.com/deckarep/golang-set" xds_route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" xds_matcher "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "github.com/golang/mock/gomock" "github.com/golang/protobuf/ptypes/wrappers" tassert "github.com/stretchr/testify/assert" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" + "github.com/openservicemesh/osm/pkg/configurator" "github.com/openservicemesh/osm/pkg/constants" "github.com/openservicemesh/osm/pkg/envoy" - "github.com/openservicemesh/osm/pkg/featureflags" "github.com/openservicemesh/osm/pkg/identity" "github.com/openservicemesh/osm/pkg/service" "github.com/openservicemesh/osm/pkg/tests" @@ -21,6 +23,10 @@ import ( func TestBuildRouteConfiguration(t *testing.T) { assert := tassert.New(t) + + mockCtrl := gomock.NewController(t) + mockCfg := configurator.NewMockConfigurator(mockCtrl) + testInbound := &trafficpolicy.InboundTrafficPolicy{ Name: "bookstore-v1-default", Hostnames: tests.BookstoreV1Hostnames, @@ -90,7 +96,10 @@ func TestBuildRouteConfiguration(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actual := BuildRouteConfiguration(tc.inbound, tc.outbound, nil) + mockCfg.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).Times(1) + actual := BuildRouteConfiguration(tc.inbound, tc.outbound, nil, mockCfg) assert.Equal(tc.expectedRouteConfigLen, len(actual)) }) } @@ -114,14 +123,12 @@ func TestBuildRouteConfiguration(t *testing.T) { for _, tc := range statsWASMTestCases { t.Run(tc.name, func(t *testing.T) { - oldWASMflag := featureflags.IsWASMStatsEnabled() - featureflags.Features.WASMStats = tc.wasmEnabled - - actual := BuildRouteConfiguration([]*trafficpolicy.InboundTrafficPolicy{testInbound}, nil, &envoy.Proxy{}) + mockCfg.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: tc.wasmEnabled, + }).Times(1) + actual := BuildRouteConfiguration([]*trafficpolicy.InboundTrafficPolicy{testInbound}, nil, &envoy.Proxy{}, mockCfg) tassert.Len(t, actual, 2) tassert.Len(t, actual[0].ResponseHeadersToAdd, tc.expectedResponseHeaderLen) - - featureflags.Features.WASMStats = oldWASMflag }) } } @@ -129,6 +136,9 @@ func TestBuildRouteConfiguration(t *testing.T) { func TestBuildIngressRouteConfiguration(t *testing.T) { assert := tassert.New(t) + mockCtrl := gomock.NewController(t) + mockCfg := configurator.NewMockConfigurator(mockCtrl) + testCases := []struct { name string ingressPolicies []*trafficpolicy.InboundTrafficPolicy @@ -205,7 +215,10 @@ func TestBuildIngressRouteConfiguration(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - actual := BuildIngressConfiguration(tc.ingressPolicies, nil) + mockCfg.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).AnyTimes() + actual := BuildIngressConfiguration(tc.ingressPolicies, nil, mockCfg) if tc.expectedRouteConfigFields == nil { assert.Nil(actual) diff --git a/pkg/envoy/xdsutil.go b/pkg/envoy/xdsutil.go index 31044436f3..b1a9526fa1 100644 --- a/pkg/envoy/xdsutil.go +++ b/pkg/envoy/xdsutil.go @@ -1,7 +1,6 @@ package envoy import ( - "errors" "fmt" "strings" @@ -16,6 +15,7 @@ import ( structpb "github.com/golang/protobuf/ptypes/struct" "github.com/golang/protobuf/ptypes/wrappers" "github.com/google/uuid" + "github.com/pkg/errors" v1 "k8s.io/api/core/v1" "github.com/openservicemesh/osm/pkg/certificate" diff --git a/pkg/featureflags/featureflags.go b/pkg/featureflags/featureflags.go deleted file mode 100644 index a50cf58252..0000000000 --- a/pkg/featureflags/featureflags.go +++ /dev/null @@ -1,49 +0,0 @@ -// Package featureflags implements routines to check if a given feature is enabled. -package featureflags - -import ( - "sync" -) - -// OptionalFeatures is a struct to enable/disable optional features -type OptionalFeatures struct { - WASMStats bool - EgressPolicy bool - MulticlusterMode bool -} - -var ( - // Features describes whether an optional feature is enabled - Features OptionalFeatures - - once sync.Once -) - -// Initialize initializes the feature flag options -func Initialize(optionalFeatures OptionalFeatures) { - once.Do(func() { - Features = optionalFeatures - }) -} - -/* Feature flag stub -// IsFeatureNameEnabled returns a boolean indicating if the feature `FeatureName` is enabled -func IsFeatureNameEnabled() bool { - return Features.FeatureName -} -*/ - -// IsWASMStatsEnabled returns a boolean indicating if custom stats will be generated by a WASM extension to Envoy -func IsWASMStatsEnabled() bool { - return Features.WASMStats -} - -// IsEgressPolicyEnabled returns a boolean indicating if OSM's Egress policy API is enabled -func IsEgressPolicyEnabled() bool { - return Features.EgressPolicy -} - -// IsMulticlusterModeEnabled returns a boolean indicating if multicluster mode is enabled in OSM -func IsMulticlusterModeEnabled() bool { - return Features.MulticlusterMode -} diff --git a/pkg/featureflags/featureflags_test.go b/pkg/featureflags/featureflags_test.go deleted file mode 100644 index c2d10d6bd8..0000000000 --- a/pkg/featureflags/featureflags_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package featureflags - -import ( - "testing" - - tassert "github.com/stretchr/testify/assert" -) - -func TestFlags(t *testing.T) { - assert := tassert.New(t) - - // 1. Verify all optional features are disabled by default - assert.Equal(false, IsWASMStatsEnabled()) - assert.Equal(false, IsEgressPolicyEnabled()) - assert.Equal(false, IsMulticlusterModeEnabled()) - - // 2. Enable all optional features and verify they are enabled - optionalFeatures := OptionalFeatures{ - WASMStats: true, - EgressPolicy: true, - MulticlusterMode: true, - } - Initialize(optionalFeatures) - assert.Equal(true, IsWASMStatsEnabled()) - assert.Equal(true, IsEgressPolicyEnabled()) - assert.Equal(true, IsMulticlusterModeEnabled()) - - // 3. Verify features cannot be reinitialized - optionalFeatures = OptionalFeatures{ - WASMStats: false, - EgressPolicy: false, - MulticlusterMode: false, - } - Initialize(optionalFeatures) - assert.Equal(true, IsWASMStatsEnabled()) - assert.Equal(true, IsEgressPolicyEnabled()) - assert.Equal(true, IsMulticlusterModeEnabled()) -} diff --git a/pkg/featureflags/suite_test.go b/pkg/featureflags/suite_test.go deleted file mode 100644 index f217f86fe6..0000000000 --- a/pkg/featureflags/suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package featureflags - -import ( - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func TestFeatureFlags(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Feature flags Test Suite") -} diff --git a/pkg/ingress/client.go b/pkg/ingress/client.go index a531e418a1..260e4e7121 100644 --- a/pkg/ingress/client.go +++ b/pkg/ingress/client.go @@ -5,6 +5,9 @@ import ( networkingV1 "k8s.io/api/networking/v1" networkingV1beta1 "k8s.io/api/networking/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" "k8s.io/client-go/discovery" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -12,6 +15,7 @@ import ( "github.com/openservicemesh/osm/pkg/announcements" "github.com/openservicemesh/osm/pkg/configurator" + "github.com/openservicemesh/osm/pkg/constants" k8s "github.com/openservicemesh/osm/pkg/kubernetes" "github.com/openservicemesh/osm/pkg/service" ) @@ -24,15 +28,21 @@ const ( var candidateVersions = []string{networkingV1.SchemeGroupVersion.String(), networkingV1beta1.SchemeGroupVersion.String()} // NewIngressClient implements ingress.Monitor and creates the Kubernetes client to monitor Ingress resources. -func NewIngressClient(kubeClient kubernetes.Interface, kubeController k8s.Controller, stop chan struct{}, cfg configurator.Configurator) (Monitor, error) { +func NewIngressClient(kubeClient kubernetes.Interface, kubeController k8s.Controller, stop chan struct{}, _ configurator.Configurator) (Monitor, error) { supportedIngressVersions, err := getSupportedIngressVersions(kubeClient.Discovery()) if err != nil { log.Error().Err(err).Msgf("Error retrieving ingress API versions supported by k8s API server") return nil, err } + // Ignore ingresses that have the ignore label + ignoreLabel, _ := labels.NewRequirement(constants.IgnoreLabel, selection.DoesNotExist, nil) + option := informers.WithTweakListOptions(func(opt *metav1.ListOptions) { + opt.LabelSelector = ignoreLabel.String() + }) + // Initialize the version specific ingress informers and caches - informerFactory := informers.NewSharedInformerFactory(kubeClient, k8s.DefaultKubeEventResyncInterval) + informerFactory := informers.NewSharedInformerFactoryWithOptions(kubeClient, k8s.DefaultKubeEventResyncInterval, option) ingressEventTypes := k8s.EventTypes{ Add: announcements.IngressAdded, Update: announcements.IngressUpdated, @@ -44,33 +54,33 @@ func NewIngressClient(kubeClient kubernetes.Interface, kubeController k8s.Contro return kubeController.IsMonitoredNamespace(ns) } - client := Client{ + c := client{ cacheSynced: make(chan interface{}), kubeController: kubeController, } if v1Supported, ok := supportedIngressVersions[networkingV1.SchemeGroupVersion.String()]; ok && v1Supported { - client.informerV1 = informerFactory.Networking().V1().Ingresses().Informer() - client.cacheV1 = client.informerV1.GetStore() - client.informerV1.AddEventHandler(k8s.GetKubernetesEventHandlers("IngressV1", "Kubernetes", shouldObserve, ingressEventTypes)) + c.informerV1 = informerFactory.Networking().V1().Ingresses().Informer() + c.cacheV1 = c.informerV1.GetStore() + c.informerV1.AddEventHandler(k8s.GetKubernetesEventHandlers("IngressV1", "Kubernetes", shouldObserve, ingressEventTypes)) } if v1beta1Supported, ok := supportedIngressVersions[networkingV1beta1.SchemeGroupVersion.String()]; ok && v1beta1Supported { - client.informerV1beta1 = informerFactory.Networking().V1beta1().Ingresses().Informer() - client.cacheV1Beta1 = client.informerV1beta1.GetStore() - client.informerV1beta1.AddEventHandler(k8s.GetKubernetesEventHandlers("IngressV1beta1", "Kubernetes", shouldObserve, ingressEventTypes)) + c.informerV1beta1 = informerFactory.Networking().V1beta1().Ingresses().Informer() + c.cacheV1Beta1 = c.informerV1beta1.GetStore() + c.informerV1beta1.AddEventHandler(k8s.GetKubernetesEventHandlers("IngressV1beta1", "Kubernetes", shouldObserve, ingressEventTypes)) } - if err := client.run(stop); err != nil { + if err := c.run(stop); err != nil { log.Error().Err(err).Msg("Could not start Kubernetes Ingress client") return nil, err } - return client, nil + return c, nil } // run executes informer collection. -func (c *Client) run(stop <-chan struct{}) error { +func (c *client) run(stop <-chan struct{}) error { log.Info().Msg("Ingress client started") if c.informerV1 == nil && c.informerV1beta1 == nil { @@ -107,7 +117,7 @@ func (c *Client) run(stop <-chan struct{}) error { } // GetIngressNetworkingV1beta1 returns the networking.k8s.io/v1beta1 ingress resources whose backends correspond to the service -func (c Client) GetIngressNetworkingV1beta1(meshService service.MeshService) ([]*networkingV1beta1.Ingress, error) { +func (c client) GetIngressNetworkingV1beta1(meshService service.MeshService) ([]*networkingV1beta1.Ingress, error) { if c.cacheV1Beta1 == nil { // The v1beta1 version is not served by the controller, return an empty list return nil, nil @@ -151,7 +161,7 @@ func (c Client) GetIngressNetworkingV1beta1(meshService service.MeshService) ([] } // GetIngressNetworkingV1 returns the networking.k8s.io/v1 ingress resources whose backends correspond to the service -func (c Client) GetIngressNetworkingV1(meshService service.MeshService) ([]*networkingV1.Ingress, error) { +func (c client) GetIngressNetworkingV1(meshService service.MeshService) ([]*networkingV1.Ingress, error) { if c.cacheV1 == nil { // The v1 version is not served by the controller, return an empty list return nil, nil diff --git a/pkg/ingress/client_test.go b/pkg/ingress/client_test.go index 3066cf074e..5f41c632d6 100644 --- a/pkg/ingress/client_test.go +++ b/pkg/ingress/client_test.go @@ -4,11 +4,23 @@ import ( "fmt" "testing" + "github.com/golang/mock/gomock" + + "github.com/pkg/errors" tassert "github.com/stretchr/testify/assert" + networkingV1 "k8s.io/api/networking/v1" + networkingV1beta1 "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/discovery" + fakeDiscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/utils/pointer" - "github.com/pkg/errors" + "github.com/openservicemesh/osm/pkg/constants" + "github.com/openservicemesh/osm/pkg/kubernetes" + "github.com/openservicemesh/osm/pkg/service" ) type fakeDiscoveryClient struct { @@ -112,3 +124,215 @@ func TestGetSupportedIngressVersions(t *testing.T) { }) } } + +func TestGetIngressNetworkingV1AndVebeta1(t *testing.T) { + mockCtrl := gomock.NewController(t) + mockKubeController := kubernetes.NewMockController(mockCtrl) + assert := tassert.New(t) + + mockKubeController.EXPECT().IsMonitoredNamespace(gomock.Any()).Return(true).AnyTimes() + meshSvc := service.MeshService{Name: "foo", Namespace: "test"} + + testCases := []struct { + name string + ingressResource runtime.Object + version string + expectedMatchCount int + }{ + { + name: "ingress v1 is not ignored", + ingressResource: &networkingV1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "networking.k8s.io/v1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-1", + Namespace: meshSvc.Namespace, + }, + Spec: networkingV1.IngressSpec{ + Rules: []networkingV1.IngressRule{ + { + Host: "fake1.com", + IngressRuleValue: networkingV1.IngressRuleValue{ + HTTP: &networkingV1.HTTPIngressRuleValue{ + Paths: []networkingV1.HTTPIngressPath{ + { + Path: "/fake1-path1", + PathType: (*networkingV1.PathType)(pointer.StringPtr(string(networkingV1.PathTypeImplementationSpecific))), + Backend: networkingV1.IngressBackend{ + Service: &networkingV1.IngressServiceBackend{ + Name: meshSvc.Name, + Port: networkingV1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + version: networkingV1.SchemeGroupVersion.String(), + expectedMatchCount: 1, + }, + { + name: "ingress v1 is ignored using a label", + ingressResource: &networkingV1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "networking.k8s.io/v1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-1", + Namespace: meshSvc.Namespace, + Labels: map[string]string{constants.IgnoreLabel: "true"}, // ignored + }, + Spec: networkingV1.IngressSpec{ + Rules: []networkingV1.IngressRule{ + { + Host: "fake1.com", + IngressRuleValue: networkingV1.IngressRuleValue{ + HTTP: &networkingV1.HTTPIngressRuleValue{ + Paths: []networkingV1.HTTPIngressPath{ + { + Path: "/fake1-path1", + PathType: (*networkingV1.PathType)(pointer.StringPtr(string(networkingV1.PathTypeImplementationSpecific))), + Backend: networkingV1.IngressBackend{ + Service: &networkingV1.IngressServiceBackend{ + Name: meshSvc.Name, + Port: networkingV1.ServiceBackendPort{ + Number: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + version: networkingV1.SchemeGroupVersion.String(), + expectedMatchCount: 0, + }, + { + name: "ingress v1beta1 is not ignored", + ingressResource: &networkingV1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "networking.k8s.io/v1beta1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-1", + Namespace: meshSvc.Namespace, + }, + Spec: networkingV1beta1.IngressSpec{ + Rules: []networkingV1beta1.IngressRule{ + { + Host: "fake1.com", + IngressRuleValue: networkingV1beta1.IngressRuleValue{ + HTTP: &networkingV1beta1.HTTPIngressRuleValue{ + Paths: []networkingV1beta1.HTTPIngressPath{ + { + Path: "/fake1-path1", + PathType: (*networkingV1beta1.PathType)(pointer.StringPtr(string(networkingV1beta1.PathTypeImplementationSpecific))), + Backend: networkingV1beta1.IngressBackend{ + ServiceName: meshSvc.Name, + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + version: networkingV1beta1.SchemeGroupVersion.String(), + expectedMatchCount: 1, + }, + { + name: "ingress v1beta1 is ignored using a label", + ingressResource: &networkingV1beta1.Ingress{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "networking.k8s.io/v1beta1", + Kind: "Ingress", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ingress-1", + Namespace: meshSvc.Namespace, + Labels: map[string]string{constants.IgnoreLabel: "true"}, // ignored + }, + Spec: networkingV1beta1.IngressSpec{ + Rules: []networkingV1beta1.IngressRule{ + { + Host: "fake1.com", + IngressRuleValue: networkingV1beta1.IngressRuleValue{ + HTTP: &networkingV1beta1.HTTPIngressRuleValue{ + Paths: []networkingV1beta1.HTTPIngressPath{ + { + Path: "/fake1-path1", + PathType: (*networkingV1beta1.PathType)(pointer.StringPtr(string(networkingV1beta1.PathTypeImplementationSpecific))), + Backend: networkingV1beta1.IngressBackend{ + ServiceName: meshSvc.Name, + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + version: networkingV1beta1.SchemeGroupVersion.String(), + expectedMatchCount: 0, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewSimpleClientset(tc.ingressResource) + fakeClient.Discovery().(*fakeDiscovery.FakeDiscovery).Resources = []*metav1.APIResourceList{ + { + GroupVersion: networkingV1.SchemeGroupVersion.String(), + APIResources: []metav1.APIResource{ + {Kind: "Ingress"}, + }, + }, + { + GroupVersion: networkingV1beta1.SchemeGroupVersion.String(), + APIResources: []metav1.APIResource{ + {Kind: "Ingress"}, + }, + }, + } + + c, err := NewIngressClient(fakeClient, mockKubeController, make(chan struct{}), nil) + assert.Nil(err) + + switch tc.version { + case networkingV1.SchemeGroupVersion.String(): + ingresses, err := c.GetIngressNetworkingV1(meshSvc) + assert.Nil(err) + assert.Len(ingresses, tc.expectedMatchCount) + + case networkingV1beta1.SchemeGroupVersion.String(): + ingresses, err := c.GetIngressNetworkingV1beta1(meshSvc) + assert.Nil(err) + assert.Len(ingresses, tc.expectedMatchCount) + } + }) + } +} diff --git a/pkg/ingress/types.go b/pkg/ingress/types.go index ec016ddb41..46b55cb88b 100644 --- a/pkg/ingress/types.go +++ b/pkg/ingress/types.go @@ -15,8 +15,8 @@ var ( log = logger.New("kube-ingress") ) -// Client is a struct for all components necessary to connect to and maintain state of a Kubernetes cluster. -type Client struct { +// client is a struct for all components necessary to connect to and maintain state of a Kubernetes cluster. +type client struct { informerV1 cache.SharedIndexInformer cacheV1 cache.Store informerV1beta1 cache.SharedIndexInformer diff --git a/pkg/injector/init_container.go b/pkg/injector/init_container.go index 7a9a185a22..68933e63e1 100644 --- a/pkg/injector/init_container.go +++ b/pkg/injector/init_container.go @@ -9,8 +9,8 @@ import ( ) func getInitContainerSpec(containerName string, cfg configurator.Configurator, outboundIPRangeExclusionList []string, outboundPortExclusionList []int, - enablePrivilegedInitContainer bool) corev1.Container { - iptablesInitCommandsList := generateIptablesCommands(outboundIPRangeExclusionList, outboundPortExclusionList) + inboundPortExclusionList []int, enablePrivilegedInitContainer bool) corev1.Container { + iptablesInitCommandsList := generateIptablesCommands(outboundIPRangeExclusionList, outboundPortExclusionList, inboundPortExclusionList) iptablesInitCommand := strings.Join(iptablesInitCommandsList, " && ") return corev1.Container{ diff --git a/pkg/injector/init_container_test.go b/pkg/injector/init_container_test.go index 04b7e0a6ce..a4d2a03e67 100644 --- a/pkg/injector/init_container_test.go +++ b/pkg/injector/init_container_test.go @@ -23,12 +23,10 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func() mockConfigurator := configurator.NewMockConfigurator(mockCtrl) Context("test getInitContainerSpec()", func() { - It("Creates init container without outbound ip range exclusion list", func() { + It("Creates init container without ip range exclusion list", func() { mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1) - var outboundIPRangeExclusionList []string = nil - var outboundPortExclusionList []int = nil privileged := privilegedFalse - actual := getInitContainerSpec(containerName, mockConfigurator, outboundIPRangeExclusionList, outboundPortExclusionList, privileged) + actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, privileged) expected := corev1.Container{ Name: "-container-name-", @@ -59,9 +57,8 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func() It("Creates init container with outbound exclusion list", func() { mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1) outboundIPRangeExclusionList := []string{"1.1.1.1/32", "10.0.0.10/24"} - var outboundPortExclusionList []int = nil privileged := privilegedFalse - actual := getInitContainerSpec(containerName, mockConfigurator, outboundIPRangeExclusionList, outboundPortExclusionList, privileged) + actual := getInitContainerSpec(containerName, mockConfigurator, outboundIPRangeExclusionList, nil, nil, privileged) expected := corev1.Container{ Name: "-container-name-", @@ -91,10 +88,8 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func() It("Creates init container with privileged true", func() { mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1) - var outboundIPRangeExclusionList []string = nil - var outboundPortExclusionList []int = nil privileged := privilegedTrue - actual := getInitContainerSpec(containerName, mockConfigurator, outboundIPRangeExclusionList, outboundPortExclusionList, privileged) + actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, privileged) expected := corev1.Container{ Name: "-container-name-", @@ -124,10 +119,8 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func() It("Creates init container without outbound port exclusion list", func() { mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1) - var outboundIPRangeExclusionList []string = nil - var outboundPortExclusionList []int = nil privileged := privilegedFalse - actual := getInitContainerSpec(containerName, mockConfigurator, outboundIPRangeExclusionList, outboundPortExclusionList, privileged) + actual := getInitContainerSpec(containerName, mockConfigurator, nil, nil, nil, privileged) expected := corev1.Container{ Name: "-container-name-", @@ -157,10 +150,9 @@ var _ = Describe("Test functions creating Envoy bootstrap configuration", func() It("init container with outbound port exclusion list", func() { mockConfigurator.EXPECT().GetInitContainerImage().Return(containerImage).Times(1) - var outboundIPRangeExclusionList []string = nil outboundPortExclusionList := []int{6060, 7070} privileged := privilegedFalse - actual := getInitContainerSpec(containerName, mockConfigurator, outboundIPRangeExclusionList, outboundPortExclusionList, privileged) + actual := getInitContainerSpec(containerName, mockConfigurator, nil, outboundPortExclusionList, nil, privileged) expected := corev1.Container{ Name: "-container-name-", diff --git a/pkg/injector/iptables.go b/pkg/injector/iptables.go index bd98f8660e..d96d434e91 100644 --- a/pkg/injector/iptables.go +++ b/pkg/injector/iptables.go @@ -69,7 +69,7 @@ var iptablesInboundStaticRules = []string{ } // generateIptablesCommands generates a list of iptables commands to set up sidecar interception and redirection -func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundPortExclusionList []int) []string { +func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundPortExclusionList []int, inboundPortExclusionList []int) []string { var cmd []string // 1. Create redirection chains @@ -89,7 +89,7 @@ func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundPor cmd = append(cmd, rule) } - // 5. Create dynamic outbound ports exclusion rule + // 5. Create dynamic outbound ports exclusion rules if len(outboundPortExclusionList) > 0 { var portExclusionListStr []string for _, port := range outboundPortExclusionList { @@ -100,5 +100,16 @@ func generateIptablesCommands(outboundIPRangeExclusionList []string, outboundPor cmd = append(cmd, rule) } + // 6. Create dynamic inbound ports exclusion rules + if len(inboundPortExclusionList) > 0 { + var portExclusionListStr []string + for _, port := range inboundPortExclusionList { + portExclusionListStr = append(portExclusionListStr, strconv.Itoa(port)) + } + inboundPortsToExclude := strings.Join(portExclusionListStr, ",") + rule := fmt.Sprintf("iptables -t nat -I PROXY_INBOUND -p tcp --match multiport --dports %s -j RETURN", inboundPortsToExclude) + cmd = append(cmd, rule) + } + return cmd } diff --git a/pkg/injector/iptables_test.go b/pkg/injector/iptables_test.go new file mode 100644 index 0000000000..f207c23bf4 --- /dev/null +++ b/pkg/injector/iptables_test.go @@ -0,0 +1,43 @@ +package injector + +import ( + "testing" + + tassert "github.com/stretchr/testify/assert" +) + +func TestGenerateIptablesCommands(t *testing.T) { + assert := tassert.New(t) + + outboundIPRangeExclusion := []string{"1.1.1.1/32", "2.2.2.2/32"} + outboundPortExclusion := []int{10, 20} + inboundPortExclusion := []int{30, 40} + + actual := generateIptablesCommands(outboundIPRangeExclusion, outboundPortExclusion, inboundPortExclusion) + + expected := []string{ + "iptables -t nat -N PROXY_INBOUND", + "iptables -t nat -N PROXY_IN_REDIRECT", + "iptables -t nat -N PROXY_OUTPUT", + "iptables -t nat -N PROXY_REDIRECT", + "iptables -t nat -A PROXY_REDIRECT -p tcp -j REDIRECT --to-port 15001", + "iptables -t nat -A PROXY_REDIRECT -p tcp --dport 15000 -j ACCEPT", + "iptables -t nat -A OUTPUT -p tcp -j PROXY_OUTPUT", + "iptables -t nat -A PROXY_OUTPUT -m owner --uid-owner 1500 -j RETURN", + "iptables -t nat -A PROXY_OUTPUT -d 127.0.0.1/32 -j RETURN", + "iptables -t nat -A PROXY_OUTPUT -j PROXY_REDIRECT", + "iptables -t nat -A PROXY_IN_REDIRECT -p tcp -j REDIRECT --to-port 15003", + "iptables -t nat -A PREROUTING -p tcp -j PROXY_INBOUND", + "iptables -t nat -A PROXY_INBOUND -p tcp --dport 15010 -j RETURN", + "iptables -t nat -A PROXY_INBOUND -p tcp --dport 15901 -j RETURN", + "iptables -t nat -A PROXY_INBOUND -p tcp --dport 15902 -j RETURN", + "iptables -t nat -A PROXY_INBOUND -p tcp --dport 15903 -j RETURN", + "iptables -t nat -A PROXY_INBOUND -p tcp -j PROXY_IN_REDIRECT", + "iptables -t nat -I PROXY_OUTPUT -d 1.1.1.1/32 -j RETURN", + "iptables -t nat -I PROXY_OUTPUT -d 2.2.2.2/32 -j RETURN", + "iptables -t nat -I PROXY_OUTPUT -p tcp --match multiport --dports 10,20 -j RETURN", + "iptables -t nat -I PROXY_INBOUND -p tcp --match multiport --dports 30,40 -j RETURN", + } + + assert.ElementsMatch(expected, actual) +} diff --git a/pkg/injector/patch.go b/pkg/injector/patch.go index edda9a1aa5..2524c3e994 100644 --- a/pkg/injector/patch.go +++ b/pkg/injector/patch.go @@ -54,13 +54,18 @@ func (wh *mutatingWebhook) createPatch(pod *corev1.Pod, req *admissionv1.Admissi // Create volume for envoy TLS secret pod.Spec.Volumes = append(pod.Spec.Volumes, getVolumeSpec(envoyBootstrapConfigName)...) - podOutboundPortExclusionList, _ := wh.getPodOutboundPortExclusionList(pod, namespace) + // Build outbound port exclusion list + podOutboundPortExclusionList, _ := wh.getPortExclusionListForPod(pod, namespace, outboundPortExclusionListAnnotation) globalOutboundPortExclusionList := wh.configurator.GetOutboundPortExclusionList() - outboundPortExclusionList := mergePortExclusionLists(podOutboundPortExclusionList, globalOutboundPortExclusionList) + // Build inbound port exclusion list + podInboundPortExclusionList, _ := wh.getPortExclusionListForPod(pod, namespace, inboundPortExclusionListAnnotation) + globalInboundPortExclusionList := wh.configurator.GetInboundPortExclusionList() + inboundPortExclusionList := mergePortExclusionLists(podInboundPortExclusionList, globalInboundPortExclusionList) + // Add the Init Container - initContainer := getInitContainerSpec(constants.InitContainerName, wh.configurator, wh.configurator.GetOutboundIPRangeExclusionList(), outboundPortExclusionList, wh.configurator.IsPrivilegedInitContainer()) + initContainer := getInitContainerSpec(constants.InitContainerName, wh.configurator, wh.configurator.GetOutboundIPRangeExclusionList(), outboundPortExclusionList, inboundPortExclusionList, wh.configurator.IsPrivilegedInitContainer()) pod.Spec.InitContainers = append(pod.Spec.InitContainers, initContainer) // Add the Envoy sidecar @@ -102,22 +107,20 @@ func makePatches(req *admissionv1.AdmissionRequest, pod *corev1.Pod) []jsonpatch return admissionResponse.Patches } -func mergePortExclusionLists(podOutboundPortExclusionList, globalOutboundPortExclusionList []int) []int { +func mergePortExclusionLists(podSpecificPortExclusionList, globalPortExclusionList []int) []int { portExclusionListMap := mapset.NewSet() var portExclusionListMerged []int // iterate over the global outbound ports to be excluded - for _, port := range globalOutboundPortExclusionList { - addedToSet := portExclusionListMap.Add(port) - if addedToSet { + for _, port := range globalPortExclusionList { + if addedToSet := portExclusionListMap.Add(port); addedToSet { portExclusionListMerged = append(portExclusionListMerged, port) } } - // iterate over the pod level outbound ports to be excluded - for _, port := range podOutboundPortExclusionList { - addedToSet := portExclusionListMap.Add(port) - if addedToSet { + // iterate over the pod specific ports to be excluded + for _, port := range podSpecificPortExclusionList { + if addedToSet := portExclusionListMap.Add(port); addedToSet { portExclusionListMerged = append(portExclusionListMerged, port) } } diff --git a/pkg/injector/patch_test.go b/pkg/injector/patch_test.go index e91fb9227f..0a24d5ad83 100644 --- a/pkg/injector/patch_test.go +++ b/pkg/injector/patch_test.go @@ -63,6 +63,7 @@ var _ = Describe("Test all patch operations", func() { mockConfigurator.EXPECT().IsPrivilegedInitContainer().Return(false).Times(1) mockConfigurator.EXPECT().GetOutboundIPRangeExclusionList().Return(nil).Times(1) mockConfigurator.EXPECT().GetOutboundPortExclusionList().Return(nil).Times(1) + mockConfigurator.EXPECT().GetInboundPortExclusionList().Return(nil).Times(1) mockConfigurator.EXPECT().GetProxyResources().Return(corev1.ResourceRequirements{}).Times(1) req := &admissionv1.AdmissionRequest{Namespace: namespace} diff --git a/pkg/injector/webhook.go b/pkg/injector/webhook.go index 87977ff2df..467c57506c 100644 --- a/pkg/injector/webhook.go +++ b/pkg/injector/webhook.go @@ -55,8 +55,11 @@ const ( // injectorServiceName is the name of the OSM sidecar injector service injectorServiceName = "osm-injector" - // outboundPortExclusionListAnnotation is the annotation used for outbound port exclusion + // outboundPortExclusionListAnnotation is the annotation used for outbound port exclusions outboundPortExclusionListAnnotation = "openservicemesh.io/outbound-port-exclusion-list" + + // inboundPortExclusionListAnnotation is the annotation used for inbound port exclusions + inboundPortExclusionListAnnotation = "openservicemesh.io/inbound-port-exclusion-list" ) // NewMutatingWebhook starts a new web server handling requests from the injector MutatingWebhookConfiguration @@ -319,18 +322,21 @@ func (wh *mutatingWebhook) mustInject(pod *corev1.Pod, namespace string) (bool, return false, nil } -// getPodOutboundPortExclusionList gets a list of ports to exclude from outbound sidecar interception. +// getPortExclusionListForPod gets a list of ports to exclude from sidecar traffic interception for the given +// pod and annotation kind. +// +// Ports are excluded from sidecar interception when the pod is explicitly annotated with a single or +// comma separate list of ports. // -// Outbound Ports are excluded from sidecar interception when: -// 1. The pod is explicitly annotated with a single or comma separate list of ports +// The kind of exclusion (inbound vs outbound) is determined by the specified annotation. // // The function returns an error when it is unable to determine whether ports need to be excluded from outbound sidecar interception. -func (wh *mutatingWebhook) getPodOutboundPortExclusionList(pod *corev1.Pod, namespace string) ([]int, error) { +func (wh *mutatingWebhook) getPortExclusionListForPod(pod *corev1.Pod, namespace string, annotation string) ([]int, error) { var ports []int // Check if the pod is annotated for outbound port exclusion - ports, err := isAnnotatedForOutboundPortExclusion(pod.Annotations, pod.Kind, fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)) + ports, err := isAnnotatedForPortExclusion(pod.Annotations, annotation, pod.Kind, fmt.Sprintf("%s/%s", pod.Namespace, pod.Name)) if err != nil { - log.Error().Err(err).Msg("Error determining outbound ports for exclusion on pod") + log.Error().Err(err).Msgf("Error determining port exclusions for annotation %s on pod %s/%s", annotation, namespace, pod.Name) return ports, err } @@ -354,19 +360,19 @@ func isAnnotatedForInjection(annotations map[string]string, objectKind string, o return } -func isAnnotatedForOutboundPortExclusion(annotations map[string]string, objectKind string, objectName string) (ports []int, err error) { - outboundPortsToExclude, ok := annotations[outboundPortExclusionListAnnotation] +func isAnnotatedForPortExclusion(annotations map[string]string, portAnnotation string, objectKind string, objectName string) (ports []int, err error) { + portsToExcludeStr, ok := annotations[portAnnotation] if !ok { return ports, err } - log.Trace().Msgf("%s %s has outbound port exclusion annotation: '%s:%s'", objectKind, objectName, outboundPortExclusionListAnnotation, outboundPortsToExclude) - portsToExclude := strings.Split(outboundPortsToExclude, ",") + log.Trace().Msgf("%s %s has port exclusion annotation: '%s:%s'", objectKind, objectName, portAnnotation, portsToExcludeStr) + portsToExclude := strings.Split(portsToExcludeStr, ",") for _, portStr := range portsToExclude { portStr := strings.TrimSpace(portStr) portInt, ok := strconv.Atoi(portStr) if ok != nil || portInt <= 0 { - err = errors.Errorf("Invalid port for key %q: %s", outboundPortExclusionListAnnotation, outboundPortsToExclude) + err = errors.Errorf("Invalid port '%s' specified for annotation '%s'", portStr, portAnnotation) ports = nil return ports, err } diff --git a/pkg/injector/webhook_test.go b/pkg/injector/webhook_test.go index 2de4188ed1..87fb03ee41 100644 --- a/pkg/injector/webhook_test.go +++ b/pkg/injector/webhook_test.go @@ -657,38 +657,56 @@ var _ = Describe("Testing Injector Functions", func() { }) }) -func TestIsAnnotatedForOutboundPortExclusion(t *testing.T) { +func TestIsAnnotatedForPortExclusion(t *testing.T) { assert := tassert.New(t) testCases := []struct { name string - annotation map[string]string + annotations map[string]string + forAnnotation string expectedError error expectedPorts []int }{ { name: "contains outbound port exclusion list annotation", - annotation: map[string]string{outboundPortExclusionListAnnotation: "6060, 7070"}, + annotations: map[string]string{outboundPortExclusionListAnnotation: "6060, 7070"}, + forAnnotation: outboundPortExclusionListAnnotation, expectedError: nil, expectedPorts: []int{6060, 7070}, }, { - name: "does not contains outbound port exclusion list annontation", - annotation: nil, + name: "contains inbound port exclusion list annotation", + annotations: map[string]string{inboundPortExclusionListAnnotation: "6060, 7070"}, + forAnnotation: inboundPortExclusionListAnnotation, expectedError: nil, + expectedPorts: []int{6060, 7070}, + }, + { + name: "does not contains port exclusion list annontation", + annotations: nil, + forAnnotation: "", + expectedError: nil, + expectedPorts: nil, + }, + { + name: "contains outbound port exclusion list annontation but invalid port", + annotations: map[string]string{outboundPortExclusionListAnnotation: "6060, -7070"}, + forAnnotation: outboundPortExclusionListAnnotation, + expectedError: errors.Errorf("Invalid port '%s' specified for annotation '%s'", "-7070", outboundPortExclusionListAnnotation), expectedPorts: nil, }, { - name: "ontains outbound port exclusion list annontation but invalid port", - annotation: map[string]string{outboundPortExclusionListAnnotation: "6060, -7070"}, - expectedError: errors.Errorf("Invalid port for key %q: %s", outboundPortExclusionListAnnotation, "6060, -7070"), + name: "contains inbound port exclusion list annontation but invalid port", + annotations: map[string]string{inboundPortExclusionListAnnotation: "6060, -7070"}, + forAnnotation: inboundPortExclusionListAnnotation, + expectedError: errors.Errorf("Invalid port '%s' specified for annotation '%s'", "-7070", inboundPortExclusionListAnnotation), expectedPorts: nil, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - ports, err := isAnnotatedForOutboundPortExclusion(tc.annotation, "-kind-", "-name-") + ports, err := isAnnotatedForPortExclusion(tc.annotations, tc.forAnnotation, "-kind-", "-name-") if err != nil { assert.EqualError(tc.expectedError, err.Error()) } else { @@ -705,25 +723,43 @@ func TestGetPodOutboundPortExclusionList(t *testing.T) { testCases := []struct { name string podAnnotation map[string]string + forAnnotation string expectedError error expectedPorts []int }{ { name: "contains outbound port exclusion list annotation", podAnnotation: map[string]string{outboundPortExclusionListAnnotation: "6060, 7070"}, + forAnnotation: outboundPortExclusionListAnnotation, expectedError: nil, expectedPorts: []int{6060, 7070}, }, { - name: "does not contains outbound port exclusion list annontation", + name: "contains inbound port exclusion list annotation", + podAnnotation: map[string]string{inboundPortExclusionListAnnotation: "6060, 7070"}, + forAnnotation: inboundPortExclusionListAnnotation, + expectedError: nil, + expectedPorts: []int{6060, 7070}, + }, + { + name: "does not contains any port exclusion list annontation", podAnnotation: nil, + forAnnotation: "", expectedError: nil, expectedPorts: nil, }, { name: "contains outbound port exclusion list annontation but invalid port", podAnnotation: map[string]string{outboundPortExclusionListAnnotation: "6060, -7070"}, - expectedError: errors.Errorf("Invalid port for key %q: %s", outboundPortExclusionListAnnotation, "6060, -7070"), + forAnnotation: outboundPortExclusionListAnnotation, + expectedError: errors.Errorf("Invalid port '%s' specified for annotation '%s'", "-7070", outboundPortExclusionListAnnotation), + expectedPorts: nil, + }, + { + name: "contains inbound port exclusion list annontation but invalid port", + podAnnotation: map[string]string{inboundPortExclusionListAnnotation: "6060, -7070"}, + forAnnotation: inboundPortExclusionListAnnotation, + expectedError: errors.Errorf("Invalid port '%s' specified for annotation '%s'", "-7070", inboundPortExclusionListAnnotation), expectedPorts: nil, }, } @@ -770,7 +806,7 @@ func TestGetPodOutboundPortExclusionList(t *testing.T) { mockKubeController.EXPECT().IsMonitoredNamespace(namespace).Return(true).Times(1) mockKubeController.EXPECT().GetNamespace(namespace).Return(retNs) - ports, err := wh.getPodOutboundPortExclusionList(pod, namespace) + ports, err := wh.getPortExclusionListForPod(pod, namespace, tc.forAnnotation) if err != nil { assert.EqualError(tc.expectedError, err.Error()) } else { diff --git a/pkg/kubernetes/client.go b/pkg/kubernetes/client.go index 717115f745..53a3633103 100644 --- a/pkg/kubernetes/client.go +++ b/pkg/kubernetes/client.go @@ -2,6 +2,7 @@ package kubernetes import ( "reflect" + "strconv" mapset "github.com/deckarep/golang-set" "github.com/pkg/errors" @@ -298,3 +299,15 @@ func (c Client) ListServiceIdentitiesForService(svc service.MeshService) ([]iden } return svcAccounts, nil } + +// IsMetricsEnabled returns true if the pod in the mesh is correctly annotated for prometheus scrapping +func (c Client) IsMetricsEnabled(pod *corev1.Pod) bool { + isScrapingEnabled := false + prometheusScrapeAnnotation, ok := pod.Annotations[constants.PrometheusScrapeAnnotation] + if !ok { + return isScrapingEnabled + } + + isScrapingEnabled, _ = strconv.ParseBool(prometheusScrapeAnnotation) + return isScrapingEnabled +} diff --git a/pkg/kubernetes/client_test.go b/pkg/kubernetes/client_test.go index 553369f6c7..4fb96121d0 100644 --- a/pkg/kubernetes/client_test.go +++ b/pkg/kubernetes/client_test.go @@ -3,12 +3,14 @@ package kubernetes import ( "context" "fmt" + "testing" "time" mapset "github.com/deckarep/golang-set" "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + tassert "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" testclient "k8s.io/client-go/kubernetes/fake" @@ -553,3 +555,79 @@ var _ = Describe("Test Namespace KubeController Methods", func() { }) }) + +func TestIsMetricsEnabled(t *testing.T) { + assert := tassert.New(t) + + testCases := []struct { + name string + addPrometheusAnnotation bool + expectedMetricsScraping bool + scrapingAnnotation string + }{ + { + name: "pod without prometheus scraping annotation", + scrapingAnnotation: "false", + addPrometheusAnnotation: false, + expectedMetricsScraping: false, + }, + { + name: "pod with prometheus scraping annotation set to true", + scrapingAnnotation: "true", + addPrometheusAnnotation: true, + expectedMetricsScraping: true, + }, + { + name: "pod with prometheus scraping annotation set to false", + scrapingAnnotation: "false", + addPrometheusAnnotation: true, + expectedMetricsScraping: false, + }, + { + name: "pod with incorrect prometheus scraping annotation", + scrapingAnnotation: "no", + addPrometheusAnnotation: true, + expectedMetricsScraping: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + kubeClient := testclient.NewSimpleClientset() + stop := make(chan struct{}) + kubeController, err := NewKubernetesController(kubeClient, testMeshName, stop) + assert.Nil(err) + assert.NotNil(kubeController) + + proxyUUID := uuid.New() + namespace := uuid.New().String() + podlabels := map[string]string{ + tests.SelectorKey: tests.SelectorValue, + constants.EnvoyUniqueIDLabelName: proxyUUID.String(), + } + + // Ensure correct presetup + pods, err := kubeClient.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) + assert.Nil(err) + assert.Len(pods.Items, 0) + + newPod1 := tests.NewPodFixture(namespace, fmt.Sprintf("pod-1-%s", proxyUUID), tests.BookstoreServiceAccountName, podlabels) + + if tc.addPrometheusAnnotation { + newPod1.Annotations = map[string]string{ + constants.PrometheusScrapeAnnotation: tc.scrapingAnnotation, + } + } + _, err = kubeClient.CoreV1().Pods(namespace).Create(context.TODO(), &newPod1, metav1.CreateOptions{}) + assert.Nil(err) + + // Ensure correct setup + pods, err = kubeClient.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) + assert.Nil(err) + assert.Len(pods.Items, 1) + + actual := kubeController.IsMetricsEnabled(&newPod1) + assert.Equal(actual, tc.expectedMetricsScraping) + }) + } +} diff --git a/pkg/kubernetes/mock_controller_generated.go b/pkg/kubernetes/mock_controller_generated.go index 0b0c2b5276..eae6311970 100644 --- a/pkg/kubernetes/mock_controller_generated.go +++ b/pkg/kubernetes/mock_controller_generated.go @@ -79,6 +79,20 @@ func (mr *MockControllerMockRecorder) GetService(arg0 interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetService", reflect.TypeOf((*MockController)(nil).GetService), arg0) } +// IsMetricsEnabled mocks base method +func (m *MockController) IsMetricsEnabled(arg0 *v1.Pod) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsMetricsEnabled", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsMetricsEnabled indicates an expected call of IsMetricsEnabled +func (mr *MockControllerMockRecorder) IsMetricsEnabled(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsMetricsEnabled", reflect.TypeOf((*MockController)(nil).IsMetricsEnabled), arg0) +} + // IsMonitoredNamespace mocks base method func (m *MockController) IsMonitoredNamespace(arg0 string) bool { m.ctrl.T.Helper() diff --git a/pkg/kubernetes/types.go b/pkg/kubernetes/types.go index d5227b5e4e..a839b6bfbb 100644 --- a/pkg/kubernetes/types.go +++ b/pkg/kubernetes/types.go @@ -100,4 +100,7 @@ type Controller interface { // GetEndpoints returns the endpoints for a given service, if found GetEndpoints(svc service.MeshService) (*corev1.Endpoints, error) + + // IsMetricsEnabled returns true if the pod in the mesh is correctly annotated for prometheus scrapping + IsMetricsEnabled(*corev1.Pod) bool } diff --git a/tests/e2e/e2e_deployment_client_server_test.go b/tests/e2e/e2e_deployment_client_server_test.go index fd9543abc6..be66092dc2 100644 --- a/tests/e2e/e2e_deployment_client_server_test.go +++ b/tests/e2e/e2e_deployment_client_server_test.go @@ -58,7 +58,8 @@ var _ = OSMDescribe("Test HTTP traffic from N deployment client -> 1 deployment Namespace: destApp, ReplicaCount: int32(replicaSetPerService), Image: "kennethreitz/httpbin", - Ports: []int{80}, + Ports: []int{DefaultUpstreamServicePort}, + Command: HttpbinCmd, }) _, err := Td.CreateServiceAccount(destApp, &svcAccDef) @@ -85,7 +86,7 @@ var _ = OSMDescribe("Test HTTP traffic from N deployment client -> 1 deployment Command: []string{"/bin/bash", "-c", "--"}, Args: []string{"while true; do sleep 30; done;"}, Image: "songrgg/alpine-debug", - Ports: []int{80}, // Can't deploy services with empty/no ports + Ports: []int{DefaultUpstreamServicePort}, // Can't deploy services with empty/no ports }) _, err = Td.CreateServiceAccount(srcClient, &svcAccDef) Expect(err).NotTo(HaveOccurred()) @@ -139,7 +140,7 @@ var _ = OSMDescribe("Test HTTP traffic from N deployment client -> 1 deployment SourcePod: pod.Name, SourceContainer: ns, // container_name == NS for this test - Destination: fmt.Sprintf("%s.%s", destApp, destApp), + Destination: fmt.Sprintf("%s.%s:%d", destApp, destApp, DefaultUpstreamServicePort), }) } } diff --git a/tests/e2e/e2e_fluentbit_deployment_test.go b/tests/e2e/e2e_fluentbit_deployment_test.go index 99a9eb2233..b91f756955 100644 --- a/tests/e2e/e2e_fluentbit_deployment_test.go +++ b/tests/e2e/e2e_fluentbit_deployment_test.go @@ -21,6 +21,9 @@ var _ = OSMDescribe("Test deployment of Fluent Bit sidecar", func() { Context("Fluent Bit deployment", func() { It("Deploys a Fluent Bit sidecar only when enabled", func() { + if Td.DeployOnOpenShift { + Skip("Skipping test: FluentBit not supported on OpenShift") + } // Install OSM with Fluentbit installOpts := Td.GetOSMInstallOpts() installOpts.DeployFluentbit = true diff --git a/tests/e2e/e2e_fluentbit_output_test.go b/tests/e2e/e2e_fluentbit_output_test.go index a552501cde..d3decf8a81 100644 --- a/tests/e2e/e2e_fluentbit_output_test.go +++ b/tests/e2e/e2e_fluentbit_output_test.go @@ -24,6 +24,9 @@ var _ = OSMDescribe("Test deployment of Fluent Bit sidecar", func() { Context("Fluent Bit output", func() { It("Forwards correctly filtered logs to stdout", func() { + if Td.DeployOnOpenShift { + Skip("Skipping test: FluentBit not supported on OpenShift") + } // Install OSM with Fluentbit installOpts := Td.GetOSMInstallOpts() installOpts.DeployFluentbit = true diff --git a/tests/e2e/e2e_helm_install_test.go b/tests/e2e/e2e_helm_install_test.go index dfc63791f2..fe597606b0 100644 --- a/tests/e2e/e2e_helm_install_test.go +++ b/tests/e2e/e2e_helm_install_test.go @@ -34,7 +34,6 @@ var _ = OSMDescribe("Test osm control plane installation with Helm", Expect(spec.Traffic.EnableEgress).To(BeFalse()) Expect(spec.Sidecar.LogLevel).To(Equal("error")) Expect(spec.Observability.EnableDebugServer).To(BeFalse()) - Expect(spec.Observability.PrometheusScraping).To(BeTrue()) Expect(spec.Observability.Tracing.Enable).To(BeFalse()) Expect(spec.Traffic.UseHTTPSIngress).To(BeFalse()) Expect(spec.Certificate.ServiceCertValidityDuration).To(Equal("24h")) diff --git a/tests/e2e/e2e_http_ingress_test.go b/tests/e2e/e2e_http_ingress_test.go index 8a61670e7e..98325b0dd0 100644 --- a/tests/e2e/e2e_http_ingress_test.go +++ b/tests/e2e/e2e_http_ingress_test.go @@ -123,7 +123,7 @@ var _ = OSMDescribe("HTTP ingress", } Td.T.Logf("> REST req failed expectedly: %d", status) return true - }, 5 /*consecutive success threshold*/, 60*time.Second /*timeout*/) + }, 5 /*consecutive success threshold*/, 120*time.Second /*timeout*/) Expect(cond).To(BeTrue()) ing := &v1beta1.Ingress{ @@ -169,7 +169,7 @@ var _ = OSMDescribe("HTTP ingress", } Td.T.Logf("> REST req succeeded: %d", status) return true - }, 5 /*consecutive success threshold*/, 60*time.Second /*timeout*/) + }, 5 /*consecutive success threshold*/, 120*time.Second /*timeout*/) Expect(cond).To(BeTrue()) }) }) diff --git a/tests/e2e/e2e_init_controller_test.go b/tests/e2e/e2e_init_controller_test.go index 0b9937a6bd..b57198a712 100644 --- a/tests/e2e/e2e_init_controller_test.go +++ b/tests/e2e/e2e_init_controller_test.go @@ -15,6 +15,10 @@ var _ = OSMDescribe("Test init-osm-controller functionalities", func() { Context("When osm-controller starts in fresh environment", func() { It("creates default MeshConfig resource", func() { + + if Td.InstType == "NoInstall" { + Skip("Skipping test: NoInstall marked on a test that requires fresh installation") + } instOpts := Td.GetOSMInstallOpts() // Install OSM @@ -26,7 +30,6 @@ var _ = OSMDescribe("Test init-osm-controller functionalities", Expect(meshConfig.Spec.Traffic.EnablePermissiveTrafficPolicyMode).Should(BeFalse()) Expect(meshConfig.Spec.Traffic.EnableEgress).Should(BeFalse()) Expect(meshConfig.Spec.Sidecar.LogLevel).Should(Equal("debug")) - Expect(meshConfig.Spec.Observability.PrometheusScraping).Should(BeTrue()) Expect(meshConfig.Spec.Observability.EnableDebugServer).Should(BeTrue()) Expect(meshConfig.Spec.Observability.Tracing.Enable).Should(BeFalse()) Expect(meshConfig.Spec.Traffic.UseHTTPSIngress).Should(BeFalse()) diff --git a/tests/e2e/e2e_metrics_test.go b/tests/e2e/e2e_metrics_test.go index 8144d4f0e4..1978a7abdc 100644 --- a/tests/e2e/e2e_metrics_test.go +++ b/tests/e2e/e2e_metrics_test.go @@ -55,7 +55,8 @@ var _ = OSMDescribe("Custom WASM metrics between one client pod and one server", Name: "server", Namespace: destNs, Image: "kennethreitz/httpbin", - Ports: []int{80}, + Ports: []int{DefaultUpstreamServicePort}, + Command: HttpbinCmd, }) _, err = Td.CreateServiceAccount(destNs, &svcAccDef) @@ -76,7 +77,7 @@ var _ = OSMDescribe("Custom WASM metrics between one client pod and one server", Command: []string{"/bin/bash", "-c", "--"}, Args: []string{"while true; do sleep 30; done;"}, Image: "songrgg/alpine-debug", - Ports: []int{80}, + Ports: []int{DefaultUpstreamServicePort}, }) _, err = Td.CreateServiceAccount(sourceNs, &svcAccDef) @@ -120,7 +121,7 @@ var _ = OSMDescribe("Custom WASM metrics between one client pod and one server", SourcePod: srcPod.Name, SourceContainer: "client", - Destination: fmt.Sprintf("%s.%s/status/200", dstSvc.Name, dstSvc.Namespace), + Destination: fmt.Sprintf("%s.%s:%d/status/200", dstSvc.Name, dstSvc.Namespace, DefaultUpstreamServicePort), }) if result.Err != nil || result.StatusCode != 200 { diff --git a/tests/e2e/e2e_trafficsplit_recursive_split.go b/tests/e2e/e2e_trafficsplit_recursive_split.go index 178f779179..237f64c409 100644 --- a/tests/e2e/e2e_trafficsplit_recursive_split.go +++ b/tests/e2e/e2e_trafficsplit_recursive_split.go @@ -35,8 +35,6 @@ func testRecursiveTrafficSplit(appProtocol string) { const ( // to name the header we will use to identify the server that replies HTTPHeaderName = "podname" - - serverPort = 80 ) clientAppBaseName := "client" @@ -80,8 +78,9 @@ func testRecursiveTrafficSplit(appProtocol string) { Namespace: serverNamespace, ReplicaCount: int32(serverReplicaSet), Image: "simonkowallik/httpbin", - Ports: []int{serverPort}, + Ports: []int{DefaultUpstreamServicePort}, AppProtocol: appProtocol, + Command: HttpbinCmd, }) // Expose an env variable such as XHTTPBIN_X_POD_NAME: @@ -123,7 +122,7 @@ func testRecursiveTrafficSplit(appProtocol string) { Command: []string{"/bin/bash", "-c", "--"}, Args: []string{"while true; do sleep 30; done;"}, Image: "songrgg/alpine-debug", - Ports: []int{80}, + Ports: []int{DefaultUpstreamServicePort}, }) _, err := Td.CreateServiceAccount(clientApp, &svcAccDef) @@ -201,7 +200,7 @@ func testRecursiveTrafficSplit(appProtocol string) { DestinationNamespace: serverNamespace, DestinationSvcAccountName: dstServer, }, - serverPort, + DefaultUpstreamServicePort, ) // Configs have to be put into same NS as server/destination @@ -234,7 +233,7 @@ func testRecursiveTrafficSplit(appProtocol string) { SourceContainer: ns, // container_name == NS for this test // Targeting the trafficsplit FQDN - Destination: fmt.Sprintf("%s.%s", trafficSplitName, serverNamespace), + Destination: fmt.Sprintf("%s.%s:%d", trafficSplitName, serverNamespace, DefaultUpstreamServicePort), }) } } @@ -295,7 +294,7 @@ func testRecursiveTrafficSplit(appProtocol string) { SourceContainer: pod.Namespace, // We generally code it like so for test purposes // direct traffic target against the specific server service in the server namespace - Destination: fmt.Sprintf("%s.%s", svcNs, serverNamespace), + Destination: fmt.Sprintf("%s.%s:%d", svcNs, serverNamespace, DefaultUpstreamServicePort), }) } } diff --git a/tests/e2e/e2e_trafficsplit_same_sa_test.go b/tests/e2e/e2e_trafficsplit_same_sa_test.go index e855ef27e1..313b378a57 100644 --- a/tests/e2e/e2e_trafficsplit_same_sa_test.go +++ b/tests/e2e/e2e_trafficsplit_same_sa_test.go @@ -80,7 +80,8 @@ var _ = OSMDescribe("Test TrafficSplit where each backend shares the same Servic Namespace: serverNamespace, ReplicaCount: int32(serverReplicaSet), Image: "simonkowallik/httpbin", - Ports: []int{80}, + Ports: []int{DefaultUpstreamServicePort}, + Command: HttpbinCmd, }) // Expose an env variable such as XHTTPBIN_X_POD_NAME: @@ -121,7 +122,7 @@ var _ = OSMDescribe("Test TrafficSplit where each backend shares the same Servic Command: []string{"/bin/bash", "-c", "--"}, Args: []string{"while true; do sleep 30; done;"}, Image: "songrgg/alpine-debug", - Ports: []int{80}, + Ports: []int{DefaultUpstreamServicePort}, }) _, err := Td.CreateServiceAccount(clientApp, &svcAccDef) @@ -164,7 +165,7 @@ var _ = OSMDescribe("Test TrafficSplit where each backend shares the same Servic _, _, trafficSplitService := Td.SimplePodApp(SimplePodAppDef{ Name: trafficSplitName, Namespace: serverNamespace, - Ports: []int{80}, + Ports: []int{DefaultUpstreamServicePort}, }) // Creating trafficsplit service in K8s @@ -211,7 +212,7 @@ var _ = OSMDescribe("Test TrafficSplit where each backend shares the same Servic SourceContainer: ns, // container_name == NS for this test // Targeting the trafficsplit FQDN - Destination: fmt.Sprintf("%s.%s", trafficSplitName, serverNamespace), + Destination: fmt.Sprintf("%s.%s:%d", trafficSplitName, serverNamespace, DefaultUpstreamServicePort), }) } } diff --git a/tests/e2e/e2e_trafficsplit_test.go b/tests/e2e/e2e_trafficsplit_test.go index d047001949..ae484a73bb 100644 --- a/tests/e2e/e2e_trafficsplit_test.go +++ b/tests/e2e/e2e_trafficsplit_test.go @@ -35,8 +35,6 @@ func testTrafficSplit(appProtocol string) { const ( // to name the header we will use to identify the server that replies HTTPHeaderName = "podname" - - serverPort = 80 ) clientAppBaseName := "client" @@ -85,8 +83,9 @@ func testTrafficSplit(appProtocol string) { Namespace: serverNamespace, ReplicaCount: int32(serverReplicaSet), Image: "simonkowallik/httpbin", - Ports: []int{serverPort}, + Ports: []int{DefaultUpstreamServicePort}, AppProtocol: appProtocol, + Command: HttpbinCmd, }) // Expose an env variable such as XHTTPBIN_X_POD_NAME: @@ -128,7 +127,7 @@ func testTrafficSplit(appProtocol string) { Command: []string{"/bin/bash", "-c", "--"}, Args: []string{"while true; do sleep 30; done;"}, Image: "songrgg/alpine-debug", - Ports: []int{80}, + Ports: []int{DefaultUpstreamServicePort}, }) _, err := Td.CreateServiceAccount(clientApp, &svcAccDef) @@ -184,7 +183,7 @@ func testTrafficSplit(appProtocol string) { DestinationNamespace: serverNamespace, DestinationSvcAccountName: dstServer, }, - serverPort, + DefaultUpstreamServicePort, ) // Configs have to be put into a monitored NS @@ -203,7 +202,7 @@ func testTrafficSplit(appProtocol string) { _, _, trafficSplitService := Td.SimplePodApp(SimplePodAppDef{ Name: trafficSplitName, Namespace: serverNamespace, - Ports: []int{80}, + Ports: []int{DefaultUpstreamServicePort}, AppProtocol: appProtocol, }) @@ -253,7 +252,7 @@ func testTrafficSplit(appProtocol string) { SourceContainer: ns, // container_name == NS for this test // Targeting the trafficsplit FQDN - Destination: fmt.Sprintf("%s.%s", trafficSplitName, serverNamespace), + Destination: fmt.Sprintf("%s.%s:%d", trafficSplitName, serverNamespace, DefaultUpstreamServicePort), }) } } @@ -314,7 +313,7 @@ func testTrafficSplit(appProtocol string) { SourceContainer: pod.Namespace, // We generally code it like so for test purposes // direct traffic target against the specific server service in the server namespace - Destination: fmt.Sprintf("%s.%s", svcNs, serverNamespace), + Destination: fmt.Sprintf("%s.%s:%d", svcNs, serverNamespace, DefaultUpstreamServicePort), }) } } diff --git a/tests/e2e/e2e_upgrade_test.go b/tests/e2e/e2e_upgrade_test.go index e601b9bf7e..ef234576d6 100644 --- a/tests/e2e/e2e_upgrade_test.go +++ b/tests/e2e/e2e_upgrade_test.go @@ -59,6 +59,8 @@ var _ = OSMDescribe("Upgrade from latest", "OpenServiceMesh": map[string]interface{}{ "deployPrometheus": true, "deployJaeger": false, + // Init container must be privileged if an OpenShift cluster is being used + "enablePrivilegedInitContainer": Td.DeployOnOpenShift, // Reduce CPU so CI (capped at 2 CPU) can handle standing // up the new control plane before tearing the old one diff --git a/tests/framework/common.go b/tests/framework/common.go index 87195215ac..07f28b5685 100644 --- a/tests/framework/common.go +++ b/tests/framework/common.go @@ -115,6 +115,8 @@ var ( defaultOSMLogLevel = "trace" // Test folder base default value testFolderBase = "/tmp" + // default enable privileged init container + defaultEnablePrivilegedInitContainer = false ) // OSMDescribeInfo is a struct to represent the Tier and Bucket of a given e2e test @@ -150,6 +152,15 @@ const ( RegistrySecretName = "acr-creds" ) +const ( + // DefaultUpstreamServicePort is the default port on which the server apps listen for connections from client apps. + // Note: Port 80 should not be used because it does not work on OpenShift. + DefaultUpstreamServicePort = 14001 +) + +// HttpbinCmd is the command to be used for httpbin applications in e2es +var HttpbinCmd = []string{"gunicorn", "-b", fmt.Sprintf("0.0.0.0:%d", DefaultUpstreamServicePort), "httpbin:app", "-k", "gevent"} + // Verifies the instType string flag option is a valid enum type func verifyValidInstallType(t InstallType) error { switch t { @@ -440,10 +451,16 @@ type InstallOSMOpts struct { EnableDebugServer bool SetOverrides []string + + EnablePrivilegedInitContainer bool } // GetOSMInstallOpts initializes install options for OSM func (td *OsmTestData) GetOSMInstallOpts() InstallOSMOpts { + enablePrivilegedInitContainer := defaultEnablePrivilegedInitContainer + if td.DeployOnOpenShift { + enablePrivilegedInitContainer = true + } return InstallOSMOpts{ ControlPlaneNS: td.OsmNamespace, CertManager: defaultCertManager, @@ -467,6 +484,8 @@ func (td *OsmTestData) GetOSMInstallOpts() InstallOSMOpts { OSMLogLevel: defaultOSMLogLevel, EnableDebugServer: defaultEnableDebugServer, SetOverrides: []string{}, + + EnablePrivilegedInitContainer: enablePrivilegedInitContainer, } } @@ -535,6 +554,25 @@ func (td *OsmTestData) LoadImagesToKind(imageNames []string) error { return nil } +func setMeshConfigToDefault(instOpts InstallOSMOpts, meshConfig *v1alpha1.MeshConfig) (defaultConfig *v1alpha1.MeshConfig) { + meshConfig.Spec.Traffic.EnableEgress = instOpts.EgressEnabled + meshConfig.Spec.Traffic.EnablePermissiveTrafficPolicyMode = instOpts.EnablePermissiveMode + meshConfig.Spec.Traffic.OutboundPortExclusionList = []int{} + meshConfig.Spec.Traffic.OutboundIPRangeExclusionList = []string{} + + meshConfig.Spec.Observability.EnableDebugServer = instOpts.EnableDebugServer + + meshConfig.Spec.Sidecar.Resources = corev1.ResourceRequirements{} + meshConfig.Spec.Sidecar.EnablePrivilegedInitContainer = false + meshConfig.Spec.Sidecar.LogLevel = instOpts.EnvoyLogLevel + meshConfig.Spec.Sidecar.MaxDataPlaneConnections = 0 + meshConfig.Spec.Sidecar.ConfigResyncInterval = "0s" + + meshConfig.Spec.Certificate.ServiceCertValidityDuration = "24h" + + return meshConfig +} + // InstallOSM installs OSM. The behavior of this function is dependant on // installType and instOpts func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error { @@ -552,21 +590,12 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error { meshConfig, _ := Td.GetMeshConfig(Td.OsmNamespace) - // This resets supported dynamic configs expected by the caller - meshConfig.Spec.Traffic.EnableEgress = instOpts.EgressEnabled - meshConfig, err := Td.UpdateOSMConfig(meshConfig) - if err != nil { - return err - } - meshConfig.Spec.Traffic.EnablePermissiveTrafficPolicyMode = instOpts.EnablePermissiveMode - meshConfig, err = Td.UpdateOSMConfig(meshConfig) - if err != nil { - return err - } - meshConfig.Spec.Observability.EnableDebugServer = instOpts.EnableDebugServer - if _, err = Td.UpdateOSMConfig(meshConfig); err != nil { + meshConfig = setMeshConfigToDefault(instOpts, meshConfig) + + if _, err := Td.UpdateOSMConfig(meshConfig); err != nil { return err } + return nil } @@ -598,6 +627,7 @@ func (td *OsmTestData) InstallOSM(instOpts InstallOSMOpts) error { fmt.Sprintf("OpenServiceMesh.deployPrometheus=%v", instOpts.DeployPrometheus), fmt.Sprintf("OpenServiceMesh.deployJaeger=%v", instOpts.DeployJaeger), fmt.Sprintf("OpenServiceMesh.enableFluentbit=%v", instOpts.DeployFluentbit), + fmt.Sprintf("OpenServiceMesh.enablePrivilegedInitContainer=%v", instOpts.EnablePrivilegedInitContainer), ) switch instOpts.CertManager { @@ -715,29 +745,39 @@ func (td *OsmTestData) LoadOSMImagesIntoKind() error { func (td *OsmTestData) installVault(instOpts InstallOSMOpts) error { td.T.Log("Installing Vault") + + appName := "vault" replicas := int32(1) terminationGracePeriodSeconds := int64(10) + + serviceAccountDefinition := Td.SimpleServiceAccount(appName, td.OsmNamespace) + svcAccount, err := Td.CreateServiceAccount(serviceAccountDefinition.Namespace, &serviceAccountDefinition) + if err != nil { + return errors.Wrap(err, "failed to create vault service account") + } + vaultDep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "vault", + Name: appName, Labels: map[string]string{ - "app": "vault", + "app": appName, }, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicas, Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ - "app": "vault", + "app": appName, }, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "app": "vault", + "app": appName, }, }, Spec: corev1.PodSpec{ + ServiceAccountName: svcAccount.Name, TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, Containers: []corev1.Container{ { @@ -832,22 +872,22 @@ tail /dev/random; }, }, } - _, err := td.Client.AppsV1().Deployments(instOpts.ControlPlaneNS).Create(context.TODO(), vaultDep, metav1.CreateOptions{}) + _, err = td.Client.AppsV1().Deployments(instOpts.ControlPlaneNS).Create(context.TODO(), vaultDep, metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "failed to create vault deployment") } vaultSvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: "vault", + Name: appName, Labels: map[string]string{ - "app": "vault", + "app": appName, }, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, Selector: map[string]string{ - "app": "vault", + "app": appName, }, Ports: []corev1.ServicePort{ { @@ -1615,7 +1655,6 @@ func (td *OsmTestData) AddOpenShiftSCC(scc, serviceAccount, namespace string) er if !td.DeployOnOpenShift { return errors.Errorf("Tests are not configured for OpenShift. Try again with -deployOnOpenShift=true") } - //oc adm policy add-scc-to-user privileged -z bookbuyer -n "$BOOKBUYER_NAMESPACE" args := []string{"adm", "policy", "add-scc-to-user", scc, "-z", serviceAccount, "-n", namespace} stdout, stderr, err := td.RunLocal("oc", args) if err != nil { diff --git a/tests/framework/common_apps.go b/tests/framework/common_apps.go index 1ce17fc61d..04074b794f 100644 --- a/tests/framework/common_apps.go +++ b/tests/framework/common_apps.go @@ -150,15 +150,10 @@ type SimplePodAppDef struct { AppProtocol string } -// SimplePodApp creates returns a set of k8s typed definitions for a pod-based k8s definition. +// SimplePodApp returns a set of k8s typed definitions for a pod-based k8s definition. // Includes Pod, Service and ServiceAccount types func (td *OsmTestData) SimplePodApp(def SimplePodAppDef) (corev1.ServiceAccount, corev1.Pod, corev1.Service) { - serviceAccountDefinition := corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: def.Name, - Namespace: def.Namespace, - }, - } + serviceAccountDefinition := Td.SimpleServiceAccount(def.Name, def.Namespace) podDefinition := corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ @@ -251,6 +246,17 @@ func (td *OsmTestData) SimplePodApp(def SimplePodAppDef) (corev1.ServiceAccount, return serviceAccountDefinition, podDefinition, serviceDefinition } +// SimpleServiceAccount returns a k8s typed definition for a service account. +func (td *OsmTestData) SimpleServiceAccount(name string, namespace string) corev1.ServiceAccount { + serviceAccountDefinition := corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + return serviceAccountDefinition +} + // getKubernetesServerVersionNumber returns the version number in chunks, ex. v1.19.3 => [1, 19, 3] func (td *OsmTestData) getKubernetesServerVersionNumber() ([]int, error) { version, err := td.Client.Discovery().ServerVersion() diff --git a/tests/scenarios/traffic_split_with_apex_service_test.go b/tests/scenarios/traffic_split_with_apex_service_test.go index 2c1eb19b13..86144f4502 100644 --- a/tests/scenarios/traffic_split_with_apex_service_test.go +++ b/tests/scenarios/traffic_split_with_apex_service_test.go @@ -14,6 +14,7 @@ import ( "k8s.io/client-go/kubernetes" testclient "k8s.io/client-go/kubernetes/fake" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" configFake "github.com/openservicemesh/osm/pkg/gen/client/config/clientset/versioned/fake" "github.com/openservicemesh/osm/pkg/catalog" @@ -56,6 +57,11 @@ var _ = Describe(``+ // ---[ Get the config from rds.NewResponse() ]------- mockConfigurator.EXPECT().IsPermissiveTrafficPolicyMode().Return(false).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + EnableEgressPolicy: false, + }).AnyTimes() + resources, err := rds.NewResponse(meshCatalog, proxy, nil, mockConfigurator, nil, proxyRegistry) It("did not return an error", func() { Expect(err).ToNot(HaveOccurred()) diff --git a/tests/scenarios/traffic_split_with_zero_weight_test.go b/tests/scenarios/traffic_split_with_zero_weight_test.go index c4ea2d9d34..70abdd4efa 100644 --- a/tests/scenarios/traffic_split_with_zero_weight_test.go +++ b/tests/scenarios/traffic_split_with_zero_weight_test.go @@ -17,6 +17,7 @@ import ( "k8s.io/client-go/kubernetes" testclient "k8s.io/client-go/kubernetes/fake" + "github.com/openservicemesh/osm/pkg/apis/config/v1alpha1" "github.com/openservicemesh/osm/pkg/catalog" "github.com/openservicemesh/osm/pkg/certificate" "github.com/openservicemesh/osm/pkg/configurator" @@ -234,6 +235,10 @@ func TestRDSRespose(t *testing.T) { mockConfigurator.EXPECT().IsPermissiveTrafficPolicyMode().Return(false).AnyTimes() + mockConfigurator.EXPECT().GetFeatureFlags().Return(v1alpha1.FeatureFlags{ + EnableWASMStats: false, + }).AnyTimes() + proxyRegistry := registry.NewProxyRegistry(registry.ExplicitProxyServiceMapper(func(*envoy.Proxy) ([]service.MeshService, error) { return []service.MeshService{tests.BookstoreV1Service}, nil }))