From 6628582360a49ec1972c8498679d155dd3cd64cd Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Mon, 14 Aug 2023 17:12:17 +0200 Subject: [PATCH 01/16] salt: Add support in CSC to manage a new configuration for Shell UI on the WorkloadPlane --- buildchain/buildchain/salt_tree.py | 1 + .../cluster_and_service_configuration.rst | 77 +++++++++++++++++++ .../workloadplane-shell-ui-config.yaml.j2 | 21 +++++ .../files/metalk8s-ui-deployment.yaml.j2 | 6 ++ .../addons/ui/deployed/ui-configuration.sls | 32 ++++++++ salt/metalk8s/addons/ui/deployed/ui.sls.in | 24 ++++++ 6 files changed, 161 insertions(+) create mode 100644 salt/metalk8s/addons/ui/config/workloadplane-shell-ui-config.yaml.j2 diff --git a/buildchain/buildchain/salt_tree.py b/buildchain/buildchain/salt_tree.py index 8349429508..ed66a6f260 100644 --- a/buildchain/buildchain/salt_tree.py +++ b/buildchain/buildchain/salt_tree.py @@ -375,6 +375,7 @@ def task(self) -> types.TaskDict: Path("salt/metalk8s/addons/ui/deployed/init.sls"), Path("salt/metalk8s/addons/ui/config/metalk8s-shell-ui-config.yaml.j2"), Path("salt/metalk8s/addons/ui/config/metalk8s-ui-config.yaml.j2"), + Path("salt/metalk8s/addons/ui/config/workloadplane-shell-ui-config.yaml.j2"), targets.TemplateFile( task_name="salt/metalk8s/addons/ui/config/deployed-ui-apps.yaml.j2", source=constants.ROOT.joinpath( diff --git a/docs/operation/cluster_and_service_configuration.rst b/docs/operation/cluster_and_service_configuration.rst index d64c62ce35..646f9ec0a0 100644 --- a/docs/operation/cluster_and_service_configuration.rst +++ b/docs/operation/cluster_and_service_configuration.rst @@ -132,6 +132,17 @@ The default Shell UI configuration values are specified below: See :ref:`csc-shell-ui-config-customization` to override these defaults. +Shell UI Workload Plane Default Configuration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Shell UI has a different configuration for the workload plane. + +The default Shell UI workload plane configuration values are specified below: + +.. literalinclude:: ../../salt/metalk8s/addons/ui/config/workloadplane-shell-ui-config.yaml.j2 + :lines: 3- + + Service Configurations Customization ------------------------------------ @@ -988,6 +999,72 @@ To change the UI navigation menu entries, follow these steps: salt-master-bootstrap -- salt-run state.sls \\ metalk8s.addons.ui.deployed saltenv=metalk8s-|version| + +MetalK8s Shell UI Workloadplane Configuration Customization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Default configuration for MetalK8s Shell UI workloadplane can be overridden by +editing its Cluster and Service ConfigMap ``workloadplane-shell-ui-config`` in +namespace ``metalk8s-ui`` under the key ``data.config\.yaml``. + +Changing UI Menu Entries +"""""""""""""""""""""""" + +To change the UI navigation menu entries on the workloadplane, follow these +steps: + +#. Edit the ConfigMap: + +.. code-block:: shell + + root@bootstrap $ kubectl --kubeconfig /etc/kubernetes/admin.conf \ + edit configmap -n metalk8s-ui \ + workloadplane-shell-ui-config + +#. Edit the ``navbar`` field. As an example, we add an entry to + the ``main`` section (there is also a ``subLogin`` section): + +.. code-block:: yaml + + apiVersion: v1 + kind: ConfigMap + data: + config.yaml: |- + apiVersion: addons.metalk8s.scality.com/v1alpha1 + kind: WorkloadplaneShellUIConfig + spec: + deployedApps: + - kind: ModuleFederatedAppKind, + name: appname, + version: x.y.z, + url: https://app.url, + appHistoryBasePath: "" + config: + # [...] + navbar: + # [...] + main: + # [...] + kind: ModuleFederatedAppKind + view: ViewToFederate + # Alternatively for a non federated app + isExternal: true + label: + en: Documentation + fr: Documentation + url: https://13.48.197.10:8443/docs/ + +#. Apply your changes by running: + +.. parsed-literal:: + + root\@bootstrap $ kubectl exec -n kube-system -c salt-master \\ + --kubeconfig /etc/kubernetes/admin.conf \\ + salt-master-bootstrap -- salt-run state.sls \\ + metalk8s.addons.ui.deployed saltenv=metalk8s-|version| + + + Replicas Count Customization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/salt/metalk8s/addons/ui/config/workloadplane-shell-ui-config.yaml.j2 b/salt/metalk8s/addons/ui/config/workloadplane-shell-ui-config.yaml.j2 new file mode 100644 index 0000000000..e487d2786f --- /dev/null +++ b/salt/metalk8s/addons/ui/config/workloadplane-shell-ui-config.yaml.j2 @@ -0,0 +1,21 @@ +#!jinja|yaml + +# Defaults for configuration of Shell-UI Workload Plane +apiVersion: addons.metalk8s.scality.com/v1alpha1 +kind: WorkloadPlaneShellUIConfig +spec: + deployedApps: [] + config: + discoveryUrl: /shell/deployed-ui-apps.json + logo: + light: /brand/assets/logo-light.svg + dark: /brand/assets/logo-dark.svg + darkRebrand: /brand/assets/logo-darkRebrand.svg + favicon: /brand/favicon-metalk8s.svg + canChangeLanguage: false + canChangeTheme: false + productName: MetalK8s + themes: + darkRebrand: + logoPath: /brand/assets/logo-darkRebrand.svg + diff --git a/salt/metalk8s/addons/ui/deployed/files/metalk8s-ui-deployment.yaml.j2 b/salt/metalk8s/addons/ui/deployed/files/metalk8s-ui-deployment.yaml.j2 index 9d2e9b957b..0df3d7f63f 100644 --- a/salt/metalk8s/addons/ui/deployed/files/metalk8s-ui-deployment.yaml.j2 +++ b/salt/metalk8s/addons/ui/deployed/files/metalk8s-ui-deployment.yaml.j2 @@ -67,6 +67,9 @@ spec: - name: deployed-ui-apps-volume mountPath: /etc/metalk8s/ui/deployed-ui-apps readOnly: true + - name: workloadplane-shell-ui-config-volume + mountPath: /usr/share/nginx/html/wp + readOnly: true volumes: - name: metalk8s-ui-volume configMap: @@ -77,3 +80,6 @@ spec: - name: deployed-ui-apps-volume configMap: name: deployed-ui-apps-generated + - name: workloadplane-shell-ui-config-volume + configMap: + name: workloadplane-shell-ui-config-generated diff --git a/salt/metalk8s/addons/ui/deployed/ui-configuration.sls b/salt/metalk8s/addons/ui/deployed/ui-configuration.sls index a54541b29f..0d7d2fa829 100644 --- a/salt/metalk8s/addons/ui/deployed/ui-configuration.sls +++ b/salt/metalk8s/addons/ui/deployed/ui-configuration.sls @@ -25,6 +25,38 @@ include: ) %} +{%- set workloadplane_shell_ui_config = salt.metalk8s_kubernetes.get_object( + kind='ConfigMap', + apiVersion='v1', + namespace='metalk8s-ui', + name='workloadplane-shell-ui-config', + ) +%} + +{%- if workloadplane_shell_ui_config is none %} + +Create workloadplane-shell-ui-config ConfigMap: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: workloadplane-shell-ui-config + namespace: metalk8s-ui + data: + config.yaml: |- + apiVersion: addons.metalk8s.scality.com/v1alpha1 + kind: WorkloadPlaneShellUIConfig + spec: {} + + +{%- else %} + +workloadplane-shell-ui-config ConfigMap already exist: + test.succeed_without_changes: [] + +{%- endif %} + {%- if deployed_ui_apps is none %} Create deployed-ui-apps ConfigMap: diff --git a/salt/metalk8s/addons/ui/deployed/ui.sls.in b/salt/metalk8s/addons/ui/deployed/ui.sls.in index 7639f35543..e69fbf90bf 100644 --- a/salt/metalk8s/addons/ui/deployed/ui.sls.in +++ b/salt/metalk8s/addons/ui/deployed/ui.sls.in @@ -31,6 +31,16 @@ include: ) %} +{%- set workloadplane_shell_ui_defaults = salt.slsutil.renderer( + 'salt://metalk8s/addons/ui/config/workloadplane-shell-ui-config.yaml.j2', saltenv=saltenv + ) +%} + +{%- set workloadplane_shell_ui_config = salt.metalk8s_service_configuration.get_service_conf( + 'metalk8s-ui', 'workloadplane-shell-ui-config', workloadplane_shell_ui_defaults + ) +%} + {%- set stripped_base_path = metalk8s_ui_config.spec.basePath.strip('/') %} {%- set normalized_base_path = '/' ~ stripped_base_path %} @@ -137,6 +147,20 @@ Create shell-ui ConfigMap: config.json: |- {{ metalk8s_shell_ui_config.spec | tojson }} +Create workloadplane-shell-ui-config-generated ConfigMap: + metalk8s_kubernetes.object_present: + - manifest: + apiVersion: v1 + kind: ConfigMap + metadata: + name: workloadplane-shell-ui-config-generated + namespace: metalk8s-ui + data: + config.json: |- + {{ workloadplane_shell_ui_config.spec.config | tojson }} + deployed-ui-apps.json: |- + {{ workloadplane_shell_ui_config.spec.deployedApps | tojson }} + Create deployed-ui-apps ConfigMap: metalk8s_kubernetes.object_present: - manifest: From a9cd71efd62f3092bb14bf8e869c57a433743a31 Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Mon, 14 Aug 2023 17:17:36 +0200 Subject: [PATCH 02/16] Changelog: Add an entry mentionning the new CSC config for shell-ui on the Workloadplane --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e63475f460..5105cb3b58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ### Enhancements +- Add support in CSC to manage a new configuration for Shell UI on the WorkloadPlane + (PR[#4124](https://github.com/scality/metalk8s/pull/4124)) + - Bump Kubernetes version to [1.26.5](https://github.com/kubernetes/kubernetes/releases/tag/v1.26.5) (PR[#4074](https://github.com/scality/metalk8s/pull/4074)) From 8945aaef50f14f468ac9f67a347388e3583a4309 Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Mon, 14 Aug 2023 17:42:37 +0200 Subject: [PATCH 03/16] salt: Define workloadplane ingress default backend to be Shell UI --- CHANGELOG.md | 3 + buildchain/buildchain/image.py | 2 - buildchain/buildchain/versions.py | 5 - charts/ingress-nginx-control-plane.yaml | 2 - charts/ingress-nginx.yaml | 17 +-- .../addons/nginx-ingress/deployed/chart.sls | 125 +----------------- tests/post/features/sanity.feature | 1 - 7 files changed, 6 insertions(+), 149 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5105cb3b58..234a9ba334 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ - Add support in CSC to manage a new configuration for Shell UI on the WorkloadPlane (PR[#4124](https://github.com/scality/metalk8s/pull/4124)) +- Define workloadplane ingress default backend to be Shell UI + (PR[#4124](https://github.com/scality/metalk8s/pull/4124)) + - Bump Kubernetes version to [1.26.5](https://github.com/kubernetes/kubernetes/releases/tag/v1.26.5) (PR[#4074](https://github.com/scality/metalk8s/pull/4074)) diff --git a/buildchain/buildchain/image.py b/buildchain/buildchain/image.py index e3e67ff512..893674dcc6 100644 --- a/buildchain/buildchain/image.py +++ b/buildchain/buildchain/image.py @@ -179,7 +179,6 @@ def _local_image(name: str, **kwargs: Any) -> targets.LocalImage: "fluent-bit", ], constants.GOOGLE_REPOSITORY: [ - "nginx-ingress-defaultbackend-amd64", "pause", ], constants.GRAFANA_REPOSITORY: [ @@ -224,7 +223,6 @@ def _local_image(name: str, **kwargs: Any) -> targets.LocalImage: "calico-node": "node", "calico-kube-controllers": "kube-controllers", "nginx-ingress-controller": "controller", - "nginx-ingress-defaultbackend-amd64": "defaultbackend-amd64", } SAVE_AS: Dict[str, List[targets.ImageSaveFormat]] = { diff --git a/buildchain/buildchain/versions.py b/buildchain/buildchain/versions.py index c13ca7036e..6eccae3510 100644 --- a/buildchain/buildchain/versions.py +++ b/buildchain/buildchain/versions.py @@ -179,11 +179,6 @@ def _version_prefix(version: str, prefix: str = "v") -> str: version="v1.8.1", digest="sha256:e5c4824e7375fcf2a393e1c03c293b69759af37a9ca6abdb91b13d78a93da8bd", ), - Image( - name="nginx-ingress-defaultbackend-amd64", - version="1.5", - digest="sha256:4dc5e07c8ca4e23bddb3153737d7b8c556e5fb2f29c4558b7cd6e6df99c512c7", - ), Image( name="node-exporter", version="v1.6.0", diff --git a/charts/ingress-nginx-control-plane.yaml b/charts/ingress-nginx-control-plane.yaml index d0a15e13f0..955c358e89 100644 --- a/charts/ingress-nginx-control-plane.yaml +++ b/charts/ingress-nginx-control-plane.yaml @@ -3,8 +3,6 @@ controller: digest: null repository: '{%- endraw -%}{{ build_image_name(\"nginx-ingress-controller\", False) }}{%- raw -%}' - defaultBackendService: 'metalk8s-ingress/nginx-ingress-default-backend' - electionID: ingress-control-plane-controller-leader ingressClassResource: diff --git a/charts/ingress-nginx.yaml b/charts/ingress-nginx.yaml index b048bd057d..24029b34b5 100644 --- a/charts/ingress-nginx.yaml +++ b/charts/ingress-nginx.yaml @@ -28,6 +28,7 @@ controller: type: ClusterIP extraArgs: + default-backend-service: metalk8s-ui/metalk8s-ui default-ssl-certificate: "metalk8s-ingress/ingress-workload-plane-default-certificate" metrics-per-host: false @@ -39,18 +40,4 @@ controller: metalk8s.scality.com/monitor: '' defaultBackend: - enabled: true - - image: - repository: '{%- endraw -%}{{ build_image_name(\"nginx-ingress-defaultbackend-amd64\", False) }}{%- raw -%}' - - tolerations: - - key: "node-role.kubernetes.io/bootstrap" - operator: "Exists" - effect: "NoSchedule" - - key: "node-role.kubernetes.io/infra" - operator: "Exists" - effect: "NoSchedule" - - nodeSelector: - node-role.kubernetes.io/infra: '' + enabled: false diff --git a/salt/metalk8s/addons/nginx-ingress/deployed/chart.sls b/salt/metalk8s/addons/nginx-ingress/deployed/chart.sls index 44b55216ec..068230b7d3 100644 --- a/salt/metalk8s/addons/nginx-ingress/deployed/chart.sls +++ b/salt/metalk8s/addons/nginx-ingress/deployed/chart.sls @@ -23,22 +23,6 @@ metadata: name: ingress-nginx namespace: metalk8s-ingress --- -apiVersion: v1 -automountServiceAccountToken: true -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: default-backend - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/managed-by: salt - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: metalk8s - app.kubernetes.io/version: 1.8.1 - helm.sh/chart: ingress-nginx-4.7.1 - heritage: metalk8s - name: ingress-nginx-backend - namespace: metalk8s-ingress ---- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -324,33 +308,6 @@ spec: app.kubernetes.io/name: ingress-nginx type: ClusterIP --- -apiVersion: v1 -kind: Service -metadata: - labels: - app.kubernetes.io/component: default-backend - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/managed-by: salt - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: metalk8s - app.kubernetes.io/version: 1.8.1 - helm.sh/chart: ingress-nginx-4.7.1 - heritage: metalk8s - name: ingress-nginx-defaultbackend - namespace: metalk8s-ingress -spec: - ports: - - appProtocol: http - name: http - port: 80 - protocol: TCP - targetPort: http - selector: - app.kubernetes.io/component: default-backend - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - type: ClusterIP ---- apiVersion: apps/v1 kind: DaemonSet metadata: @@ -388,13 +345,13 @@ spec: containers: - args: - /nginx-ingress-controller - - --default-backend-service=$(POD_NAMESPACE)/ingress-nginx-defaultbackend - --publish-service=$(POD_NAMESPACE)/ingress-nginx-controller - --election-id=ingress-nginx-leader - --controller-class=k8s.io/ingress-nginx - --ingress-class=nginx - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - --watch-ingress-without-class=true + - --default-backend-service=metalk8s-ui/metalk8s-ui - --default-ssl-certificate=metalk8s-ingress/ingress-workload-plane-default-certificate - --metrics-per-host=false env: @@ -474,86 +431,6 @@ spec: key: node-role.kubernetes.io/infra operator: Exists --- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/component: default-backend - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/managed-by: salt - app.kubernetes.io/name: ingress-nginx - app.kubernetes.io/part-of: metalk8s - app.kubernetes.io/version: 1.8.1 - helm.sh/chart: ingress-nginx-4.7.1 - heritage: metalk8s - name: ingress-nginx-defaultbackend - namespace: metalk8s-ingress -spec: - minReadySeconds: 0 - replicas: 1 - revisionHistoryLimit: 10 - selector: - matchLabels: - app.kubernetes.io/component: default-backend - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - template: - metadata: - labels: - app.kubernetes.io/component: default-backend - app.kubernetes.io/instance: ingress-nginx - app.kubernetes.io/name: ingress-nginx - spec: - containers: - - image: '{%- endraw -%}{{ build_image_name("nginx-ingress-defaultbackend-amd64", - False) }}{%- raw -%}:1.5' - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - name: ingress-nginx-default-backend - ports: - - containerPort: 8080 - name: http - protocol: TCP - readinessProbe: - failureThreshold: 6 - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 0 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 5 - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 65534 - nodeSelector: - kubernetes.io/os: linux - node-role.kubernetes.io/infra: '' - serviceAccountName: ingress-nginx-backend - terminationGracePeriodSeconds: 60 - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/bootstrap - operator: Exists - - effect: NoSchedule - key: node-role.kubernetes.io/infra - operator: Exists ---- apiVersion: networking.k8s.io/v1 kind: IngressClass metadata: diff --git a/tests/post/features/sanity.feature b/tests/post/features/sanity.feature index 4c93502294..2f3f0d4e42 100644 --- a/tests/post/features/sanity.feature +++ b/tests/post/features/sanity.feature @@ -36,7 +36,6 @@ Feature: Cluster Sanity Checks | kube-system | coredns | | kube-system | metalk8s-operator-controller-manager | | kube-system | storage-operator-controller-manager | - | metalk8s-ingress | ingress-nginx-defaultbackend | | metalk8s-monitoring | prometheus-adapter | | metalk8s-monitoring | prometheus-operator-grafana | | metalk8s-monitoring | prometheus-operator-kube-state-metrics | From f2864526972b948a17c0e9efaf145368046cebb7 Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Thu, 17 Aug 2023 14:54:34 +0200 Subject: [PATCH 04/16] tests: refactor tests according to new default backend behaviour --- tests/post/features/ingress.feature | 19 +++++++++++-------- tests/post/steps/test_ingress.py | 9 +++++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/post/features/ingress.feature b/tests/post/features/ingress.feature index 38194e28e3..e135b2d981 100644 --- a/tests/post/features/ingress.feature +++ b/tests/post/features/ingress.feature @@ -4,13 +4,15 @@ Feature: Ingress Given the Kubernetes API is available And pods with label 'app.kubernetes.io/name=ingress-nginx' are 'Ready' When we perform an HTTP request on port 80 on a workload-plane IP - Then the server returns 404 'Not Found' + Then the server returns 200 'OK' + And the server should respond with shell-ui index Scenario: Access HTTPS services Given the Kubernetes API is available And pods with label 'app.kubernetes.io/name=ingress-nginx' are 'Ready' When we perform an HTTPS request on port 443 on a workload-plane IP - Then the server returns 404 'Not Found' + Then the server returns 200 'OK' + And the server should respond with shell-ui index Scenario: Create new Ingress object (without class) Given the Kubernetes API is available @@ -31,7 +33,8 @@ Feature: Ingress And pods with label 'app.kubernetes.io/name=ingress-nginx' are 'Ready' When we create a 'metalk8s-test-3' Ingress with class 'invalid-class' on path '/_metalk8s-test-3' on 'repositories' service on 'http' in 'kube-system' namespace And we perform an HTTPS request on path '/_metalk8s-test-3' on port 443 on a workload-plane IP - Then the server returns 404 'Not Found' + Then the server returns 200 'OK' + And the server should respond with shell-ui index Scenario: Access HTTP services on control-plane IP Given the Kubernetes API is available @@ -59,7 +62,7 @@ Feature: Ingress And we trigger a rollout restart of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' And we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete Then the '{wp_ingress_vips}' IPs are spread on nodes - And an HTTP request on port 80 on '{wp_ingress_vips}' IPs returns 404 'Not Found' + And an HTTP request on port 80 on '{wp_ingress_vips}' IPs returns 200 'OK' Scenario: Workload Plane Ingress VIPs reconfiguration Given the Kubernetes API is available @@ -72,7 +75,7 @@ Feature: Ingress And we trigger a rollout restart of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' And we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete Then the '{wp_ingress_second_pool}' IPs are spread on nodes - And an HTTP request on port 80 on '{wp_ingress_second_pool}' IPs returns 404 'Not Found' + And an HTTP request on port 80 on '{wp_ingress_second_pool}' IPs returns 200 'OK' And the '{wp_ingress_first_pool}' IPs are no longer available on nodes And an HTTP request on port 80 on '{wp_ingress_first_pool}' IPs should not return @@ -87,9 +90,9 @@ Feature: Ingress And we trigger a rollout restart of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' And we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete Then the '{wp_ingress_first_pool}' IPs are spread on nodes - And an HTTP request on port 80 on '{wp_ingress_first_pool}' IPs returns 404 'Not Found' + And an HTTP request on port 80 on '{wp_ingress_first_pool}' IPs returns 200 'OK' And the '{wp_ingress_second_pool}' IPs are spread on nodes - And an HTTP request on port 80 on '{wp_ingress_second_pool}' IPs returns 404 'Not Found' + And an HTTP request on port 80 on '{wp_ingress_second_pool}' IPs returns 200 'OK' Scenario: Failover of Workload Plane Ingress VIPs Given the Kubernetes API is available @@ -100,7 +103,7 @@ Feature: Ingress When we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete And we stop the node 'node-1' Workload Plane Ingress Then the '{wp_ingress_vips}' IPs should no longer sit on the node 'node-1' - And an HTTP request on port 80 on '{wp_ingress_vips}' IPs returns 404 'Not Found' + And an HTTP request on port 80 on '{wp_ingress_vips}' IPs returns 200 'OK' @authentication Scenario: Failover of Control Plane Ingress VIP diff --git a/tests/post/steps/test_ingress.py b/tests/post/steps/test_ingress.py index 883c08c1de..8e8acedb02 100644 --- a/tests/post/steps/test_ingress.py +++ b/tests/post/steps/test_ingress.py @@ -569,6 +569,15 @@ def server_returns(host, context, status_code, reason): assert response.status_code == int(status_code) assert response.reason == reason +@then( + parsers.re(r"the server should respond with shell-ui index"), + converters=dict(status_code=int), +) +def shell_ui_returns(host, context): + response = context.get("response") + assert response is not None + assert "window.shellUIRemoteEntryUrl = \"/shell/remoteEntry.js" in response.text + @then("the server should not respond") def server_does_not_respond(host, context): From 24005e58632229644f35aa98d8e611ae80e7c003 Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Thu, 17 Aug 2023 15:12:30 +0200 Subject: [PATCH 05/16] tests: add the should not respond with shell-ui check --- tests/post/features/ingress.feature | 7 +++++++ tests/post/steps/test_ingress.py | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/tests/post/features/ingress.feature b/tests/post/features/ingress.feature index e135b2d981..157673bb78 100644 --- a/tests/post/features/ingress.feature +++ b/tests/post/features/ingress.feature @@ -20,6 +20,7 @@ Feature: Ingress When we create a 'metalk8s-test-1' Ingress on path '/_metalk8s-test-1' on 'repositories' service on 'http' in 'kube-system' namespace And we perform an HTTPS request on path '/_metalk8s-test-1' on port 443 on a workload-plane IP Then the server returns 200 'OK' + And the server should not respond with shell-ui index Scenario: Create new Ingress object (nginx class) Given the Kubernetes API is available @@ -27,6 +28,7 @@ Feature: Ingress When we create a 'metalk8s-test-2' Ingress with class 'nginx' on path '/_metalk8s-test-2' on 'repositories' service on 'http' in 'kube-system' namespace And we perform an HTTPS request on path '/_metalk8s-test-2' on port 443 on a workload-plane IP Then the server returns 200 'OK' + And the server should not respond with shell-ui index Scenario: Create new Ingress object (invalid class) Given the Kubernetes API is available @@ -63,6 +65,7 @@ Feature: Ingress And we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete Then the '{wp_ingress_vips}' IPs are spread on nodes And an HTTP request on port 80 on '{wp_ingress_vips}' IPs returns 200 'OK' + And the server should not respond with shell-ui index Scenario: Workload Plane Ingress VIPs reconfiguration Given the Kubernetes API is available @@ -76,6 +79,7 @@ Feature: Ingress And we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete Then the '{wp_ingress_second_pool}' IPs are spread on nodes And an HTTP request on port 80 on '{wp_ingress_second_pool}' IPs returns 200 'OK' + And the server should not respond with shell-ui index And the '{wp_ingress_first_pool}' IPs are no longer available on nodes And an HTTP request on port 80 on '{wp_ingress_first_pool}' IPs should not return @@ -91,8 +95,10 @@ Feature: Ingress And we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete Then the '{wp_ingress_first_pool}' IPs are spread on nodes And an HTTP request on port 80 on '{wp_ingress_first_pool}' IPs returns 200 'OK' + And the server should not respond with shell-ui index And the '{wp_ingress_second_pool}' IPs are spread on nodes And an HTTP request on port 80 on '{wp_ingress_second_pool}' IPs returns 200 'OK' + And the server should not respond with shell-ui index Scenario: Failover of Workload Plane Ingress VIPs Given the Kubernetes API is available @@ -104,6 +110,7 @@ Feature: Ingress And we stop the node 'node-1' Workload Plane Ingress Then the '{wp_ingress_vips}' IPs should no longer sit on the node 'node-1' And an HTTP request on port 80 on '{wp_ingress_vips}' IPs returns 200 'OK' + And the server should not respond with shell-ui index @authentication Scenario: Failover of Control Plane Ingress VIP diff --git a/tests/post/steps/test_ingress.py b/tests/post/steps/test_ingress.py index 8e8acedb02..d29d404c54 100644 --- a/tests/post/steps/test_ingress.py +++ b/tests/post/steps/test_ingress.py @@ -578,6 +578,15 @@ def shell_ui_returns(host, context): assert response is not None assert "window.shellUIRemoteEntryUrl = \"/shell/remoteEntry.js" in response.text +@then( + parsers.re(r"the server should not respond with shell-ui index"), + converters=dict(status_code=int), +) +def shell_ui_not_returns(host, context): + response = context.get("response") + assert response is not None + assert "window.shellUIRemoteEntryUrl = \"/shell/remoteEntry.js" not in response.text + @then("the server should not respond") def server_does_not_respond(host, context): From f79c8357380d43c4fdb8c363551cf350efbcee2b Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Thu, 17 Aug 2023 15:18:42 +0200 Subject: [PATCH 06/16] tests: add the should not respond with shell-ui check for cases when WP is exposed on the CP --- tests/post/features/ingress.feature | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/post/features/ingress.feature b/tests/post/features/ingress.feature index 157673bb78..4e9a8ace99 100644 --- a/tests/post/features/ingress.feature +++ b/tests/post/features/ingress.feature @@ -53,7 +53,8 @@ Feature: Ingress And we trigger a rollout restart of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' And we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete Then an HTTP request on port 80 on a workload-plane IP should not return - And an HTTP request on port 80 on a control-plane IP returns 404 'Not Found' + And an HTTP request on port 80 on a control-plane IP returns 200 'OK' + And the server should respond with shell-ui index Scenario: Expose Workload Plane Ingress on some VIPs Given the Kubernetes API is available From bcf201bae77728feefd862716f2e338b35075384 Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Thu, 17 Aug 2023 15:22:38 +0200 Subject: [PATCH 07/16] Remove unneccessary converter --- tests/post/steps/test_ingress.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/post/steps/test_ingress.py b/tests/post/steps/test_ingress.py index d29d404c54..028f3e2809 100644 --- a/tests/post/steps/test_ingress.py +++ b/tests/post/steps/test_ingress.py @@ -571,7 +571,6 @@ def server_returns(host, context, status_code, reason): @then( parsers.re(r"the server should respond with shell-ui index"), - converters=dict(status_code=int), ) def shell_ui_returns(host, context): response = context.get("response") @@ -580,7 +579,6 @@ def shell_ui_returns(host, context): @then( parsers.re(r"the server should not respond with shell-ui index"), - converters=dict(status_code=int), ) def shell_ui_not_returns(host, context): response = context.get("response") From 9724b93abb04ef483f2463bc6c7f86173bb3cba1 Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Thu, 17 Aug 2023 15:41:17 +0200 Subject: [PATCH 08/16] Ensure default backend service and deployment are deleted during upgrade --- .../addons/nginx-ingress/deployed/init.sls | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/salt/metalk8s/addons/nginx-ingress/deployed/init.sls b/salt/metalk8s/addons/nginx-ingress/deployed/init.sls index 9a57e0c9ce..7064702c1e 100644 --- a/salt/metalk8s/addons/nginx-ingress/deployed/init.sls +++ b/salt/metalk8s/addons/nginx-ingress/deployed/init.sls @@ -6,3 +6,25 @@ include: - .dashboards - .service-configuration - .config-map + +{#- In MetalK8s 126.0 we remove the default backend for the workloadplane, + let's remove old objects #} +{#- This logic can be removed in `development/127.0` #} + +Delete old service for the default backend: + metalk8s_kubernetes.object_absent: + - apiVersion: v1 + - kind: Service + - name: ingress-nginx-defaultbackend + - namespace: metalk8s-ingress + - require: + - sls: metalk8s.addons.nginx-ingress.deployed.chart + +Delete old deployment for the default backend: + metalk8s_kubernetes.object_absent: + - apiVersion: v1 + - kind: Deployment + - name: ingress-nginx-defaultbackend + - namespace: metalk8s-ingress + - require: + - sls: metalk8s.addons.nginx-ingress.deployed.chart From 4af40f5e161e7a7afad58469f3704ce8027e5a05 Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Thu, 17 Aug 2023 15:43:52 +0200 Subject: [PATCH 09/16] Ensure default backend service account is deleted during upgrade --- salt/metalk8s/addons/nginx-ingress/deployed/init.sls | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/salt/metalk8s/addons/nginx-ingress/deployed/init.sls b/salt/metalk8s/addons/nginx-ingress/deployed/init.sls index 7064702c1e..0cca277a3c 100644 --- a/salt/metalk8s/addons/nginx-ingress/deployed/init.sls +++ b/salt/metalk8s/addons/nginx-ingress/deployed/init.sls @@ -28,3 +28,12 @@ Delete old deployment for the default backend: - namespace: metalk8s-ingress - require: - sls: metalk8s.addons.nginx-ingress.deployed.chart + +Delete old serviceaccount for the default backend: + metalk8s_kubernetes.object_absent: + - apiVersion: v1 + - kind: ServiceAccount + - name: ingress-nginx-backend + - namespace: metalk8s-ingress + - require: + - sls: metalk8s.addons.nginx-ingress.deployed.chart From d54f0e384162b6a695eec59342a779f072593554 Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Thu, 17 Aug 2023 15:49:49 +0200 Subject: [PATCH 10/16] Ensure default backend deployment is deleted during upgrade --- salt/metalk8s/addons/nginx-ingress/deployed/init.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/metalk8s/addons/nginx-ingress/deployed/init.sls b/salt/metalk8s/addons/nginx-ingress/deployed/init.sls index 0cca277a3c..e07a283f60 100644 --- a/salt/metalk8s/addons/nginx-ingress/deployed/init.sls +++ b/salt/metalk8s/addons/nginx-ingress/deployed/init.sls @@ -22,7 +22,7 @@ Delete old service for the default backend: Delete old deployment for the default backend: metalk8s_kubernetes.object_absent: - - apiVersion: v1 + - apiVersion: apps/v1 - kind: Deployment - name: ingress-nginx-defaultbackend - namespace: metalk8s-ingress From b224440910b169baa4f554ad1e19848eef270376 Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Thu, 17 Aug 2023 15:50:17 +0200 Subject: [PATCH 11/16] Fix test useless parser --- tests/post/steps/test_ingress.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/post/steps/test_ingress.py b/tests/post/steps/test_ingress.py index 028f3e2809..eab84fc0b4 100644 --- a/tests/post/steps/test_ingress.py +++ b/tests/post/steps/test_ingress.py @@ -569,17 +569,13 @@ def server_returns(host, context, status_code, reason): assert response.status_code == int(status_code) assert response.reason == reason -@then( - parsers.re(r"the server should respond with shell-ui index"), -) +@then("the server should respond with shell-ui index") def shell_ui_returns(host, context): response = context.get("response") assert response is not None assert "window.shellUIRemoteEntryUrl = \"/shell/remoteEntry.js" in response.text -@then( - parsers.re(r"the server should not respond with shell-ui index"), -) +@then("the server should not respond with shell-ui index") def shell_ui_not_returns(host, context): response = context.get("response") assert response is not None From 25ff21afa0f75bd4bbe0e04268cfc164f5d056c3 Mon Sep 17 00:00:00 2001 From: JBWatenbergScality <75977494+JBWatenbergScality@users.noreply.github.com> Date: Wed, 6 Sep 2023 11:25:27 +0200 Subject: [PATCH 12/16] Fix linter issues --- tests/post/steps/test_ingress.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/post/steps/test_ingress.py b/tests/post/steps/test_ingress.py index eab84fc0b4..46cf5e5434 100644 --- a/tests/post/steps/test_ingress.py +++ b/tests/post/steps/test_ingress.py @@ -569,17 +569,19 @@ def server_returns(host, context, status_code, reason): assert response.status_code == int(status_code) assert response.reason == reason + @then("the server should respond with shell-ui index") def shell_ui_returns(host, context): response = context.get("response") assert response is not None - assert "window.shellUIRemoteEntryUrl = \"/shell/remoteEntry.js" in response.text + assert "window.shellUIRemoteEntryUrl = "/shell/remoteEntry.js" in response.text + @then("the server should not respond with shell-ui index") def shell_ui_not_returns(host, context): response = context.get("response") assert response is not None - assert "window.shellUIRemoteEntryUrl = \"/shell/remoteEntry.js" not in response.text + assert "window.shellUIRemoteEntryUrl = "/shell/remoteEntry.js" not in response.text @then("the server should not respond") From 53329db05f1382de17b04e7de87a4d8ebb764566 Mon Sep 17 00:00:00 2001 From: JBWatenbergScality <75977494+JBWatenbergScality@users.noreply.github.com> Date: Wed, 6 Sep 2023 11:31:21 +0200 Subject: [PATCH 13/16] Fix linter issues --- tests/post/steps/test_ingress.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/post/steps/test_ingress.py b/tests/post/steps/test_ingress.py index 46cf5e5434..343e86561f 100644 --- a/tests/post/steps/test_ingress.py +++ b/tests/post/steps/test_ingress.py @@ -574,14 +574,14 @@ def server_returns(host, context, status_code, reason): def shell_ui_returns(host, context): response = context.get("response") assert response is not None - assert "window.shellUIRemoteEntryUrl = "/shell/remoteEntry.js" in response.text + assert 'window.shellUIRemoteEntryUrl = "/shell/remoteEntry.js' in response.text @then("the server should not respond with shell-ui index") def shell_ui_not_returns(host, context): response = context.get("response") assert response is not None - assert "window.shellUIRemoteEntryUrl = "/shell/remoteEntry.js" not in response.text + assert 'window.shellUIRemoteEntryUrl = "/shell/remoteEntry.js' not in response.text @then("the server should not respond") From e49c4cbf3a1650c915b8d3164c0629da70e01d5a Mon Sep 17 00:00:00 2001 From: Patrick Ear Date: Thu, 7 Sep 2023 15:18:37 +0200 Subject: [PATCH 14/16] tests: fix e2e test by adding contexr --- tests/post/steps/test_ingress.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/post/steps/test_ingress.py b/tests/post/steps/test_ingress.py index 343e86561f..e453327df9 100644 --- a/tests/post/steps/test_ingress.py +++ b/tests/post/steps/test_ingress.py @@ -641,6 +641,7 @@ def server_request_returns_multiple_ips( ) try: response = requests.get(endpoint, verify=False) + context["response"] = response except Exception as exc: raise AssertionError(f"Unable to reach ingress on '{endpoint}': {exc}") From 087f2c4f5bbc8633605cfeea2bb9000195599eb3 Mon Sep 17 00:00:00 2001 From: Jean-Baptiste WATENBERG Date: Fri, 8 Sep 2023 16:56:33 +0200 Subject: [PATCH 15/16] Fix end to end tests --- tests/post/features/ingress.feature | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/post/features/ingress.feature b/tests/post/features/ingress.feature index 4e9a8ace99..a9258597a3 100644 --- a/tests/post/features/ingress.feature +++ b/tests/post/features/ingress.feature @@ -66,7 +66,7 @@ Feature: Ingress And we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete Then the '{wp_ingress_vips}' IPs are spread on nodes And an HTTP request on port 80 on '{wp_ingress_vips}' IPs returns 200 'OK' - And the server should not respond with shell-ui index + And the server should respond with shell-ui index Scenario: Workload Plane Ingress VIPs reconfiguration Given the Kubernetes API is available @@ -80,7 +80,7 @@ Feature: Ingress And we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete Then the '{wp_ingress_second_pool}' IPs are spread on nodes And an HTTP request on port 80 on '{wp_ingress_second_pool}' IPs returns 200 'OK' - And the server should not respond with shell-ui index + And the server should respond with shell-ui index And the '{wp_ingress_first_pool}' IPs are no longer available on nodes And an HTTP request on port 80 on '{wp_ingress_first_pool}' IPs should not return @@ -96,10 +96,10 @@ Feature: Ingress And we wait for the rollout of 'daemonset/ingress-nginx-controller' in namespace 'metalk8s-ingress' to complete Then the '{wp_ingress_first_pool}' IPs are spread on nodes And an HTTP request on port 80 on '{wp_ingress_first_pool}' IPs returns 200 'OK' - And the server should not respond with shell-ui index + And the server should respond with shell-ui index And the '{wp_ingress_second_pool}' IPs are spread on nodes And an HTTP request on port 80 on '{wp_ingress_second_pool}' IPs returns 200 'OK' - And the server should not respond with shell-ui index + And the server should respond with shell-ui index Scenario: Failover of Workload Plane Ingress VIPs Given the Kubernetes API is available @@ -111,7 +111,7 @@ Feature: Ingress And we stop the node 'node-1' Workload Plane Ingress Then the '{wp_ingress_vips}' IPs should no longer sit on the node 'node-1' And an HTTP request on port 80 on '{wp_ingress_vips}' IPs returns 200 'OK' - And the server should not respond with shell-ui index + And the server should respond with shell-ui index @authentication Scenario: Failover of Control Plane Ingress VIP From 75a7a7e874e34748e9dd5c778e7c1aa8cfeb594e Mon Sep 17 00:00:00 2001 From: Jean-Baptiste WATENBERG Date: Fri, 8 Sep 2023 18:09:28 +0200 Subject: [PATCH 16/16] ci