From 4cb7cb1729288e2a26825082b43313b0e2cb2099 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 3 Dec 2024 19:35:55 +0100 Subject: [PATCH] Refactor Keycloak --- hack/e2e.sh | 28 ++++++-- .../templates/dashboard-resourcemap.yaml | 13 ++++ .../apps/tenant/templates/keycloakgroups.yaml | 4 ++ .../apps/tenant/templates/kubeconfig.yaml | 24 ++----- .../core/platform/bundles/distro-full.yaml | 7 -- .../core/platform/bundles/distro-hosted.yaml | 7 -- packages/core/platform/bundles/paas-full.yaml | 20 ++++-- .../core/platform/bundles/paas-hosted.yaml | 21 ++++-- .../templates/configure-kk.yaml | 69 +------------------ 9 files changed, 78 insertions(+), 115 deletions(-) create mode 100644 packages/apps/tenant/templates/dashboard-resourcemap.yaml diff --git a/hack/e2e.sh b/hack/e2e.sh index 31fa3c97..493ef303 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -124,6 +124,12 @@ machine: op: create cluster: + apiServer: + extraArgs: + oidc-issuer-url: "https://keycloak.example.org/realms/cozy" + oidc-client-id: "kubernetes" + oidc-username-claim: "preferred_username" + oidc-groups-claim: "groups" network: cni: name: none @@ -182,7 +188,8 @@ timeout 60 sh -c 'until nc -nzv 192.168.123.11 50000 && nc -nzv 192.168.123.12 5 timeout 10 sh -c 'until talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11; do sleep 1; done' # Wait for etcd -timeout 180 sh -c 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1 | grep "rpc error"; do sleep 1; done' +timeout 180 sh -c 'until timeout -s 9 2 talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1; do sleep 1; done' +timeout 60 sh -c 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1 | grep "rpc error"; do sleep 1; done' rm -f kubeconfig talosctl kubeconfig kubeconfig -e 192.168.123.10 -n 192.168.123.10 @@ -203,6 +210,8 @@ data: ipv4-pod-gateway: "10.244.0.1" ipv4-svc-cidr: "10.96.0.0/16" ipv4-join-cidr: "100.64.0.0/16" + root-host: example.org + api-server-endpoint: https://192.168.123.10:6443 EOT # @@ -287,13 +296,13 @@ spec: avoidBuggyIPs: false EOT -kubectl patch -n tenant-root hr/tenant-root --type=merge -p '{"spec":{ "values":{ +kubectl patch -n tenant-root tenants.apps.cozystack.io root --type=merge -p '{"spec":{ "host": "example.org", "ingress": true, "monitoring": true, "etcd": true, "isolated": true -}}}' +}}' # Wait for HelmRelease be created timeout 60 sh -c 'until kubectl get hr -n tenant-root etcd ingress monitoring tenant-root; do sleep 1; done' @@ -301,9 +310,9 @@ timeout 60 sh -c 'until kubectl get hr -n tenant-root etcd ingress monitoring te # Wait for HelmReleases be installed kubectl wait --timeout=2m --for=condition=ready -n tenant-root hr etcd ingress monitoring tenant-root -kubectl patch -n tenant-root hr/ingress --type=merge -p '{"spec":{ "values":{ +kubectl patch -n tenant-root ingresses.apps.cozystack.io ingress --type=merge -p '{"spec":{ "dashboard": true -}}}' +}}' # Wait for nginx-ingress-controller timeout 60 sh -c 'until kubectl get deploy -n tenant-root root-ingress-controller; do sleep 1; done' @@ -326,3 +335,12 @@ ip=$(kubectl get svc -n tenant-root root-ingress-controller -o jsonpath='{.statu # Check Grafana curl -sS -k "https://$ip" -H 'Host: grafana.example.org' | grep Found + + +# Test OIDC +kubectl patch -n cozy-system cm/cozystack --type=merge -p '{"data":{ + "oidc-enabled": "true" +}}' + +timeout 60 sh -c 'until kubectl get hr -n cozy-keycloak keycloak keycloak-configure keycloak-operator; do sleep 1; done' +kubectl wait --timeout=10m --for=condition=ready -n cozy-keycloak hr keycloak keycloak-configure keycloak-operator diff --git a/packages/apps/tenant/templates/dashboard-resourcemap.yaml b/packages/apps/tenant/templates/dashboard-resourcemap.yaml new file mode 100644 index 00000000..9020b8a0 --- /dev/null +++ b/packages/apps/tenant/templates/dashboard-resourcemap.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "tenant.name" . }}-dashboard-resources + namespace: {{ .Release.namespace }} +rules: +- apiGroups: + - "" + resources: + - secrets + resourceNames: + - kubeconfig-{{ include "tenant.name" . }} + verbs: ["get", "list", "watch"] diff --git a/packages/apps/tenant/templates/keycloakgroups.yaml b/packages/apps/tenant/templates/keycloakgroups.yaml index e4b956e2..cd759eab 100644 --- a/packages/apps/tenant/templates/keycloakgroups.yaml +++ b/packages/apps/tenant/templates/keycloakgroups.yaml @@ -1,3 +1,6 @@ +{{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }} +{{- $oidcEnabled := index $cozyConfig.data "oidc-enabled" }} +{{- if $oidcEnabled }} apiVersion: v1.edp.epam.com/v1 kind: KeycloakRealmGroup metadata: @@ -47,3 +50,4 @@ spec: realmRef: name: keycloakrealm-cozy kind: ClusterKeycloakRealm +{{- end }} diff --git a/packages/apps/tenant/templates/kubeconfig.yaml b/packages/apps/tenant/templates/kubeconfig.yaml index 59092e95..601945f0 100644 --- a/packages/apps/tenant/templates/kubeconfig.yaml +++ b/packages/apps/tenant/templates/kubeconfig.yaml @@ -1,28 +1,13 @@ {{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }} {{- $host := index $cozyConfig.data "root-host" }} -{{- $apiServerAdress := index $cozyConfig.data "api-server-adress" }} {{- $k8sClientSecret := lookup "v1" "Secret" "cozy-keycloak" "k8s-client" }} + +{{- if $k8sClientSecret }} +{{- $apiServerEndpoint := index $cozyConfig.data "api-server-endpoint" }} {{- $k8sClient := index $k8sClientSecret.data "client-secret-key" | b64dec }} {{- $rootSaConfigMap := lookup "v1" "ConfigMap" "kube-system" "kube-root-ca.crt" }} {{- $k8sCa := index $rootSaConfigMap.data "ca.crt" | b64enc }} - -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "tenant.name" . }}-dashboard-resources - namespace: {{ .Release.namespace }} -rules: -- apiGroups: - - "" - resources: - - secrets - resourceNames: - - kubeconfig-{{ include "tenant.name" . }} - verbs: ["get", "list", "watch"] - - --- - apiVersion: v1 kind: Secret metadata: @@ -33,7 +18,7 @@ stringData: apiVersion: v1 clusters: - cluster: - server: https://{{ $apiServerAdress }}:6443 + server: {{ $apiServerEndpoint }} certificate-authority-data: {{ $k8sCa }} name: cluster contexts: @@ -57,3 +42,4 @@ stringData: - --skip-open-browser - --grant-type=password command: kubectl +{{- end }} diff --git a/packages/core/platform/bundles/distro-full.yaml b/packages/core/platform/bundles/distro-full.yaml index 0d06471c..26ef8aac 100644 --- a/packages/core/platform/bundles/distro-full.yaml +++ b/packages/core/platform/bundles/distro-full.yaml @@ -188,10 +188,3 @@ releases: namespace: cozy-keycloak optional: true dependsOn: [keycloak] - -- name: keycloak-configure - releaseName: keycloak-configure - chart: cozy-keycloak-configure - namespace: cozy-keycloak - optional: true - dependsOn: [keycloak-operator] diff --git a/packages/core/platform/bundles/distro-hosted.yaml b/packages/core/platform/bundles/distro-hosted.yaml index 57573553..7b138a8d 100644 --- a/packages/core/platform/bundles/distro-hosted.yaml +++ b/packages/core/platform/bundles/distro-hosted.yaml @@ -138,10 +138,3 @@ releases: namespace: cozy-keycloak optional: true dependsOn: [keycloak] - -- name: keycloak-configure - releaseName: keycloak-configure - chart: cozy-keycloak-configure - namespace: cozy-keycloak - optional: true - dependsOn: [keycloak-operator] diff --git a/packages/core/platform/bundles/paas-full.yaml b/packages/core/platform/bundles/paas-full.yaml index 6a252b9d..784058e3 100644 --- a/packages/core/platform/bundles/paas-full.yaml +++ b/packages/core/platform/bundles/paas-full.yaml @@ -1,8 +1,13 @@ {{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }} +{{- $oidcEnabled := index $cozyConfig.data "oidc-enabled" }} {{- $host := index $cozyConfig.data "root-host" }} {{- if not $host }} {{- fail "ERROR need root-host in cozystack ConfigMap" }} {{- end }} +{{- $apiServerEndpoint := index $cozyConfig.data "api-server-endpoint" }} +{{- if not $apiServerEndpoint }} +{{- fail "ERROR need api-server-endpoint in cozystack ConfigMap" }} +{{- end }} releases: - name: fluxcd-operator @@ -205,10 +210,6 @@ releases: chart: cozy-dashboard namespace: cozy-dashboard dependsOn: [cilium,kubeovn,keycloak-configure] - valuesFrom: - - kind: ConfigMap - name: kubeapps-auth-config - valuesKey: values.yaml {{- if .Capabilities.APIVersions.Has "source.toolkit.fluxcd.io/v1" }} {{- with (lookup "source.toolkit.fluxcd.io/v1" "HelmRepository" "cozy-public" "").items }} values: @@ -222,6 +223,15 @@ releases: {{- end }} {{- end }} {{- end }} + {{- if $oidcEnabled }} + dependsOn: [keycloak-configure] + valuesFrom: + - kind: ConfigMap + name: kubeapps-auth-config + valuesKey: values.yaml + {{- else }} + dependsOn: [] + {{- end }} - name: kamaji releaseName: kamaji @@ -257,6 +267,7 @@ releases: optional: true dependsOn: [cilium,kubeovn] +{{- if $oidcEnabled }} - name: keycloak releaseName: keycloak chart: cozy-keycloak @@ -274,3 +285,4 @@ releases: chart: cozy-keycloak-configure namespace: cozy-keycloak dependsOn: [keycloak-operator] +{{- end }} diff --git a/packages/core/platform/bundles/paas-hosted.yaml b/packages/core/platform/bundles/paas-hosted.yaml index 08d269ea..4fd31f6b 100644 --- a/packages/core/platform/bundles/paas-hosted.yaml +++ b/packages/core/platform/bundles/paas-hosted.yaml @@ -1,8 +1,13 @@ {{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }} +{{- $oidcEnabled := index $cozyConfig.data "oidc-enabled" }} {{- $host := index $cozyConfig.data "root-host" }} {{- if not $host }} {{- fail "ERROR need root-host in cozystack ConfigMap" }} {{- end }} +{{- $apiServerEndpoint := index $cozyConfig.data "api-server-endpoint" }} +{{- if not $apiServerEndpoint }} +{{- fail "ERROR need api-server-endpoint in cozystack ConfigMap" }} +{{- end }} releases: - name: fluxcd-operator @@ -134,11 +139,6 @@ releases: releaseName: dashboard chart: cozy-dashboard namespace: cozy-dashboard - dependsOn: [keycloak-configure] - valuesFrom: - - kind: ConfigMap - name: kubeapps-auth-config - valuesKey: values.yaml {{- if .Capabilities.APIVersions.Has "source.toolkit.fluxcd.io/v1" }} {{- with (lookup "source.toolkit.fluxcd.io/v1" "HelmRepository" "cozy-public" "").items }} values: @@ -153,7 +153,17 @@ releases: {{- end }} {{- end }} {{- end }} + {{- if $oidcEnabled }} + dependsOn: [keycloak-configure] + valuesFrom: + - kind: ConfigMap + name: kubeapps-auth-config + valuesKey: values.yaml + {{- else }} + dependsOn: [] + {{- end }} +{{- if $oidcEnabled }} - name: keycloak releaseName: keycloak chart: cozy-keycloak @@ -171,3 +181,4 @@ releases: chart: cozy-keycloak-configure namespace: cozy-keycloak dependsOn: [keycloak-operator] +{{- end }} diff --git a/packages/system/keycloak-configure/templates/configure-kk.yaml b/packages/system/keycloak-configure/templates/configure-kk.yaml index b5530697..7ad67846 100644 --- a/packages/system/keycloak-configure/templates/configure-kk.yaml +++ b/packages/system/keycloak-configure/templates/configure-kk.yaml @@ -1,6 +1,5 @@ {{- $cozyConfig := lookup "v1" "ConfigMap" "cozy-system" "cozystack" }} {{- $host := index $cozyConfig.data "root-host" }} -{{- $apiServerAdress := index $cozyConfig.data "api-server-adress" }} {{- $k8sClient := randAlphaNum 32 -}} {{- $kubeappsClient := randAlphaNum 32 -}} {{- $rootSaConfigMap := lookup "v1" "ConfigMap" "kube-system" "kube-root-ca.crt" }} @@ -82,7 +81,7 @@ spec: clientId: kubernetes directAccess: true public: false - webUrl: https://{{ $apiServerAdress }}/oauth2/callback + webUrl: https://localhost:8000/oauth2/callback webOrigins: - /* defaultClientScopes: @@ -175,69 +174,3 @@ data: - --cookie-secure=false - --scope=openid email groups - --oidc-issuer-url=https://keycloak.{{ $host }}/realms/cozy - ---- - -apiVersion: v1.edp.epam.com/v1 -kind: KeycloakRealmGroup -metadata: - name: kubeapps-admin - namespace: {{ include "tenant.name" . }} -spec: - name: kubeapps-admin - realmRef: - name: keycloakrealm-cozy - kind: ClusterKeycloakRealm - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: tenant-root-dashboard-resources - namespace: tenant-root -rules: -- apiGroups: - - "" - resources: - - secrets - resourceNames: - - kubeconfig - verbs: ["get", "list", "watch"] - - ---- - -apiVersion: v1 -kind: Secret -metadata: - name: kubeconfig - namespace: tenant-root -stringData: - kubeconfig: | - apiVersion: v1 - clusters: - - cluster: - server: https://{{ $apiServerAdress }}:6443 - certificate-authority-data: {{ $k8sCa }} - name: cluster - contexts: - - context: - cluster: cluster - user: keycloak - name: default - current-context: default - users: - - name: keycloak - user: - exec: - apiVersion: client.authentication.k8s.io/v1beta1 - args: - - oidc-login - - get-token - - --oidc-issuer-url=https://keycloak.{{ $host }}/realms/cozy - - --oidc-client-id=kubernetes - - --oidc-client-secret={{ $k8sClient }} - - --skip-open-browser - - --grant-type=password - command: kubectl