From 6b3b09e6e1d44ef8431a5770655701bad5b3668f Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Tue, 7 Nov 2023 17:35:28 -0500 Subject: [PATCH 1/4] Refactor oauth configuration We're using a similar oauth configuration on all of our clusters (typically consisting of one or both of GitHub and the NERC Keycloak instance). This commits factors out the oauth configuration into a set of common components so that instead of replicating the same configuration in multiple overlays, we can include one or more reusable components. E.g., for a cluster that only needs keycloak authentication: components: - ../../components/nerc-oauth-keycloak Or for a cluster that wants both keycloak and GitHub: components: - ../../components/nerc-oauth-keycloak - ../../components/nerc-oauth-github In either case, the overlay would apply the necessary patches to override things like client ids, secret names, etc. --- .../oauths/cluster/kustomization.yaml | 2 +- .../oauths/cluster/oauth.yaml | 2 +- .../github-client-secret/externalsecret.yaml | 24 +++++++++++++++ .../github-client-secret/kustomization.yaml | 4 +++ .../externalsecret.yaml | 24 +++++++++++++++ .../kustomization.yaml | 4 +++ .../nerc-oauth-github/kustomization.yaml | 23 ++++++++++++++ .../nerc-oauth-keycloak/kustomization.yaml | 30 +++++++++++++++++++ 8 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 cluster-scope/base/external-secrets.io/externalsecrets/github-client-secret/externalsecret.yaml create mode 100644 cluster-scope/base/external-secrets.io/externalsecrets/github-client-secret/kustomization.yaml create mode 100644 cluster-scope/base/external-secrets.io/externalsecrets/oauths-clientsecret-nerc/externalsecret.yaml create mode 100644 cluster-scope/base/external-secrets.io/externalsecrets/oauths-clientsecret-nerc/kustomization.yaml create mode 100644 cluster-scope/components/nerc-oauth-github/kustomization.yaml create mode 100644 cluster-scope/components/nerc-oauth-keycloak/kustomization.yaml diff --git a/cluster-scope/base/config.openshift.io/oauths/cluster/kustomization.yaml b/cluster-scope/base/config.openshift.io/oauths/cluster/kustomization.yaml index e7b18965..d40f1e27 100644 --- a/cluster-scope/base/config.openshift.io/oauths/cluster/kustomization.yaml +++ b/cluster-scope/base/config.openshift.io/oauths/cluster/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - oauth.yaml + - oauth.yaml diff --git a/cluster-scope/base/config.openshift.io/oauths/cluster/oauth.yaml b/cluster-scope/base/config.openshift.io/oauths/cluster/oauth.yaml index c52b9228..324dfe1a 100644 --- a/cluster-scope/base/config.openshift.io/oauths/cluster/oauth.yaml +++ b/cluster-scope/base/config.openshift.io/oauths/cluster/oauth.yaml @@ -1,11 +1,11 @@ apiVersion: config.openshift.io/v1 kind: OAuth metadata: + name: cluster annotations: include.release.openshift.io/ibm-cloud-managed: "true" include.release.openshift.io/self-managed-high-availability: "true" include.release.openshift.io/single-node-developer: "true" release.openshift.io/create-only: "true" - name: cluster spec: identityProviders: [] diff --git a/cluster-scope/base/external-secrets.io/externalsecrets/github-client-secret/externalsecret.yaml b/cluster-scope/base/external-secrets.io/externalsecrets/github-client-secret/externalsecret.yaml new file mode 100644 index 00000000..3bec0e7b --- /dev/null +++ b/cluster-scope/base/external-secrets.io/externalsecrets/github-client-secret/externalsecret.yaml @@ -0,0 +1,24 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: github-client-secret + namespace: openshift-config +spec: + secretStoreRef: + name: nerc-secret-store + kind: SecretStore + target: + name: github-client-secret + # Prevent generated Secret from inheriting the labels from this + # ExternalSecret. OpenShift will create a copy of the Secret, and the + # labels will causse it to show up as out-of-sync in ArgoCD. See + # https://github.com/OCP-on-NERC/operations/issues/42 for additional + # details. + template: + metadata: + labels: {} + data: + - secretKey: clientSecret + remoteRef: + key: REPLACE_IN_OVERLAY + property: clientSecret diff --git a/cluster-scope/base/external-secrets.io/externalsecrets/github-client-secret/kustomization.yaml b/cluster-scope/base/external-secrets.io/externalsecrets/github-client-secret/kustomization.yaml new file mode 100644 index 00000000..e82804d5 --- /dev/null +++ b/cluster-scope/base/external-secrets.io/externalsecrets/github-client-secret/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - externalsecret.yaml diff --git a/cluster-scope/base/external-secrets.io/externalsecrets/oauths-clientsecret-nerc/externalsecret.yaml b/cluster-scope/base/external-secrets.io/externalsecrets/oauths-clientsecret-nerc/externalsecret.yaml new file mode 100644 index 00000000..a3186d28 --- /dev/null +++ b/cluster-scope/base/external-secrets.io/externalsecrets/oauths-clientsecret-nerc/externalsecret.yaml @@ -0,0 +1,24 @@ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: oauths-clientsecret-nerc + namespace: openshift-config +spec: + secretStoreRef: + kind: SecretStore + name: nerc-secret-store + target: + name: oauths-clientsecret-nerc + # Prevent generated Secret from inheriting the labels from this + # ExternalSecret. OpenShift will create a copy of the Secret, and the + # labels will causse it to show up as out-of-sync in ArgoCD. See + # https://github.com/OCP-on-NERC/operations/issues/42 for additional + # details. + template: + metadata: + labels: {} + data: + - secretKey: clientSecret + remoteRef: + key: REPLACE_IN_OVERLAY + property: clientSecret diff --git a/cluster-scope/base/external-secrets.io/externalsecrets/oauths-clientsecret-nerc/kustomization.yaml b/cluster-scope/base/external-secrets.io/externalsecrets/oauths-clientsecret-nerc/kustomization.yaml new file mode 100644 index 00000000..e82804d5 --- /dev/null +++ b/cluster-scope/base/external-secrets.io/externalsecrets/oauths-clientsecret-nerc/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - externalsecret.yaml diff --git a/cluster-scope/components/nerc-oauth-github/kustomization.yaml b/cluster-scope/components/nerc-oauth-github/kustomization.yaml new file mode 100644 index 00000000..c9543952 --- /dev/null +++ b/cluster-scope/components/nerc-oauth-github/kustomization.yaml @@ -0,0 +1,23 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - ../../base/external-secrets.io/externalsecrets/github-client-secret + +patches: + - patch: | + apiVersion: config.openshift.io/v1 + kind: OAuth + metadata: + name: cluster + spec: + identityProviders: + - name: github + mappingMethod: claim + type: GitHub + github: + clientID: REPLACE_IN_OVERLAY + clientSecret: + name: github-client-secret + teams: + - ocp-on-nerc/nerc-ops diff --git a/cluster-scope/components/nerc-oauth-keycloak/kustomization.yaml b/cluster-scope/components/nerc-oauth-keycloak/kustomization.yaml new file mode 100644 index 00000000..5a7b75ac --- /dev/null +++ b/cluster-scope/components/nerc-oauth-keycloak/kustomization.yaml @@ -0,0 +1,30 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +resources: + - ../../base/external-secrets.io/externalsecrets/oauths-clientsecret-nerc + +patches: + - patch: | + apiVersion: config.openshift.io/v1 + kind: OAuth + metadata: + name: cluster + spec: + identityProviders: + - mappingMethod: lookup + name: mss-keycloak + openID: + claims: + email: + - email + name: + - name + preferredUsername: + - preferred_username + clientID: REPLACE_IN_OVERLAY + clientSecret: + name: oauths-clientsecret-nerc + extraScopes: [] + issuer: https://keycloak.mss.mghpcc.org/auth/realms/mss + type: OpenID From 8a0d5fb8c81240c0620d5244125ffd776987b6fb Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Tue, 7 Nov 2023 17:34:59 -0500 Subject: [PATCH 2/4] Use new oauth config in nerc-ocp-prod --- .../externalsecrets/github-client-secret.yaml | 24 ----------- .../externalsecrets/kustomization.yaml | 2 - .../oauths-clientsecret-nerc.yaml | 24 ----------- .../overlays/nerc-ocp-prod/kustomization.yaml | 40 ++++++++++++++++++- 4 files changed, 39 insertions(+), 51 deletions(-) delete mode 100644 cluster-scope/overlays/nerc-ocp-prod/externalsecrets/github-client-secret.yaml delete mode 100644 cluster-scope/overlays/nerc-ocp-prod/externalsecrets/oauths-clientsecret-nerc.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/externalsecrets/github-client-secret.yaml b/cluster-scope/overlays/nerc-ocp-prod/externalsecrets/github-client-secret.yaml deleted file mode 100644 index 55493593..00000000 --- a/cluster-scope/overlays/nerc-ocp-prod/externalsecrets/github-client-secret.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret -metadata: - name: github-client-secret - namespace: openshift-config -spec: - secretStoreRef: - name: nerc-secret-store - kind: SecretStore - target: - name: github-client-secret - # Prevent generated Secret from inheriting the labels from this - # ExternalSecret. OpenShift will create a copy of the Secret, and the - # labels will causse it to show up as out-of-sync in ArgoCD. See - # https://github.com/OCP-on-NERC/operations/issues/42 for additional - # details. - template: - metadata: - labels: {} - data: - - secretKey: clientSecret - remoteRef: - key: nerc/nerc-ocp-prod/openshift-config/github-client-secret - property: clientSecret diff --git a/cluster-scope/overlays/nerc-ocp-prod/externalsecrets/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/externalsecrets/kustomization.yaml index 2b3a7908..40b9db5c 100644 --- a/cluster-scope/overlays/nerc-ocp-prod/externalsecrets/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-prod/externalsecrets/kustomization.yaml @@ -1,6 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- oauths-clientsecret-nerc.yaml -- github-client-secret.yaml - github-group-sync.yaml diff --git a/cluster-scope/overlays/nerc-ocp-prod/externalsecrets/oauths-clientsecret-nerc.yaml b/cluster-scope/overlays/nerc-ocp-prod/externalsecrets/oauths-clientsecret-nerc.yaml deleted file mode 100644 index 5a88d8b8..00000000 --- a/cluster-scope/overlays/nerc-ocp-prod/externalsecrets/oauths-clientsecret-nerc.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: external-secrets.io/v1beta1 -kind: ExternalSecret -metadata: - name: oauths-clientsecret-nerc - namespace: openshift-config -spec: - secretStoreRef: - kind: SecretStore - name: nerc-secret-store - target: - name: oauths-clientsecret-nerc - # Prevent generated Secret from inheriting the labels from this - # ExternalSecret. OpenShift will create a copy of the Secret, and the - # labels will causse it to show up as out-of-sync in ArgoCD. See - # https://github.com/OCP-on-NERC/operations/issues/42 for additional - # details. - template: - metadata: - labels: {} - data: - - secretKey: clientSecret - remoteRef: - key: nerc/nerc-ocp-prod/openshift-config/oauths-clientsecret-nerc - property: clientSecret diff --git a/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml index e5073b20..79e7b6be 100644 --- a/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-prod/kustomization.yaml @@ -39,6 +39,11 @@ resources: - odhdashboardconfigs components: + - ../../components/nerc-oauth-keycloak + - ../../components/nerc-oauth-github + + # this must come last in order to apply + # to all resources. - ../../components/argocd-skip-dryrun generatorOptions: @@ -52,7 +57,6 @@ configMapGenerator: patches: - path: ingresscontrollers/default_patch.yaml -- path: oauths/cluster_patch.yaml - path: kubeletconfigs/system-reserved-patch.yaml - target: kind: SecretStore @@ -60,3 +64,37 @@ patches: - op: replace path: /spec/provider/vault/auth/kubernetes/mountPath value: kubernetes/nerc-ocp-prod +- patch: | + apiVersion: config.openshift.io/v1 + kind: OAuth + metadata: + name: cluster + spec: + identityProviders: + - name: mss-keycloak + openID: + clientID: ocp-prod +- patch: | + apiVersion: config.openshift.io/v1 + kind: OAuth + metadata: + name: cluster + spec: + identityProviders: + - name: github + github: + clientID: 771ea98004d436c6e529 +- target: + kind: ExternalSecret + name: oauths-clientsecret-nerc + patch: | + - op: replace + path: /spec/data/0/remoteRef/key + value: nerc/nerc-ocp-prod/openshift-config/oauths-clientsecret-nerc +- target: + kind: ExternalSecret + name: github-client-secret + patch: | + - op: replace + path: /spec/data/0/remoteRef/key + value: nerc/nerc-ocp-prod/openshift-config/github-client-secret From 70bfd1a610028d9122c1d621c284291ca88ec342 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Fri, 29 Sep 2023 11:23:22 -0400 Subject: [PATCH 3/4] Use new oauth config in nerc-ocp-infra --- .../externalsecrets/kustomization.yaml | 1 - .../nerc-ocp-infra/kustomization.yaml | 32 ++++++++++++++++++- .../nerc-ocp-infra/oauths/cluster_patch.yaml | 16 ---------- 3 files changed, 31 insertions(+), 18 deletions(-) delete mode 100644 cluster-scope/overlays/nerc-ocp-infra/oauths/cluster_patch.yaml diff --git a/cluster-scope/overlays/nerc-ocp-infra/externalsecrets/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-infra/externalsecrets/kustomization.yaml index 7eb96158..cce49e92 100644 --- a/cluster-scope/overlays/nerc-ocp-infra/externalsecrets/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-infra/externalsecrets/kustomization.yaml @@ -1,7 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github-client-secret.yaml - rook-ceph-external-cluster-details.yaml - default-ingress-certificate.yaml - default-api-certificate.yaml diff --git a/cluster-scope/overlays/nerc-ocp-infra/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-infra/kustomization.yaml index ee1ff2fe..737d2a44 100644 --- a/cluster-scope/overlays/nerc-ocp-infra/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-infra/kustomization.yaml @@ -32,6 +32,9 @@ resources: - grafana-dashboards - persistentvolumeclaims +components: + - ../../components/nerc-oauth-github + generatorOptions: disableNameSuffixHash: true @@ -42,7 +45,6 @@ configMapGenerator: namespace: openshift-monitoring patches: -- path: oauths/cluster_patch.yaml - path: consoles.operator.openshift.io/cluster_patch.yaml - path: storageclasses/ocs-external-storagecluster-ceph-rbd_patch.yaml - path: clustersecretstores/nerc-cluster-secrets_patch.yaml @@ -50,3 +52,31 @@ patches: - path: machineconfigs/hostpath-provisioner-selinux_patch.yaml - path: externalsecrets/open-cluster-management-observability-multiclusterhub-operator-pull-secret_patch.yaml - path: externalsecrets/open-cluster-management-observability-thanos-object-storage_patch.yaml +- patch: | + apiVersion: config.openshift.io/v1 + kind: OAuth + metadata: + name: cluster + spec: + identityProviders: + - name: github + github: + clientID: 77915cd4cdb5c4df7723 + teams: + - ocp-on-nerc/nerc-ops + - ocp-on-nerc/nerc-logs-metrics +- target: + kind: ExternalSecret + name: github-client-secret + patch: | + - op: replace + path: /spec/data/0/remoteRef/key + value: nerc/nerc-ocp-infra/openshift-config/github-client-secret +- target: + kind: ExternalSecret + patch: | + - op: replace + path: /spec/secretStoreRef + value: + kind: ClusterSecretStore + name: nerc-cluster-secrets diff --git a/cluster-scope/overlays/nerc-ocp-infra/oauths/cluster_patch.yaml b/cluster-scope/overlays/nerc-ocp-infra/oauths/cluster_patch.yaml deleted file mode 100644 index 18264616..00000000 --- a/cluster-scope/overlays/nerc-ocp-infra/oauths/cluster_patch.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: config.openshift.io/v1 -kind: OAuth -metadata: - name: cluster -spec: - identityProviders: - - name: github - mappingMethod: claim - type: GitHub - github: - clientID: 77915cd4cdb5c4df7723 - clientSecret: - name: github-client-secret - teams: - - ocp-on-nerc/nerc-ops - - ocp-on-nerc/nerc-logs-metrics From 48ea04b89650198243bb517afb5b14d0ad1486df Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Fri, 29 Sep 2023 11:30:04 -0400 Subject: [PATCH 4/4] Use new oauth config on nerc-ocp-test --- .../externalsecrets/kustomization.yaml | 1 - .../overlays/nerc-ocp-test/kustomization.yaml | 25 ++++++++++++++++++- .../nerc-ocp-test/oauths/cluster_patch.yaml | 17 ------------- 3 files changed, 24 insertions(+), 19 deletions(-) delete mode 100644 cluster-scope/overlays/nerc-ocp-test/oauths/cluster_patch.yaml diff --git a/cluster-scope/overlays/nerc-ocp-test/externalsecrets/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-test/externalsecrets/kustomization.yaml index 1498632d..40b9db5c 100644 --- a/cluster-scope/overlays/nerc-ocp-test/externalsecrets/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-test/externalsecrets/kustomization.yaml @@ -1,5 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- github-client-secret.yaml - github-group-sync.yaml diff --git a/cluster-scope/overlays/nerc-ocp-test/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-test/kustomization.yaml index d059e4ee..4cc9e1c8 100644 --- a/cluster-scope/overlays/nerc-ocp-test/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-test/kustomization.yaml @@ -21,11 +21,34 @@ resources: - nodenetworkconfigurationpolicies/vlan-2175-nese.yaml - secretstores +components: + - ../../components/nerc-oauth-github + patches: -- path: oauths/cluster_patch.yaml - target: kind: SecretStore patch: | - op: replace path: /spec/provider/vault/auth/kubernetes/mountPath value: kubernetes/nerc-ocp-test +- patch: | + apiVersion: config.openshift.io/v1 + kind: OAuth + metadata: + name: cluster + spec: + identityProviders: + - name: github + github: + clientID: e87d9a48533084f2aa5f + teams: + - ocp-on-nerc/nerc-ops + - ocp-on-nerc/nerc-logs-metrics + - ocp-on-nerc/nerc-rhods +- target: + kind: ExternalSecret + name: github-client-secret + patch: | + - op: replace + path: /spec/data/0/remoteRef/key + value: nerc/nerc-ocp-test/openshift-config/github-client-secret diff --git a/cluster-scope/overlays/nerc-ocp-test/oauths/cluster_patch.yaml b/cluster-scope/overlays/nerc-ocp-test/oauths/cluster_patch.yaml deleted file mode 100644 index daee3c58..00000000 --- a/cluster-scope/overlays/nerc-ocp-test/oauths/cluster_patch.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: config.openshift.io/v1 -kind: OAuth -metadata: - name: cluster -spec: - identityProviders: - - name: github - mappingMethod: claim - type: GitHub - github: - clientID: e87d9a48533084f2aa5f - clientSecret: - name: github-client-secret - teams: - - ocp-on-nerc/nerc-ops - - ocp-on-nerc/nerc-logs-metrics - - ocp-on-nerc/nerc-rhods