creating PR for small helm fixes (#219)

* creating PR for small helm fixes * updating pelican export image to "master" as the code has been fixed to work with Helm * fixing gen3 chart versions * fixing etl environment variable error * fixing syntax error, updating comment, and removing checksum for deleted configmap. * fixing aws-configuration error * bumping gen3 chart version * feat(helm-fixes): Added a few more fixes * version bump * wrong gen3 version * adding a new line at the end of values.yaml --------- Co-authored-by: Ed <[email protected]>
uc-cdis · Dec 9, 2024 · 0c71be3 · 0c71be3
1 parent 1ba82ac
commit 0c71be3
Show file tree

Hide file tree

Showing 24 changed files with 135 additions and 51 deletions.
diff --git a/helm/etl/Chart.yaml b/helm/etl/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.6
+version: 0.1.7
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/etl/README.md b/helm/etl/README.md
@@ -1,6 +1,6 @@
 # etl
 
-![Version: 0.1.6](https://img.shields.io/badge/Version-0.1.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.7](https://img.shields.io/badge/Version-0.1.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 etl
 

diff --git a/helm/etl/templates/etl-job.yaml b/helm/etl/templates/etl-job.yaml
@@ -52,7 +52,7 @@ spec:
                 name: etl-mapping
             - name: fence-yaml
               configMap:
-                name: useryaml
+                name: fence
           containers:
             - name: gen3-spark
               image: {{ .Values.image.spark.repository }}:{{ .Values.image.spark.tag }}
@@ -156,7 +156,7 @@ spec:
                 - name: SPARK_DRIVER_MEMORY
                   value: 6g
                 - name: ETL_FORCED
-                  value: {{ .Values.etlForced }}
+                  value: {{ .Values.etlForced | quote }}
                 - name: gen3Env
                   valueFrom:
                     configMapKeyRef:
@@ -183,7 +183,7 @@ spec:
                 - name: "fence-yaml"
                   readOnly: true
                   mountPath: "/gen3/tube/user.yaml"
-                  subPath: useryaml
+                  subPath: user.yaml
               resources:
                 requests:
                   cpu: {{  .Values.resources.tube.requests.cpu }}

diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.26
+version: 0.1.27
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/fence/README.md b/helm/fence/README.md
@@ -1,6 +1,6 @@
 # fence
 
-![Version: 0.1.26](https://img.shields.io/badge/Version-0.1.26-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.27](https://img.shields.io/badge/Version-0.1.27-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 Fence
 

diff --git a/helm/fence/templates/usersync-cron.yaml b/helm/fence/templates/usersync-cron.yaml
@@ -119,8 +119,8 @@ spec:
                     echo "ERROR: failed to generate ETL config"
                     exit 1
                   fi
-                  # kubectl delete configmap fence > /dev/null 2>&1
-                  # kubectl create configmap fence --from-file=/tmp/user.yaml
+                  kubectl delete configmap fence > /dev/null 2>&1
+                  kubectl create configmap fence --from-file=/tmp/user.yaml
                   if [ "${slackWebHook}" != 'None' ]; then
                     curl -X POST --data-urlencode "payload={\"text\": \"AWSHelper: Syncing users on ${gen3Env}\"}" "${slackWebHook}"
                   fi

diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml
@@ -28,15 +28,15 @@ dependencies:
     version: 0.1.16
     repository: file://../common
   - name: etl
-    version: 0.1.6
+    version: 0.1.7
     repository: file://../etl
     condition: etl.enabled
   - name: frontend-framework
     version: 0.1.6
     repository: "file://../frontend-framework"
     condition: frontend-framework.enabled
   - name: fence
-    version: 0.1.26
+    version: 0.1.27
     repository: "file://../fence"
     condition: fence.enabled
   - name: guppy
@@ -52,11 +52,11 @@ dependencies:
     repository: "file://../indexd"
     condition: indexd.enabled
   - name: manifestservice
-    version: 0.1.17
+    version: 0.1.18
     repository: "file://../manifestservice"
     condition: manifestservice.enabled
   - name: metadata
-    version: 0.1.16
+    version: 0.1.17
     repository: "file://../metadata"
     condition: metadata.enabled
   - name: peregrine
@@ -84,7 +84,7 @@ dependencies:
     repository: "file://../ssjdispatcher"
     condition: ssjdispatcher.enabled
   - name: sower
-    version: 0.1.15
+    version: 0.1.16
     condition: sower.enabled
     repository: "file://../sower"
   - name: wts
@@ -128,7 +128,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.51
+version: 0.1.52
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/gen3/README.md b/helm/gen3/README.md
@@ -1,6 +1,6 @@
 # gen3
 
-![Version: 0.1.51](https://img.shields.io/badge/Version-0.1.51-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.52](https://img.shields.io/badge/Version-0.1.52-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 Helm chart to deploy Gen3 Data Commons
 
@@ -24,22 +24,22 @@ Helm chart to deploy Gen3 Data Commons
 | file://../audit | audit | 0.1.16 |
 | file://../aws-es-proxy | aws-es-proxy | 0.1.13 |
 | file://../common | common | 0.1.16 |
-| file://../etl | etl | 0.1.6 |
-| file://../fence | fence | 0.1.26 |
+| file://../etl | etl | 0.1.7 |
+| file://../fence | fence | 0.1.27 |
 | file://../frontend-framework | frontend-framework | 0.1.6 |
 | file://../gen3-network-policies | gen3-network-policies | 0.1.2 |
 | file://../guppy | guppy | 0.1.16 |
 | file://../hatchery | hatchery | 0.1.12 |
 | file://../indexd | indexd | 0.1.18 |
-| file://../manifestservice | manifestservice | 0.1.17 |
-| file://../metadata | metadata | 0.1.16 |
+| file://../manifestservice | manifestservice | 0.1.18 |
+| file://../metadata | metadata | 0.1.17 |
 | file://../neuvector | neuvector | 0.1.2 |
 | file://../peregrine | peregrine | 0.1.17 |
 | file://../portal | portal | 0.1.22 |
 | file://../requestor | requestor | 0.1.16 |
 | file://../revproxy | revproxy | 0.1.19 |
 | file://../sheepdog | sheepdog | 0.1.20 |
-| file://../sower | sower | 0.1.15 |
+| file://../sower | sower | 0.1.16 |
 | file://../ssjdispatcher | ssjdispatcher | 0.1.15 |
 | file://../wts | wts | 0.1.18 |
 | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 |
@@ -176,6 +176,9 @@ Helm chart to deploy Gen3 Data Commons
 | revproxy.ingress.enabled | bool | `false` | Whether to create the custom revproxy ingress |
 | revproxy.ingress.hosts | list | `[{"host":"chart-example.local"}]` | Where to route the traffic. |
 | revproxy.ingress.tls | list | `[]` | To secure an Ingress by specifying a secret that contains a TLS private key and certificate. |
+| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for External Secrets and DB Secrets. |
+| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. |
+| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. |
 | sheepdog.enabled | bool | `true` | Whether to deploy the sheepdog subchart. |
 | ssjdispatcher.enabled | bool | `false` | Whether to deploy the ssjdispatcher subchart. |
 | wts.enabled | bool | `true` | Whether to deploy the wts subchart. |
diff --git a/helm/gen3/templates/cluster-secret-store.yaml b/helm/gen3/templates/cluster-secret-store.yaml
@@ -12,10 +12,16 @@ spec:
       region: {{ .Values.global.aws.region }}
       auth:
         secretRef:
+        {{- if .Values.global.aws.useLocalSecret.localSecretName }}
           accessKeyIDSecretRef:
             name: {{ .Values.global.aws.useLocalSecret.localSecretName }}
             key: access-key
           secretAccessKeySecretRef:
             name: {{ .Values.global.aws.useLocalSecret.localSecretName }}
             key: secret-access-key
+        {{- else }}
+         jwt:
+          serviceAccountRef:
+            name: {{ .Values.global.aws.secretStoreServiceAccount.name }}
+        {{- end }}  
 {{- end }}
diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml
@@ -321,3 +321,10 @@ neuvector:
   DB_HOST: development-gen3-postgresql
   # hostname/service name for our ElasitcSearch instance, used to allow egress from containers
   ES_HOST: gen3-elasticsearch-master
+
+# -- (map) Secret information for External Secrets and DB Secrets.
+secrets:
+  # -- (str) AWS access key ID. Overrides global key.
+  awsAccessKeyId:
+  # -- (str) AWS secret access key ID. Overrides global key.
+  awsSecretAccessKey:
diff --git a/helm/manifestservice/Chart.yaml b/helm/manifestservice/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.17
+version: 0.1.18
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/manifestservice/README.md b/helm/manifestservice/README.md
@@ -1,6 +1,6 @@
 # manifestservice
 
-![Version: 0.1.17](https://img.shields.io/badge/Version-0.1.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.18](https://img.shields.io/badge/Version-0.1.18-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for Kubernetes
 

diff --git a/helm/manifestservice/templates/manifestservice-creds.yaml b/helm/manifestservice/templates/manifestservice-creds.yaml
@@ -9,8 +9,10 @@ stringData:
       {
         "manifest_bucket_name": "{{ .Values.manifestserviceG3auto.bucketName }}",
         "hostname": "{{ .Values.global.hostname }}",
+        {{ if and .Values.manifestserviceG3auto.awsaccesskey .Values.manifestserviceG3auto.awssecretkey }}
         "aws_access_key_id": "{{ .Values.manifestserviceG3auto.awsaccesskey }}",
         "aws_secret_access_key": "{{ .Values.manifestserviceG3auto.awssecretkey }}",
+        {{ end }}
         "prefix": "{{ .Values.manifestserviceG3auto.prefix }}"
       }
 {{- end }}
diff --git a/helm/metadata/Chart.yaml b/helm/metadata/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.16
+version: 0.1.17
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/metadata/README.md b/helm/metadata/README.md
@@ -1,6 +1,6 @@
 # metadata
 
-![Version: 0.1.16](https://img.shields.io/badge/Version-0.1.16-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.17](https://img.shields.io/badge/Version-0.1.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 Metadata Service
 
@@ -118,5 +118,5 @@ A Helm chart for gen3 Metadata Service
 | strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy |
 | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. |
 | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. |
-| useAggMds | bool | `"True"` | Set to true to aggregate metadata from multiple other Metadata Service instances. |
+| useAggMds | bool | `"False"` | Set to true to aggregate metadata from multiple other Metadata Service instances. |
 | volumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/mds/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/aggregate_config.json","name":"config-volume","readOnly":true,"subPath":"aggregate_config.json"},{"mountPath":"/metadata.json","name":"config-manifest","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. |
diff --git a/helm/metadata/templates/deployment.yaml b/helm/metadata/templates/deployment.yaml
@@ -25,7 +25,6 @@ spec:
         {{- include "metadata.selectorLabels" . | nindent 8 }}
         {{- include "common.extraLabels" . | nindent 8 }}
       annotations: 
-        checksum/config: {{ include (print $.Template.BasePath "/metadata.yaml") . | sha256sum }}
         {{- if .Values.metricsEnabled }}
         {{- include "common.grafanaAnnotations" . | nindent 8 }}
         {{- end }}

diff --git a/helm/metadata/templates/metadata.yaml b/helm/metadata/templates/metadata.yaml
diff --git a/helm/metadata/values.yaml b/helm/metadata/values.yaml
@@ -173,7 +173,7 @@ debug: false
 # -- (string) Elasticsearch endpoint.
 esEndpoint: http://gen3-elasticsearch-master:9200
 # -- (bool) Set to true to aggregate metadata from multiple other Metadata Service instances.
-useAggMds: "True"
+useAggMds: "False"
 # -- (string) Namespae to use if AggMds is enabled.
 aggMdsNamespace: default
 

diff --git a/helm/sower/Chart.yaml b/helm/sower/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.15
+version: 0.1.16
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to