diff --git a/helm/etl/Chart.yaml b/helm/etl/Chart.yaml index d55e0945..83ba97e5 100644 --- a/helm/etl/Chart.yaml +++ b/helm/etl/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.6 +version: 0.1.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/etl/README.md b/helm/etl/README.md index 4c1c5995..3ef0e7e9 100644 --- a/helm/etl/README.md +++ b/helm/etl/README.md @@ -1,6 +1,6 @@ # etl -![Version: 0.1.6](https://img.shields.io/badge/Version-0.1.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.7](https://img.shields.io/badge/Version-0.1.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 etl diff --git a/helm/etl/templates/etl-job.yaml b/helm/etl/templates/etl-job.yaml index 9272940c..0056c497 100644 --- a/helm/etl/templates/etl-job.yaml +++ b/helm/etl/templates/etl-job.yaml @@ -52,7 +52,7 @@ spec: name: etl-mapping - name: fence-yaml configMap: - name: useryaml + name: fence containers: - name: gen3-spark image: {{ .Values.image.spark.repository }}:{{ .Values.image.spark.tag }} @@ -156,7 +156,7 @@ spec: - name: SPARK_DRIVER_MEMORY value: 6g - name: ETL_FORCED - value: {{ .Values.etlForced }} + value: {{ .Values.etlForced | quote }} - name: gen3Env valueFrom: configMapKeyRef: @@ -183,7 +183,7 @@ spec: - name: "fence-yaml" readOnly: true mountPath: "/gen3/tube/user.yaml" - subPath: useryaml + subPath: user.yaml resources: requests: cpu: {{ .Values.resources.tube.requests.cpu }} diff --git a/helm/fence/Chart.yaml b/helm/fence/Chart.yaml index 02930bbf..4df81c7f 100644 --- a/helm/fence/Chart.yaml +++ b/helm/fence/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.26 +version: 0.1.27 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/fence/README.md b/helm/fence/README.md index 690caa00..89908be8 100644 --- a/helm/fence/README.md +++ b/helm/fence/README.md @@ -1,6 +1,6 @@ # fence -![Version: 0.1.26](https://img.shields.io/badge/Version-0.1.26-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.27](https://img.shields.io/badge/Version-0.1.27-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Fence diff --git a/helm/fence/templates/usersync-cron.yaml b/helm/fence/templates/usersync-cron.yaml index 2349f8ce..d10ad669 100644 --- a/helm/fence/templates/usersync-cron.yaml +++ b/helm/fence/templates/usersync-cron.yaml @@ -119,8 +119,8 @@ spec: echo "ERROR: failed to generate ETL config" exit 1 fi - # kubectl delete configmap fence > /dev/null 2>&1 - # kubectl create configmap fence --from-file=/tmp/user.yaml + kubectl delete configmap fence > /dev/null 2>&1 + kubectl create configmap fence --from-file=/tmp/user.yaml if [ "${slackWebHook}" != 'None' ]; then curl -X POST --data-urlencode "payload={\"text\": \"AWSHelper: Syncing users on ${gen3Env}\"}" "${slackWebHook}" fi diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml index 6224603b..84b9ccd7 100644 --- a/helm/gen3/Chart.yaml +++ b/helm/gen3/Chart.yaml @@ -28,7 +28,7 @@ dependencies: version: 0.1.16 repository: file://../common - name: etl - version: 0.1.6 + version: 0.1.7 repository: file://../etl condition: etl.enabled - name: frontend-framework @@ -36,7 +36,7 @@ dependencies: repository: "file://../frontend-framework" condition: frontend-framework.enabled - name: fence - version: 0.1.26 + version: 0.1.27 repository: "file://../fence" condition: fence.enabled - name: guppy @@ -52,11 +52,11 @@ dependencies: repository: "file://../indexd" condition: indexd.enabled - name: manifestservice - version: 0.1.17 + version: 0.1.18 repository: "file://../manifestservice" condition: manifestservice.enabled - name: metadata - version: 0.1.16 + version: 0.1.17 repository: "file://../metadata" condition: metadata.enabled - name: peregrine @@ -84,7 +84,7 @@ dependencies: repository: "file://../ssjdispatcher" condition: ssjdispatcher.enabled - name: sower - version: 0.1.15 + version: 0.1.16 condition: sower.enabled repository: "file://../sower" - name: wts @@ -128,7 +128,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.51 +version: 0.1.52 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 174aacf3..a39e4aed 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -1,6 +1,6 @@ # gen3 -![Version: 0.1.51](https://img.shields.io/badge/Version-0.1.51-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.52](https://img.shields.io/badge/Version-0.1.52-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) Helm chart to deploy Gen3 Data Commons @@ -24,22 +24,22 @@ Helm chart to deploy Gen3 Data Commons | file://../audit | audit | 0.1.16 | | file://../aws-es-proxy | aws-es-proxy | 0.1.13 | | file://../common | common | 0.1.16 | -| file://../etl | etl | 0.1.6 | -| file://../fence | fence | 0.1.26 | +| file://../etl | etl | 0.1.7 | +| file://../fence | fence | 0.1.27 | | file://../frontend-framework | frontend-framework | 0.1.6 | | file://../gen3-network-policies | gen3-network-policies | 0.1.2 | | file://../guppy | guppy | 0.1.16 | | file://../hatchery | hatchery | 0.1.12 | | file://../indexd | indexd | 0.1.18 | -| file://../manifestservice | manifestservice | 0.1.17 | -| file://../metadata | metadata | 0.1.16 | +| file://../manifestservice | manifestservice | 0.1.18 | +| file://../metadata | metadata | 0.1.17 | | file://../neuvector | neuvector | 0.1.2 | | file://../peregrine | peregrine | 0.1.17 | | file://../portal | portal | 0.1.22 | | file://../requestor | requestor | 0.1.16 | | file://../revproxy | revproxy | 0.1.19 | | file://../sheepdog | sheepdog | 0.1.20 | -| file://../sower | sower | 0.1.15 | +| file://../sower | sower | 0.1.16 | | file://../ssjdispatcher | ssjdispatcher | 0.1.15 | | file://../wts | wts | 0.1.18 | | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 | @@ -176,6 +176,9 @@ Helm chart to deploy Gen3 Data Commons | revproxy.ingress.enabled | bool | `false` | Whether to create the custom revproxy ingress | | revproxy.ingress.hosts | list | `[{"host":"chart-example.local"}]` | Where to route the traffic. | | revproxy.ingress.tls | list | `[]` | To secure an Ingress by specifying a secret that contains a TLS private key and certificate. | +| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for External Secrets and DB Secrets. | +| secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | +| secrets.awsSecretAccessKey | str | `nil` | AWS secret access key ID. Overrides global key. | | sheepdog.enabled | bool | `true` | Whether to deploy the sheepdog subchart. | | ssjdispatcher.enabled | bool | `false` | Whether to deploy the ssjdispatcher subchart. | | wts.enabled | bool | `true` | Whether to deploy the wts subchart. | diff --git a/helm/gen3/templates/cluster-secret-store.yaml b/helm/gen3/templates/cluster-secret-store.yaml index 28ffe29e..38650a4c 100644 --- a/helm/gen3/templates/cluster-secret-store.yaml +++ b/helm/gen3/templates/cluster-secret-store.yaml @@ -12,10 +12,16 @@ spec: region: {{ .Values.global.aws.region }} auth: secretRef: + {{- if .Values.global.aws.useLocalSecret.localSecretName }} accessKeyIDSecretRef: name: {{ .Values.global.aws.useLocalSecret.localSecretName }} key: access-key secretAccessKeySecretRef: name: {{ .Values.global.aws.useLocalSecret.localSecretName }} key: secret-access-key + {{- else }} + jwt: + serviceAccountRef: + name: {{ .Values.global.aws.secretStoreServiceAccount.name }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index 4caa59a8..05519614 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -321,3 +321,10 @@ neuvector: DB_HOST: development-gen3-postgresql # hostname/service name for our ElasitcSearch instance, used to allow egress from containers ES_HOST: gen3-elasticsearch-master + +# -- (map) Secret information for External Secrets and DB Secrets. +secrets: + # -- (str) AWS access key ID. Overrides global key. + awsAccessKeyId: + # -- (str) AWS secret access key ID. Overrides global key. + awsSecretAccessKey: diff --git a/helm/manifestservice/Chart.yaml b/helm/manifestservice/Chart.yaml index 84194a1a..cde53eee 100644 --- a/helm/manifestservice/Chart.yaml +++ b/helm/manifestservice/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.17 +version: 0.1.18 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/manifestservice/README.md b/helm/manifestservice/README.md index cc328847..5fef5aa6 100644 --- a/helm/manifestservice/README.md +++ b/helm/manifestservice/README.md @@ -1,6 +1,6 @@ # manifestservice -![Version: 0.1.17](https://img.shields.io/badge/Version-0.1.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.18](https://img.shields.io/badge/Version-0.1.18-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for Kubernetes diff --git a/helm/manifestservice/templates/manifestservice-creds.yaml b/helm/manifestservice/templates/manifestservice-creds.yaml index 54c5f29a..11fd9990 100644 --- a/helm/manifestservice/templates/manifestservice-creds.yaml +++ b/helm/manifestservice/templates/manifestservice-creds.yaml @@ -9,8 +9,10 @@ stringData: { "manifest_bucket_name": "{{ .Values.manifestserviceG3auto.bucketName }}", "hostname": "{{ .Values.global.hostname }}", + {{ if and .Values.manifestserviceG3auto.awsaccesskey .Values.manifestserviceG3auto.awssecretkey }} "aws_access_key_id": "{{ .Values.manifestserviceG3auto.awsaccesskey }}", "aws_secret_access_key": "{{ .Values.manifestserviceG3auto.awssecretkey }}", + {{ end }} "prefix": "{{ .Values.manifestserviceG3auto.prefix }}" } {{- end }} \ No newline at end of file diff --git a/helm/metadata/Chart.yaml b/helm/metadata/Chart.yaml index ecaa6636..02368daa 100644 --- a/helm/metadata/Chart.yaml +++ b/helm/metadata/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.16 +version: 0.1.17 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/metadata/README.md b/helm/metadata/README.md index e577d5fc..642e62b3 100644 --- a/helm/metadata/README.md +++ b/helm/metadata/README.md @@ -1,6 +1,6 @@ # metadata -![Version: 0.1.16](https://img.shields.io/badge/Version-0.1.16-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.17](https://img.shields.io/badge/Version-0.1.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 Metadata Service @@ -118,5 +118,5 @@ A Helm chart for gen3 Metadata Service | strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | -| useAggMds | bool | `"True"` | Set to true to aggregate metadata from multiple other Metadata Service instances. | +| useAggMds | bool | `"False"` | Set to true to aggregate metadata from multiple other Metadata Service instances. | | volumeMounts | list | `[{"mountPath":"/src/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/mds/.env","name":"config-volume-g3auto","readOnly":true,"subPath":"metadata.env"},{"mountPath":"/aggregate_config.json","name":"config-volume","readOnly":true,"subPath":"aggregate_config.json"},{"mountPath":"/metadata.json","name":"config-manifest","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. | diff --git a/helm/metadata/templates/deployment.yaml b/helm/metadata/templates/deployment.yaml index e4c6ee87..58e06603 100644 --- a/helm/metadata/templates/deployment.yaml +++ b/helm/metadata/templates/deployment.yaml @@ -25,7 +25,6 @@ spec: {{- include "metadata.selectorLabels" . | nindent 8 }} {{- include "common.extraLabels" . | nindent 8 }} annotations: - checksum/config: {{ include (print $.Template.BasePath "/metadata.yaml") . | sha256sum }} {{- if .Values.metricsEnabled }} {{- include "common.grafanaAnnotations" . | nindent 8 }} {{- end }} diff --git a/helm/metadata/templates/metadata.yaml b/helm/metadata/templates/metadata.yaml deleted file mode 100644 index 58700a6e..00000000 --- a/helm/metadata/templates/metadata.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: manifest-metadata -data: - metadata.json: | - { - "USE_AGG_MDS": "{{ .Values.USE_AGG_MDS }}", - "AGG_MDS_NAMESPACE": "{{ .Values.AGG_MDS_NAMESPACE }}" - } \ No newline at end of file diff --git a/helm/metadata/values.yaml b/helm/metadata/values.yaml index 99cbd8ba..fd63e46d 100644 --- a/helm/metadata/values.yaml +++ b/helm/metadata/values.yaml @@ -173,7 +173,7 @@ debug: false # -- (string) Elasticsearch endpoint. esEndpoint: http://gen3-elasticsearch-master:9200 # -- (bool) Set to true to aggregate metadata from multiple other Metadata Service instances. -useAggMds: "True" +useAggMds: "False" # -- (string) Namespae to use if AggMds is enabled. aggMdsNamespace: default diff --git a/helm/sower/Chart.yaml b/helm/sower/Chart.yaml index b47cda9f..c683d22b 100644 --- a/helm/sower/Chart.yaml +++ b/helm/sower/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.15 +version: 0.1.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/sower/README.md b/helm/sower/README.md index 4b883c3c..3aa7dc4a 100644 --- a/helm/sower/README.md +++ b/helm/sower/README.md @@ -1,6 +1,6 @@ # sower -![Version: 0.1.15](https://img.shields.io/badge/Version-0.1.15-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +![Version: 0.1.16](https://img.shields.io/badge/Version-0.1.16-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) A Helm chart for gen3 sower @@ -31,9 +31,11 @@ A Helm chart for gen3 sower | awsStsRegionalEndpoints | string | `"regional"` | AWS STS to issue temporary credentials to users and roles that make an AWS STS request. Values regional or global. | | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | | criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | -| externalSecrets | map | `{"createK8sPelicanServiceSecret":false,"pelicanserviceG3auto":null}` | External Secrets settings. | -| externalSecrets.createK8sPelicanServiceSecret | string | `false` | Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | +| externalSecrets | map | `{"createK8sPelicanServiceSecret":false,"createK8sSowerJobsSecret":false,"pelicanserviceG3auto":null,"sowerjobsG3auto":null}` | External Secrets settings. | +| externalSecrets.createK8sPelicanServiceSecret | string | `false` | Will create the Helm "pelicanservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | +| externalSecrets.createK8sSowerJobsSecret | string | `false` | Will create the Helm "sower-jobs-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. | | externalSecrets.pelicanserviceG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "pelicanservice-g3auto" | +| externalSecrets.sowerjobsG3auto | string | `nil` | Will override the name of the aws secrets manager secret. Default is "sower-jobs-g3auto" | | fullnameOverride | string | `""` | Override the full name of the deployment. | | gen3Namespace | string | `"default"` | Namespace to deploy the job. | | global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false}` | AWS configuration | @@ -45,7 +47,7 @@ A Helm chart for gen3 sower | global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | | global.environment | string | `"default"` | Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. | | global.externalSecrets | map | `{"deploy":false,"separateSecretStore":false}` | External Secrets settings. | -| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed. | +| global.externalSecrets.deploy | bool | `false` | Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any sower secrets you have deployed. | | global.externalSecrets.separateSecretStore | string | `false` | Will deploy a separate External Secret Store for this service. | | global.hostname | string | `"localhost"` | Hostname for the deployment. | | global.kubeBucket | string | `"kube-gen3"` | S3 bucket name for Kubernetes manifest files. | @@ -85,7 +87,7 @@ A Helm chart for gen3 sower | resources.requests | map | `{"cpu":"100m","memory":"20Mi"}` | The amount of resources that the container requests | | resources.requests.cpu | string | `"100m"` | The amount of CPU requested | | resources.requests.memory | string | `"20Mi"` | The amount of memory requested | -| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Secret information for Usersync and External Secrets. | +| secrets | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null}` | Values for sower secrets and keys for External Secrets. | | secrets.awsAccessKeyId | str | `nil` | AWS access key ID. Overrides global key. | | secrets.awsSecretAccessKey | str | `nil` | AWS access key ID. Overrides global key. | | securityContext | map | `{}` | Security context for the containers in the pod | @@ -122,7 +124,7 @@ A Helm chart for gen3 sower | sowerConfig[0].container.env[7].name | string | `"SHEEPDOG"` | | | sowerConfig[0].container.env[7].valueFrom.secretKeyRef.key | string | `"sheepdog"` | | | sowerConfig[0].container.env[7].valueFrom.secretKeyRef.name | string | `"indexd-service-creds"` | | -| sowerConfig[0].container.image | string | `"quay.io/cdis/pelican-export:GPE-1252"` | | +| sowerConfig[0].container.image | string | `"quay.io/cdis/pelican-export:master"` | | | sowerConfig[0].container.memory-limit | string | `"12Gi"` | | | sowerConfig[0].container.name | string | `"job-task"` | | | sowerConfig[0].container.pull_policy | string | `"Always"` | | @@ -161,7 +163,7 @@ A Helm chart for gen3 sower | sowerConfig[1].container.env[8].name | string | `"SHEEPDOG"` | | | sowerConfig[1].container.env[8].valueFrom.secretKeyRef.key | string | `"sheepdog"` | | | sowerConfig[1].container.env[8].valueFrom.secretKeyRef.name | string | `"indexd-service-creds"` | | -| sowerConfig[1].container.image | string | `"quay.io/cdis/pelican-export:GPE-1252"` | | +| sowerConfig[1].container.image | string | `"quay.io/cdis/pelican-export:master"` | | | sowerConfig[1].container.memory-limit | string | `"12Gi"` | | | sowerConfig[1].container.name | string | `"job-task"` | | | sowerConfig[1].container.pull_policy | string | `"Always"` | | @@ -177,6 +179,7 @@ A Helm chart for gen3 sower | sowerConfig[1].restart_policy | string | `"Never"` | | | sowerConfig[1].volumes[0].name | string | `"pelican-creds-volume"` | | | sowerConfig[1].volumes[0].secret.secretName | string | `"pelicanservice-g3auto"` | | +| sowerjobsG3auto | string | `"{\n \"index-object-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\",\n \"indexd_user\": \"diirm\",\n \"indexd_password\": \"$indexdPassword\"\n },\n \"download-indexd-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n },\n \"get-dbgap-metadata\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n },\n \"ingest-metadata-manifest\": {\n \"job_requires\": {\n \"arborist_url\": \"http://arborist-service\",\n \"job_access_req\": []\n },\n \"bucket\": \"$bucketName\"\n }\n}\n"` | Additional configuration for Sower Jobs Passed in as a multiline string. This secret can be mounted in sowerConfig. | | strategy | map | `{"rollingUpdate":{"maxSurge":1,"maxUnavailable":0},"type":"RollingUpdate"}` | Rolling update deployment strategy | | strategy.rollingUpdate.maxSurge | int | `1` | Number of additional replicas to add during rollout. | | strategy.rollingUpdate.maxUnavailable | int | `0` | Maximum amount of pods that can be unavailable during the update. | diff --git a/helm/sower/templates/_helpers.tpl b/helm/sower/templates/_helpers.tpl index 1815359e..8f5b72b8 100644 --- a/helm/sower/templates/_helpers.tpl +++ b/helm/sower/templates/_helpers.tpl @@ -72,4 +72,11 @@ Create the name of the service account to use */}} {{- define "pelicanservice-g3auto" -}} {{- default "pelicanservice-g3auto" .Values.externalSecrets.pelicanserviceG3auto }} +{{- end }} + +{{/* + Sowerjobs g3 Auto Secrets Manager Name +*/}} +{{- define "sower-jobs-g3auto" -}} +{{- default "sower-jobs-g3auto" .Values.externalSecrets.sowerjobsG3auto }} {{- end }} \ No newline at end of file diff --git a/helm/sower/templates/external-secret.yaml b/helm/sower/templates/external-secret.yaml index 43132663..5296a3d0 100644 --- a/helm/sower/templates/external-secret.yaml +++ b/helm/sower/templates/external-secret.yaml @@ -16,4 +16,22 @@ spec: remoteRef: #name of secret in secrets manager key: {{include "pelicanservice-g3auto" .}} +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: sower-jobs-g3auto +spec: + refreshInterval: 5m + secretStoreRef: + name: {{include "common.SecretStore" .}} + kind: SecretStore + target: + name: sower-jobs-g3auto + creationPolicy: Owner + data: + - secretKey: config.json + remoteRef: + #name of secret in secrets manager + key: {{include "sower-jobs-g3auto" .}} {{- end }} \ No newline at end of file diff --git a/helm/sower/templates/sower-jobs-g3auto.yaml b/helm/sower/templates/sower-jobs-g3auto.yaml new file mode 100644 index 00000000..0a789e88 --- /dev/null +++ b/helm/sower/templates/sower-jobs-g3auto.yaml @@ -0,0 +1,10 @@ +{{- if or (not .Values.global.externalSecrets.deploy) (and .Values.global.externalSecrets.deploy .Values.externalSecrets.createK8sSowerJobsSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: sower-jobs-g3auto +type: Opaque +stringData: + config.json: | + {{ .Values.sowerjobsG3auto | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/helm/sower/values.yaml b/helm/sower/values.yaml index 8725100a..d3b7d5a9 100644 --- a/helm/sower/values.yaml +++ b/helm/sower/values.yaml @@ -55,7 +55,7 @@ global: dispatcherJobNum: "10" # -- (map) External Secrets settings. externalSecrets: - # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any manifestservice secrets you have deployed. + # -- (bool) Will use ExternalSecret resources to pull secrets from Secrets Manager instead of creating them locally. Be cautious as this will override any sower secrets you have deployed. deploy: false # -- (string) Will deploy a separate External Secret Store for this service. separateSecretStore: false @@ -65,12 +65,16 @@ metricsEnabled: false # -- (map) External Secrets settings. externalSecrets: - # -- (string) Will create the Helm "manifestservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + # -- (string) Will create the Helm "pelicanservice-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. createK8sPelicanServiceSecret: false # -- (string) Will override the name of the aws secrets manager secret. Default is "pelicanservice-g3auto" pelicanserviceG3auto: + # -- (string) Will create the Helm "sower-jobs-g3auto" secret even if Secrets Manager is enabled. This is helpful if you are wanting to use External Secrets for some, but not all secrets. + createK8sSowerJobsSecret: false + # -- (string) Will override the name of the aws secrets manager secret. Default is "sower-jobs-g3auto" + sowerjobsG3auto: -# -- (map) Secret information for Usersync and External Secrets. +# -- (map) Values for sower secrets and keys for External Secrets. secrets: # -- (str) AWS access key ID. Overrides global key. awsAccessKeyId: @@ -222,7 +226,7 @@ sowerConfig: action: export container: name: job-task - image: quay.io/cdis/pelican-export:GPE-1252 + image: quay.io/cdis/pelican-export:master pull_policy: Always env: - name: DICTIONARY_URL @@ -278,7 +282,7 @@ sowerConfig: action: export-files container: name: job-task - image: quay.io/cdis/pelican-export:GPE-1252 + image: quay.io/cdis/pelican-export:master pull_policy: Always env: - name: DICTIONARY_URL @@ -337,6 +341,41 @@ sowerConfig: secretName: pelicanservice-g3auto restart_policy: Never +# -- (string) Additional configuration for Sower Jobs Passed in as a multiline string. This secret can be mounted in sowerConfig. +sowerjobsG3auto: | + { + "index-object-manifest": { + "job_requires": { + "arborist_url": "http://arborist-service", + "job_access_req": [] + }, + "bucket": "$bucketName", + "indexd_user": "diirm", + "indexd_password": "$indexdPassword" + }, + "download-indexd-manifest": { + "job_requires": { + "arborist_url": "http://arborist-service", + "job_access_req": [] + }, + "bucket": "$bucketName" + }, + "get-dbgap-metadata": { + "job_requires": { + "arborist_url": "http://arborist-service", + "job_access_req": [] + }, + "bucket": "$bucketName" + }, + "ingest-metadata-manifest": { + "job_requires": { + "arborist_url": "http://arborist-service", + "job_access_req": [] + }, + "bucket": "$bucketName" + } + } + # -- (map) Service account to use or create. serviceAccount: # -- (bool) Specifies whether a service account should be created.