Elasticsearch 7 support (uc-cdis#143)

* Initial ES7 and ETL integration * Update gitops.json * Add test-connection pods * Add test-connection pods * Add chart install + test * Add chart install + test * Add chart install + test * Add chart install + test * Add chart install + test * Add agg-mds sync job * Bump versions * Update gh actions * Update gh actions * Fix linting issues * Update KIND doc * Update KIND doc * Update ETL and ES values * Delete local ES chart and update ETL job * move aggMDS to cron that can be triggered
ACED-IDP · Feb 16, 2024 · d5f63af · d5f63af
1 parent 28095ed
commit d5f63af
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 40 deletions.
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2023-07-27T21:47:16Z",
+  "generated_at": "2023-11-20T21:39:41Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -374,7 +374,7 @@
         "hashed_secret": "1740c48fa3141d4851b14f97e3bc0f46f7670672",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 119,
+        "line_number": 127,
         "type": "Secret Keyword"
       }
     ],

diff --git a/helm/gen3/Chart.yaml b/helm/gen3/Chart.yaml
@@ -84,7 +84,7 @@ dependencies:
   repository: "file://../ssjdispatcher"
   condition: ssjdispatcher.enabled
 - name: sower
-  version: "0.1.6"
+  version: 0.1.7
   condition: sower.enabled
   repository: "file://../sower"
 - name: wts
@@ -115,7 +115,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.19
+version: 0.1.21
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/gen3/README.md b/helm/gen3/README.md
@@ -1,6 +1,6 @@
 # gen3
 
-![Version: 0.1.19](https://img.shields.io/badge/Version-0.1.19-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.21](https://img.shields.io/badge/Version-0.1.21-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 Helm chart to deploy Gen3 Data Commons
 
@@ -18,28 +18,28 @@ Helm chart to deploy Gen3 Data Commons
 
 | Repository | Name | Version |
 |------------|------|---------|
-| file://../ambassador | ambassador | 0.1.8 |
-| file://../arborist | arborist | 0.1.8 |
-| file://../argo-wrapper | argo-wrapper | 0.1.4 |
-| file://../audit | audit | 0.1.9 |
-| file://../aws-es-proxy | aws-es-proxy | 0.1.6 |
-| file://../common | common | 0.1.7 |
-| file://../elasticsearch | elasticsearch | 0.1.5 |
-| file://../fence | fence | 0.1.13 |
-| file://../guppy | guppy | 0.1.8 |
-| file://../hatchery | hatchery | 0.1.6 |
-| file://../indexd | indexd | 0.1.10 |
-| file://../manifestservice | manifestservice | 0.1.10 |
-| file://../metadata | metadata | 0.1.8 |
-| file://../peregrine | peregrine | 0.1.9 |
-| file://../pidgin | pidgin | 0.1.7 |
-| file://../portal | portal | 0.1.7 |
-| file://../requestor | requestor | 0.1.8 |
-| file://../revproxy | revproxy | 0.1.10 |
-| file://../sheepdog | sheepdog | 0.1.10 |
-| file://../sower | sower | 0.1.6 |
-| file://../ssjdispatcher | ssjdispatcher | 0.1.6 |
-| file://../wts | wts | 0.1.10 |
+| file://../ambassador | ambassador | 0.1.9 |
+| file://../arborist | arborist | 0.1.9 |
+| file://../argo-wrapper | argo-wrapper | 0.1.5 |
+| file://../audit | audit | 0.1.10 |
+| file://../aws-es-proxy | aws-es-proxy | 0.1.7 |
+| file://../common | common | 0.1.8 |
+| file://../etl | etl | 0.1.0 |
+| file://../fence | fence | 0.1.14 |
+| file://../guppy | guppy | 0.1.9 |
+| file://../hatchery | hatchery | 0.1.7 |
+| file://../indexd | indexd | 0.1.11 |
+| file://../manifestservice | manifestservice | 0.1.11 |
+| file://../metadata | metadata | 0.1.9 |
+| file://../peregrine | peregrine | 0.1.10 |
+| file://../pidgin | pidgin | 0.1.8 |
+| file://../portal | portal | 0.1.8 |
+| file://../requestor | requestor | 0.1.9 |
+| file://../revproxy | revproxy | 0.1.12 |
+| file://../sheepdog | sheepdog | 0.1.11 |
+| file://../sower | sower | 0.1.7 |
+| file://../ssjdispatcher | ssjdispatcher | 0.1.7 |
+| file://../wts | wts | 0.1.11 |
 | https://charts.bitnami.com/bitnami | postgresql | 11.9.13 |
 | https://helm.elastic.co | elasticsearch | 7.10.2 |
 

diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml
@@ -110,15 +110,9 @@ aws-es-proxy:
     # -- (str) AWS secret access key for aws-es-proxy
     awsSecretAccessKey: ""
 
-etl-pod:
-  # -- (bool) Whether to deploy the etl-pod subchart.
+etl:
+  # -- (bool) Whether to deploy the etl subchart.
   enabled: true
-  # -- (map) Docker image information.
-  image:
-    # -- (string) The Docker image repository for the etl-po service.
-    repository:
-    # -- (string) Overrides the image tag whose default is the chart appVersion.
-    tag:
 
 fence:
   # -- (bool) Whether to deploy the fence subchart.
@@ -398,4 +392,14 @@ postgresql:
   primary:
     persistence:
       # -- (bool) Option to persist the dbs data.
-      enabled: true
+      enabled: false
+
+elasticsearch:
+  clusterName: gen3-elasticsearch
+  maxUnavailable: 0
+  singleNode: true
+  replicas: 1
+  clusterHealthCheckParams: "wait_for_status=yellow&timeout=1s"
+  esConfig:
+    elasticsearch.yml: |
+      # Here we can add elasticsearch config
diff --git a/helm/guppy/templates/deployment.yaml b/helm/guppy/templates/deployment.yaml
@@ -74,8 +74,6 @@ spec:
               value: {{ .Values.global.tierAccessLevel }}
             - name: TIER_ACCESS_LIMIT
               value: {{ .Values.global.tierAccessLimit | quote }}
-
-
           {{- with .Values.volumeMounts }}
           volumeMounts:
             {{- toYaml . | nindent 10 }}

diff --git a/helm/sower/Chart.yaml b/helm/sower/Chart.yaml
@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.6
+version: 0.1.7
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/helm/sower/README.md b/helm/sower/README.md
@@ -1,6 +1,6 @@
 # sower
 
-![Version: 0.1.6](https://img.shields.io/badge/Version-0.1.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
+![Version: 0.1.7](https://img.shields.io/badge/Version-0.1.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square)
 
 A Helm chart for gen3 sower