Pass through all service spec values.

Support healthchecks for gateway configurations without http enabled. Add tests for https and NodePort service with both https and http exposed. Add retries to conection tests to improve test resiliency Co-authored-by: Jan Waś <[email protected]>
trinodb · Jan 13, 2025 · 0f7cbd4 · 0f7cbd4
1 parent 014a177
commit 0f7cbd4
Show file tree

Hide file tree

Showing 11 changed files with 209 additions and 43 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,3 @@
 .idea/*
-tests/trino/cert.key
-tests/trino/cert.crt
+*/*/cert.key
+*/*/cert.crt
diff --git a/charts/gateway/README.md b/charts/gateway/README.md
@@ -39,6 +39,7 @@ A Helm chart for Trino Gateway
   ```
 * `config.serverConfig."node.environment"` - string, default: `"test"`
 * `config.serverConfig."http-server.http.port"` - int, default: `8080`
+* `config.serverConfig."http-server.http.enabled"` - bool, default: `true`
 * `config.dataStore.jdbcUrl` - string, default: `"jdbc:postgresql://localhost:5432/gateway"`  
 
   The connection details for the backend database for Trino Gateway and Trino query history
@@ -53,9 +54,27 @@ A Helm chart for Trino Gateway
 * `command` - list, default: `["java","-XX:MinRAMPercentage=80.0","-XX:MaxRAMPercentage=80.0","-jar","/usr/lib/trino/gateway-ha-jar-with-dependencies.jar","/etc/gateway/config.yaml"]`  
 
   Startup command for Trino Gateway process. Add additional Java options and other modifications as desired.
-* `service.type` - string, default: `"ClusterIP"`
-* `service.port` - int, default: `8080`
-* `service.annotations` - object, default: `{}`
+* `service` - object, default: `{"ports":[{"name":"gateway","protocol":"TCP"}],"type":"ClusterIP"}`  
+
+  Service for accessing the gateway. The contents of this dictionary are used  for the [service spec](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport). The `port` and `targetPort` of the first element of the ports list will automatically be set to the value of `config.serverConfig."http-server.http[s].port"`. If both https and http ports are defined the https port is used. In this case, an additional service for the http port must be configured manually. Additional ports, such as for JMX or a Java Agent can be configured by adding elements to the ports list. The selector is also automatically configured. All other values are passed through as is.  Example configuration for exposing both https and http:
+  ```yaml
+   service:
+     type: NodePort
+     ports:
+       - protocol: TCP
+         name: request
+         nodePort: 30443
+         # targetPort and port will automatically pulled from serverConfig.http-server.https.port
+       - protocol: TCP
+         name: gateway-http
+         nodePort: 30080
+         port: 8080
+         # targetPort must be explicitly set to the same value as serverConfig.http-server.http.port
+         targetPort: 8080
+  ```
+* `serviceName` - string, default: `"trino-gateway"`  
+
+  Set a custom name for the gateway service
 * `ingress.enabled` - bool, default: `false`
 * `ingress.className` - string, default: `""`
 * `ingress.annotations` - object, default: `{}`

diff --git a/charts/gateway/templates/NOTES.txt b/charts/gateway/templates/NOTES.txt
@@ -6,23 +6,19 @@ You can get the Trino Gateway endpoints by running these commands:
   {{- end }}
 {{- end }}
 {{- else if contains "NodePort" .Values.service.type }}
-  export REQUEST_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "request")].nodePort}' services {{ include "trino-gateway.fullname" . }})
-  export APP_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "app")].nodePort}' services {{ include "trino-gateway.fullname" . }})
-  export ADMIN_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "admin")].nodePort}' services {{ include "trino-gateway.fullname" . }})
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[0].nodePort}' svc trino-gateway)
   export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath='{.items[0].status.addresses[0].address}')
-  echo http://$NODE_IP:$REQUEST_NODE_PORT
+  echo http://$NODE_IP:$NODE_PORT
 {{- else if contains "LoadBalancer" .Values.service.type }}
      NOTE: It may take a few minutes for the LoadBalancer IP to be available.
-           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "trino-gateway.fullname" . }}'
-  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "trino-gateway.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w trino-gateway'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} trino-gateway --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:'{{ .Values.service.ports | first | get "port" }}'
 {{- else if contains "ClusterIP" .Values.service.type }}
   export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "trino-gateway.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export REQUEST_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "request")].containerPort}')
-  export APP_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "app")].containerPort}')
-  export ADMIN_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "admin")].containerPort}')
+  export PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[0].containerPort}')
   echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$REQUEST_PORT
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$PORT
 {{- end }}
 
 Happy Helming!
diff --git a/charts/gateway/templates/deployment.yaml b/charts/gateway/templates/deployment.yaml
@@ -1,3 +1,15 @@
+{{- $probePort := -1 }}
+{{- $probeScheme := "" }}
+{{- if index .Values "config" "serverConfig" "http-server.http.enabled" }}
+{{- $probePort =  index .Values "config" "serverConfig" "http-server.http.port" }}
+{{- $probeScheme = "HTTP" }}
+{{- else if index .Values "config" "serverConfig" "http-server.https.enabled" }}
+{{  $probePort =  index .Values "config" "serverConfig" "http-server.https.port" }}
+{{- $probeScheme = "HTTPS" }}
+{{- else }}
+  {{- fail "Error: Either https or http must be enabled in serverConfig!" }}
+{{- end }}
+
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -41,21 +53,30 @@ spec:
           envFrom:
             {{- toYaml .Values.envFrom | nindent 12}}
           ports:
-            - name: request
+            {{- if index .Values "config" "serverConfig" "http-server.http.enabled" }}
+            - name: http
               containerPort: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
               protocol: TCP
+            {{- end }}
+            {{- if index .Values "config" "serverConfig" "http-server.https.enabled" }}
+            - name: https
+              containerPort: {{ index .Values "config" "serverConfig" "http-server.https.port" }}
+              protocol: TCP
+            {{- end }}
           livenessProbe:
             httpGet:
-                path: /trino-gateway
-                port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
+              path: /trino-gateway
+              port: {{ $probePort }}
+              scheme: {{ $probeScheme }}
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
             timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
           readinessProbe:
             httpGet:
-                path: /trino-gateway
-                port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
+              path: /trino-gateway
+              port: {{ $probePort }}
+              scheme: {{ $probeScheme }}
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}

diff --git a/charts/gateway/templates/ingress.yaml b/charts/gateway/templates/ingress.yaml
@@ -1,6 +1,16 @@
+
 {{- if .Values.ingress.enabled -}}
-{{- $fullName := include "trino-gateway.fullname" . -}}
-{{- $svcPort := .Values.service.port -}}
+{{- $fullName := .Values.serviceName -}}
+{{- $svcPort := -1 }}
+{{- if index .Values "config" "serverConfig" "http-server.http.enabled" }}
+  {{- $svcPort =  index .Values "config" "serverConfig" "http-server.http.port" }}
+{{- else if index .Values "config" "serverConfig" "http-server.https.enabled" }}
+  {{  $svcPort =  index .Values "config" "serverConfig" "http-server.https.port" }}
+{{- else }}
+  {{- fail "Error: Either https or http must be enabled in serverConfig!" }}
+{{- end }}
+
+(index .Values "config" "serverConfig" "http-server.http.port") -}}
 {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
   {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
   {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}

diff --git a/charts/gateway/templates/service.yaml b/charts/gateway/templates/service.yaml
@@ -1,18 +1,28 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "trino-gateway.fullname" . }}
+  name: {{ .Values.serviceName }}
   labels:
     {{- include "trino-gateway.labels" . | nindent 4 }}
-  {{- with .Values.service.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  type: {{ .Values.service.type }}
-  ports:
-    - port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
-      protocol: TCP
-      name: request
-  selector:
-    {{- include "trino-gateway.selectorLabels" . | nindent 4 }}
+{{- $gatewayPort := "" }}
+{{- if index .Values "config" "serverConfig" "http-server.http.enabled" }}
+    {{- $gatewayPort = index .Values "config" "serverConfig" "http-server.http.port" }}
+{{- end }}
+{{- if index .Values "config" "serverConfig" "http-server.https.enabled" }}
+{{- $gatewayPort = index .Values "config" "serverConfig" "http-server.https.port" }}
+{{- end }}
+{{- if empty $gatewayPort }}
+{{- fail "Error: No port defined in serverConfig!" $gatewayPort }}
+{{- end}}
+{{- $portDefault := dict "port" $gatewayPort "targetPort" $gatewayPort }}
+{{- $portValues := .Values.service.ports | default list | first | default $portDefault}}
+{{- $_0 := set $portValues "port" $gatewayPort}}
+{{- $_1 := set $portValues "targetPort" $gatewayPort}}
+{{- $ports := list $portValues }}
+{{- $additionalPorts := .Values.service.ports | default list | rest }}
+{{- $allPorts := concat $ports $additionalPorts}}
+{{- $spec := .Values.service }}
+{{- $_2 := set $spec "ports" $allPorts }}
+{{- $selectorLabels := include "trino-gateway.selectorLabels" . | fromYaml }}
+{{- $_3 := set $spec "selector" $selectorLabels }}
+spec: {{ $spec | toYaml | nindent 2}}
diff --git a/charts/gateway/templates/tests/test-connection.yaml b/charts/gateway/templates/tests/test-connection.yaml
@@ -37,14 +37,29 @@ spec:
         - name: persistence-sql
           mountPath: /etc/persistence
   containers:
-    - name: wget
-      image: busybox
+    - name: curl
+      image: alpine
+      env:
+        - name: NODE_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
       # Get the list of backends, which should return an empty list, "[]". For this test to pass
       # the gateway must successfully connect to an initialized backend database
       command:
         - "sh"
         - "-c"
-        - '[ "$(wget {{ include "trino-gateway.fullname" . }}:{{ .Values.service.port }}/entity/GATEWAY_BACKEND -O -)" = "[]" ]'
+        - |
+          apk add curl
+          {{- if eq .Values.service.type "NodePort" -}}
+             && [ "$(curl -k --retry 3 --retry-all-errors --connect-timeout 5 --retry-delay 5 https://${NODE_IP}:30443/entity/GATEWAY_BACKEND )" = "[]" ] && [ "$(curl --retry 3 --retry-all-errors --connect-timeout 5 --retry-delay 5 http://${NODE_IP}:30080/entity/GATEWAY_BACKEND )" = "[]" ]
+          {{- end }}
+          {{- if index .Values "config" "serverConfig" "http-server.https.enabled" -}}
+             && [ "$(curl -k --retry 3 --retry-all-errors --connect-timeout 5 --retry-delay 5 -v https://{{ .Values.serviceName }}:8443/entity/GATEWAY_BACKEND )" = "[]" ]
+          {{- end }}
+          {{- if index .Values "config" "serverConfig" "http-server.http.enabled" -}}
+             && [ "$(curl --retry 3 --retry-all-errors --connect-timeout 5 --retry-delay 5 -v http://{{ .Values.serviceName }}:8080/entity/GATEWAY_BACKEND )" = "[]" ]
+          {{- end }}
   volumes:
     - name: persistence-sql
       emptyDir:

diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml
@@ -34,6 +34,7 @@ config:
   serverConfig:
     node.environment: test
     http-server.http.port: 8080
+    http-server.http.enabled: true
   dataStore:
     # -- The connection details for the backend database for Trino Gateway and Trino query history
     jdbcUrl: jdbc:postgresql://localhost:5432/gateway
@@ -58,10 +59,42 @@ command:
   - "/usr/lib/trino/gateway-ha-jar-with-dependencies.jar"
   - "/etc/gateway/config.yaml"
 
+# -- Service for accessing the gateway. The contents of this dictionary are used
+#  for the [service spec](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport).
+# The `port` and `targetPort` of the first element
+# of the ports list will automatically be set to the value of
+# `config.serverConfig."http-server.http[s].port"`. If both https and http ports are defined
+# the https port is used. In this case, an additional service for the http port must be
+# configured manually. Additional ports, such as for JMX or a Java Agent
+# can be configured by adding elements to the ports list. The selector is
+# also automatically configured. All other values are passed through as is.
+#
+# Example configuration for exposing both https and http:
+# @raw
+# ```yaml
+#  service:
+#    type: NodePort
+#    ports:
+#      - protocol: TCP
+#        name: request
+#        nodePort: 30443
+#        # targetPort and port will automatically pulled from serverConfig.http-server.https.port
+#      - protocol: TCP
+#        name: gateway-http
+#        nodePort: 30080
+#        port: 8080
+#        # targetPort must be explicitly set to the same value as serverConfig.http-server.http.port
+#        targetPort: 8080
+# ```
+
 service:
   type: ClusterIP
-  port: 8080
-  annotations: {}
+  ports:
+    - protocol: TCP
+      name: gateway
+
+# -- Set a custom name for the gateway service
+serviceName: trino-gateway
 
 ingress:
   enabled: false

diff --git a/tests/gateway/test-https.yaml b/tests/gateway/test-https.yaml
@@ -0,0 +1,28 @@
+command:
+  - "/bin/sh"
+  - "-c"
+  - |
+    cat /etc/certificates/tls.crt /etc/certificates/tls.key > /etc/scratch/tls.pem && \
+    java -XX:MinRAMPercentage=80.0 -XX:MaxRAMPercentage=80.0 -jar /usr/lib/trino/gateway-ha-jar-with-dependencies.jar /etc/gateway/config.yaml
+
+config:
+  serverConfig:
+    http-server.http.enabled: false
+    http-server.https.enabled: true
+    http-server.https.port: 8443
+    http-server.https.keystore.path: /etc/scratch/tls.pem
+
+volumes:
+  - name: certificates
+    secret:
+      secretName: certificates
+  - name: scratch
+    emptyDir:
+      sizeLimit: 10Mi
+
+volumeMounts:
+  - name: certificates
+    mountPath: /etc/certificates
+    readOnly: true
+  - name: scratch
+    mountPath: /etc/scratch
diff --git a/tests/gateway/test-nodeport.yaml b/tests/gateway/test-nodeport.yaml
@@ -0,0 +1,19 @@
+config:
+  serverConfig:
+    http-server.http.enabled: true
+    http-server.http.port: 8080
+    http-server.https.enabled: true
+    http-server.https.port: 8443
+    http-server.https.keystore.path: /etc/scratch/tls.pem
+
+service:
+  type: NodePort
+  ports:
+    - protocol: TCP
+      name: request
+      nodePort: 30443
+    - protocol: TCP
+      name: gateway-http
+      nodePort: 30080
+      port: 8080
+      targetPort: 8080
diff --git a/tests/gateway/test.sh b/tests/gateway/test.sh
@@ -5,13 +5,19 @@ set -euo pipefail
 declare -A testCases=(
     [complete_values]="--values test-values.yaml"
     [env_from]="--values test-values-with-env.yaml"
+    [nodeport]="--values test-values.yaml --values test-https.yaml --values test-nodeport.yaml"
+    [https]="--values test-values.yaml --values test-https.yaml"
 )
 
 declare -A testCaseCharts=(
     [complete_values]="../../charts/gateway"
     [env_from]="../../charts/gateway"
+    [nodeport]="../../charts/gateway"
+    [https]="../../charts/gateway"
 )
 
+TEST_NAMES=(complete_values env_from nodeport https)
+
 function join_by {
     local d=${1-} f=${2-}
     if shift 2; then
@@ -24,13 +30,21 @@ NAMESPACE=trino-gateway-$(LC_ALL=C tr -dc 'a-z0-9' </dev/urandom | head -c 6 ||
 DB_NAMESPACE=postgres-gateway
 kubectl create namespace "${NAMESPACE}" --dry-run=client --output yaml | kubectl apply --filename -
 kubectl create namespace "${DB_NAMESPACE}" --dry-run=client --output yaml | kubectl apply --filename -
+
+echo 1>&2 "Generating a self-signed TLS certificate"
+NODE_IP=$(kubectl get nodes -o json  -o jsonpath='{.items[0].status.addresses[0].address}')
+openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
+    -subj "/O=Trino Software Foundation" \
+    -addext "subjectAltName=DNS:trino-gateway,DNS:localhost,DNS:*.$NAMESPACE,DNS:*.$NAMESPACE.svc,DNS:*.$NAMESPACE.svc.cluster.local,IP:127.0.0.1,IP:${NODE_IP}" \
+    -keyout cert.key -out cert.crt
+kubectl -n "$NAMESPACE" create secret tls certificates --cert=cert.crt --key=cert.key --dry-run=client --output yaml | kubectl apply --filename -
+
 HELM_EXTRA_SET_ARGS=
 CT_ARGS=(
     --skip-clean-up
-    --helm-extra-args="--timeout 2m"
+    --helm-extra-args="--timeout 4m"
 )
 CLEANUP_NAMESPACE=true
-TEST_NAMES=(complete_values env_from)
 
 usage() {
     cat <<EOF 1>&2
@@ -96,6 +110,7 @@ for test_name in "${TEST_NAMES[@]}"; do
     echo 1>&2 ""
     echo 1>&2 "🧪 Running test $test_name"
     echo 1>&2 ""
+    HELM_EXTRA_SET_ARGS="$HELM_EXTRA_SET_ARGS --set=serviceName=trino-gateway-${test_name//_/-}"
     if ! time ct install "${CT_ARGS[@]}" --charts="${testCaseCharts[$test_name]}" --helm-extra-set-args "$HELM_EXTRA_SET_ARGS ${testCases[$test_name]}"; then
         echo 1>&2 "❌ Test $test_name failed"
         echo 1>&2 "Test logs:"