Pass through all service spec values.

Support healthchecks for gateway configurations without http enabled. Add tests for https and NodePort service with both https and http exposed. Co-authored-by: Jan Waś <[email protected]>
trinodb · Dec 18, 2024 · 36b728f · 36b728f
1 parent 706f94c
commit 36b728f
Show file tree

Hide file tree

Showing 11 changed files with 189 additions and 37 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,3 @@
 .idea/*
-tests/trino/cert.key
-tests/trino/cert.crt
+*/*/cert.key
+*/*/cert.crt
diff --git a/charts/gateway/README.md b/charts/gateway/README.md
@@ -39,6 +39,7 @@ A Helm chart for Trino Gateway
   ```
 * `config.serverConfig."node.environment"` - string, default: `"test"`
 * `config.serverConfig."http-server.http.port"` - int, default: `8080`
+* `config.serverConfig."http-server.http.enabled"` - bool, default: `true`
 * `config.dataStore.jdbcUrl` - string, default: `"jdbc:postgresql://localhost:5432/gateway"`  
 
   The connection details for the backend database for Trino Gateway and Trino query history
@@ -53,9 +54,23 @@ A Helm chart for Trino Gateway
 * `command` - list, default: `["java","-XX:MinRAMPercentage=80.0","-XX:MaxRAMPercentage=80.0","-jar","/usr/lib/trino/gateway-ha-jar-with-dependencies.jar","/etc/gateway/config.yaml"]`  
 
   Startup command for Trino Gateway process. Add additional Java options and other modifications as desired.
-* `service.type` - string, default: `"ClusterIP"`
-* `service.port` - int, default: `8080`
-* `service.annotations` - object, default: `{}`
+* `service` - object, default: `{"ports":[{"name":"gateway","protocol":"TCP"}],"type":"ClusterIP"}`  
+
+  Service for accessing the gateway. The contents of this dictionary are used  for the [service spec](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport). `The `port` and `targetPort` of the first element of the ports list will automatically be set to the value of `config.serverConfig."http-server.http[s].port"`. If both https and http ports are defined the https port is used. In this case, an additional service for the http port must be configured manually. Additional ports, such as for JMX or a Java Agent can be configured by adding elements to the ports list. The selector is also automatically configured. All other values are passed through as is.  Example configuration for exposing both https and http:
+  ```yaml
+   service:
+     type: NodePort
+     ports:
+       - protocol: TCP
+         name: request
+         nodePort: 30443
+         # targetPort and port will automatically pulled from serverConfig.http-server.https.port
+       - protocol: TCP
+         name: gateway-http
+         nodePort: 30080
+         port: 8080
+         targetPort: 8080
+         # targetPort and port should be manually set to serverConfig.http-server.http.port
 * `ingress.enabled` - bool, default: `false`
 * `ingress.className` - string, default: `""`
 * `ingress.annotations` - object, default: `{}`

diff --git a/charts/gateway/templates/NOTES.txt b/charts/gateway/templates/NOTES.txt
@@ -6,23 +6,19 @@ You can get the Trino Gateway endpoints by running these commands:
   {{- end }}
 {{- end }}
 {{- else if contains "NodePort" .Values.service.type }}
-  export REQUEST_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "request")].nodePort}' services {{ include "trino-gateway.fullname" . }})
-  export APP_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "app")].nodePort}' services {{ include "trino-gateway.fullname" . }})
-  export ADMIN_NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[?(@.name == "admin")].nodePort}' services {{ include "trino-gateway.fullname" . }})
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath='{.spec.ports[0].nodePort}' services {{ include "trino-gateway.fullname" . }})
   export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath='{.items[0].status.addresses[0].address}')
-  echo http://$NODE_IP:$REQUEST_NODE_PORT
+  echo http://$NODE_IP:$NODE_PORT
 {{- else if contains "LoadBalancer" .Values.service.type }}
      NOTE: It may take a few minutes for the LoadBalancer IP to be available.
            You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "trino-gateway.fullname" . }}'
   export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "trino-gateway.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
-  echo http://$SERVICE_IP:{{ .Values.service.port }}
+  echo http://$SERVICE_IP:'{{ .Values.service.ports | first | get "port" }}'
 {{- else if contains "ClusterIP" .Values.service.type }}
   export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "trino-gateway.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
-  export REQUEST_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "request")].containerPort}')
-  export APP_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "app")].containerPort}')
-  export ADMIN_PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[?(@.name == "admin")].containerPort}')
+  export PORT=$(kubectl get pod --namespace test $POD_NAME -o jsonpath='{.spec.containers[0].ports[0].containerPort}')
   echo "Visit http://127.0.0.1:8080 to use your application"
-  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$REQUEST_PORT
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$PORT
 {{- end }}
 
 Happy Helming!
diff --git a/charts/gateway/templates/deployment.yaml b/charts/gateway/templates/deployment.yaml
@@ -41,21 +41,34 @@ spec:
           envFrom:
             {{- toYaml .Values.envFrom | nindent 12}}
           ports:
-            - name: request
+            {{- if index .Values "config" "serverConfig" "http-server.http.port" }}
+            - name: http
               containerPort: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
               protocol: TCP
+            {{- end }}
+            {{- if index .Values "config" "serverConfig" "http-server.https.port" }}
+            - name: https
+              containerPort: {{ index .Values "config" "serverConfig" "http-server.https.port" }}
+              protocol: TCP
+            {{- end }}
           livenessProbe:
             httpGet:
-                path: /trino-gateway
-                port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
+              path: /trino-gateway
+              port: {{ coalesce (index .Values "config" "serverConfig" "http-server.https.port") (index .Values "config" "serverConfig" "http-server.http.port") }}
+              {{- if index .Values "config" "serverConfig" "http-server.https.port" }}
+              scheme: HTTPS
+              {{- end }}
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
             timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
           readinessProbe:
             httpGet:
-                path: /trino-gateway
-                port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
+              path: /trino-gateway
+              port: {{ coalesce (index .Values "config" "serverConfig" "http-server.https.port") (index .Values "config" "serverConfig" "http-server.http.port") }}
+              {{- if index .Values "config" "serverConfig" "http-server.https.port" }}
+              scheme: HTTPS
+              {{- end }}
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}

diff --git a/charts/gateway/templates/ingress.yaml b/charts/gateway/templates/ingress.yaml
@@ -1,6 +1,7 @@
 {{- if .Values.ingress.enabled -}}
 {{- $fullName := include "trino-gateway.fullname" . -}}
-{{- $svcPort := .Values.service.port -}}
+{{- $svcPort := coalesce (index .Values "config" "serverConfig" "http-server.https.port")
+(index .Values "config" "serverConfig" "http-server.http.port") -}}
 {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
   {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
   {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}

diff --git a/charts/gateway/templates/service.yaml b/charts/gateway/templates/service.yaml
@@ -4,15 +4,25 @@ metadata:
   name: {{ include "trino-gateway.fullname" . }}
   labels:
     {{- include "trino-gateway.labels" . | nindent 4 }}
-  {{- with .Values.service.annotations }}
-  annotations:
-    {{- toYaml . | nindent 4 }}
-  {{- end }}
-spec:
-  type: {{ .Values.service.type }}
-  ports:
-    - port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
-      protocol: TCP
-      name: request
-  selector:
-    {{- include "trino-gateway.selectorLabels" . | nindent 4 }}
+{{- $gatewayPort := "" }}
+{{- if index .Values "config" "serverConfig" "http-server.http.enabled" }}
+    {{- $gatewayPort = index .Values "config" "serverConfig" "http-server.http.port" }}
+{{- end }}
+{{- if index .Values "config" "serverConfig" "http-server.https.enabled" }}
+{{- $gatewayPort = index .Values "config" "serverConfig" "http-server.https.port" }}
+{{- end }}
+{{- if empty $gatewayPort }}
+{{- fail "Error: No port defined in serverConfig!" $gatewayPort }}
+{{- end}}
+{{- $portDefault := dict "port" $gatewayPort "targetPort" $gatewayPort }}
+{{- $portValues := .Values.service.ports | default list | first | default $portDefault}}
+{{- $_0 := set $portValues "port" $gatewayPort}}
+{{- $_1 := set $portValues "targetPort" $gatewayPort}}
+{{- $ports := list $portValues }}
+{{- $additionalPorts := .Values.service.ports | default list | rest }}
+{{- $allPorts := concat $ports $additionalPorts}}
+{{- $spec := .Values.service }}
+{{- $_2 := set $spec "ports" $allPorts }}
+{{- $selectorLabels := include "trino-gateway.selectorLabels" . | fromYaml }}
+{{- $_3 := set $spec "selector" $selectorLabels }}
+spec: {{ $spec | toYaml | nindent 2}}
diff --git a/charts/gateway/templates/tests/test-connection.yaml b/charts/gateway/templates/tests/test-connection.yaml
@@ -38,13 +38,20 @@ spec:
           mountPath: /etc/persistence
   containers:
     - name: wget
-      image: busybox
+      image: alpine
       # Get the list of backends, which should return an empty list, "[]". For this test to pass
       # the gateway must successfully connect to an initialized backend database
       command:
         - "sh"
         - "-c"
-        - '[ "$(wget {{ include "trino-gateway.fullname" . }}:{{ .Values.service.port }}/entity/GATEWAY_BACKEND -O -)" = "[]" ]'
+        - |
+          [ 1 = 1 ]
+          {{- if index .Values "config" "serverConfig" "http-server.https.enabled" -}}
+             && [ "$(wget --no-check-certificate https://{{ include "trino-gateway.fullname" . }}:{{ index .Values "config" "serverConfig" "http-server.https.port"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ]
+          {{- end }}
+          {{- if index .Values "config" "serverConfig" "http-server.http.enabled" -}}
+             && [ "$(wget http://{{ include "trino-gateway.fullname" . }}:{{ index .Values "config" "serverConfig" "http-server.http.port"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ]
+          {{- end }}
   volumes:
     - name: persistence-sql
       emptyDir:

diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml
@@ -34,6 +34,7 @@ config:
   serverConfig:
     node.environment: test
     http-server.http.port: 8080
+    http-server.http.enabled: true
   dataStore:
     # -- The connection details for the backend database for Trino Gateway and Trino query history
     jdbcUrl: jdbc:postgresql://localhost:5432/gateway
@@ -58,10 +59,38 @@ command:
   - "/usr/lib/trino/gateway-ha-jar-with-dependencies.jar"
   - "/etc/gateway/config.yaml"
 
+# -- Service for accessing the gateway. The contents of this dictionary are used
+#  for the [service spec](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport).
+# `The `port` and `targetPort` of the first element
+# of the ports list will automatically be set to the value of
+# `config.serverConfig."http-server.http[s].port"`. If both https and http ports are defined
+# the https port is used. In this case, an additional service for the http port must be
+# configured manually. Additional ports, such as for JMX or a Java Agent
+# can be configured by adding elements to the ports list. The selector is
+# also automatically configured. All other values are passed through as is.
+#
+# Example configuration for exposing both https and http:
+# @raw
+# ```yaml
+#  service:
+#    type: NodePort
+#    ports:
+#      - protocol: TCP
+#        name: request
+#        nodePort: 30443
+#        # targetPort and port will automatically pulled from serverConfig.http-server.https.port
+#      - protocol: TCP
+#        name: gateway-http
+#        nodePort: 30080
+#        port: 8080
+#        targetPort: 8080
+#        # targetPort and port should be manually set to serverConfig.http-server.http.port
+
 service:
   type: ClusterIP
-  port: 8080
-  annotations: {}
+  ports:
+    - protocol: TCP
+      name: gateway
 
 ingress:
   enabled: false

diff --git a/tests/gateway/test-https.yaml b/tests/gateway/test-https.yaml
@@ -0,0 +1,46 @@
+command:
+  - "/bin/sh"
+  - "-c"
+  - |
+    cat /etc/certificates/tls.crt /etc/certificates/tls.key > /etc/scratch/tls.pem && \
+    java -XX:MinRAMPercentage=80.0 -XX:MaxRAMPercentage=80.0 -jar /usr/lib/trino/gateway-ha-jar-with-dependencies.jar /etc/gateway/config.yaml
+
+config:
+  serverConfig:
+    http-server.http.enabled: false
+    http-server.https.enabled: true
+    http-server.https.port: 8443
+    http-server.https.keystore.path: /etc/scratch/tls.pem
+
+volumes:
+  - name: certificates
+    secret:
+      secretName: certificates
+  - name: scratch
+    emptyDir:
+      sizeLimit: 10Mi
+
+volumeMounts:
+  - name: certificates
+    mountPath: /etc/certificates
+    readOnly: true
+  - name: scratch
+    mountPath: /etc/scratch
+
+ingress:
+  enabled: true
+  className: ""
+  annotations:
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+    cert-manager.io/cluster-issuer: acme
+    kubernetes.io/ingress.class: nginx
+  hosts:
+    - host: request.tg.starburst-customer-success.com
+      paths:
+        - path: /
+          pathType: ImplementationSpecific
+  tls:
+    - secretName: tls-secret-tg-green
+      hosts:
+        - request.tg.starburst-customer-success.com
diff --git a/tests/gateway/test-nodeport.yaml b/tests/gateway/test-nodeport.yaml
@@ -0,0 +1,19 @@
+config:
+  serverConfig:
+    http-server.http.enabled: true
+    http-server.http.port: 8080
+    http-server.https.enabled: true
+    http-server.https.port: 8443
+    http-server.https.keystore.path: /etc/scratch/tls.pem
+
+service:
+  type: NodePort
+  ports:
+    - protocol: TCP
+      name: request
+      nodePort: 30443
+    - protocol: TCP
+      name: gateway-http
+      nodePort: 30080
+      port: 8080
+      targetPort: 8080
diff --git a/tests/gateway/test.sh b/tests/gateway/test.sh
@@ -5,13 +5,19 @@ set -euo pipefail
 declare -A testCases=(
     [complete_values]="--values test-values.yaml"
     [env_from]="--values test-values-with-env.yaml"
+    [nodeport]="--values test-values.yaml --values test-https.yaml --values test-nodeport.yaml"
+    [https]="--values test-values.yaml --values test-https.yaml"
 )
 
 declare -A testCaseCharts=(
     [complete_values]="../../charts/gateway"
     [env_from]="../../charts/gateway"
+    [nodeport]="../../charts/gateway"
+    [https]="../../charts/gateway"
 )
 
+TEST_NAMES=(complete_values env_from nodeport https)
+
 function join_by {
     local d=${1-} f=${2-}
     if shift 2; then
@@ -24,13 +30,23 @@ NAMESPACE=trino-gateway-$(LC_ALL=C tr -dc 'a-z0-9' </dev/urandom | head -c 6 ||
 DB_NAMESPACE=postgres-gateway
 kubectl create namespace "${NAMESPACE}" --dry-run=client --output yaml | kubectl apply --filename -
 kubectl create namespace "${DB_NAMESPACE}" --dry-run=client --output yaml | kubectl apply --filename -
+
+NODE_IP=$(kubectl get nodes -o json  -o jsonpath='{.items[0].status.addresses[0].address}')
+kubectl -n "${NAMESPACE}" create secret generic node-ip --from-literal=NODE_IP="${NODE_IP}"
+
+echo 1>&2 "Generating a self-signed TLS certificate"
+openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
+    -subj "/O=Trino Software Foundation" \
+    -addext "subjectAltName=DNS:localhost,DNS:*.$NAMESPACE,DNS:*.$NAMESPACE.svc,DNS:*.$NAMESPACE.svc.cluster.local,IP:127.0.0.1,IP:${NODE_IP}" \
+    -keyout cert.key -out cert.crt
+kubectl -n "$NAMESPACE" create secret tls certificates --cert=cert.crt --key=cert.key --dry-run=client --output yaml | kubectl apply --filename -
+
 HELM_EXTRA_SET_ARGS=
 CT_ARGS=(
     --skip-clean-up
     --helm-extra-args="--timeout 2m"
 )
 CLEANUP_NAMESPACE=true
-TEST_NAMES=(complete_values env_from)
 
 usage() {
     cat <<EOF 1>&2