Add Kubernetes e2e tests using kind

.github/workflows/test.yml

@@ -245,3 +245,41 @@ jobs:
       - name: Test ${{ matrix.search-version }}
         run: yarn --silent test:${{ matrix.search-version }} --node-version ${{ matrix.node-version }}
         working-directory: ./e2e
+
+  e2e-k8s-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      # opensearch is finiky, keep testing others if it fails
+      fail-fast: false
+      matrix:
+        node-version: [16.19.1, 18.16.0]
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+
+      - name: Setup Node ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'yarn'
+
+      # we login to docker to avoid docker pull limit rates
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Install and build packages
+        run: yarn setup
+        env:
+          YARN_SETUP_ARGS: "--prod=false --silent"
+
+      - name: Install Kind and Kubectl
+        uses: helm/[email protected]
+        with:
+          install_only: "true"
+
+      - name: Test k8s elasticsearch7
+        run: yarn --silent test:k8s --node-version ${{ matrix.node-version }}
+        working-directory: ./e2e

docker-compose.yml

@@ -73,16 +73,32 @@ services:
     cap_add:
       - IPC_LOCK
   kafka:
-    image: terascope/kafka-zookeeper:v1.1.0
-    platform: linux/amd64  # platform specified to force amd64 on arm MacOS docker
+    image: confluentinc/cp-kafka:7.2.0 # Has kafka 3.2.0
+    ports:
+      - "9094:9094"
+    restart: unless-stopped
+    depends_on:
+      - zookeeper
+    networks:
+      - cluster
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_LISTENERS: INTERNAL://kafka:9092,OUTSIDE://0.0.0.0:9094
+      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,OUTSIDE://localhost:9094
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+  zookeeper:
+    image: confluentinc/cp-zookeeper:7.2.0
     ports:
       - "2181:2181"
-      - "9092:9092"
     restart: unless-stopped
     networks:
       - cluster
-    volumes:
-        - kafka-data:/kafka
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
   minio:
     image: minio/minio:RELEASE.2023-09-30T07-02-29Z
     ports:

e2e/jest.config.js

@@ -2,6 +2,10 @@
 
 const config = require('../jest.config.base')(__dirname);
 
+// TODO: update arrays to run tests specific to platform.
+// First array is for tests skipped in kubernetes.
+// Second array is for tests skipped in native.
+config.testPathIgnorePatterns = process.env.TEST_PLATFORM === 'kubernetes' ? ['data/recovery-spec', 'cluster/worker-allocation-spec', 'cluster/state-spec'] : [];
 config.collectCoverage = false;
 delete config.transform;
 module.exports = config;

e2e/k8s/elasticsearchDeployment.yaml

@@ -0,0 +1,45 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: elasticsearch
+  labels:
+      app.kubernetes.io/name: elasticsearch
+      app.kubernetes.io/component: master
+spec:
+  replicas: 1
+  selector:
+      matchLabels:
+          app.kubernetes.io/name: elasticsearch
+          app.kubernetes.io/component: master
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: elasticsearch
+        app.kubernetes.io/component: master
+    spec:
+      containers:
+      - name: elasticsearch
+        image: elasticsearch:7.9.3
+        ports:
+        - containerPort: 9200
+        env:
+        - name: ES_JAVA_OPTS
+          value: "-Xms512m -Xmx512m"
+        - name: discovery.type
+          value: single-node
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: elasticsearch
+  labels:
+    app.kubernetes.io/name: elasticsearch
+spec:
+  selector:
+      app.kubernetes.io/name: elasticsearch
+      app.kubernetes.io/component: master
+  ports:
+  - port: 9200
+    targetPort: 9200
+    nodePort: 30200  # the external port teraslice can be accessed on
+  type: NodePort

e2e/k8s/kafkaDeployment.yaml

@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cpkafka
+  labels:
+      app.kubernetes.io/name: cpkafka
+      app.kubernetes.io/component: master
+spec:
+  replicas: 1
+  selector:
+      matchLabels:
+          app.kubernetes.io/name: cpkafka
+          app.kubernetes.io/component: master
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: cpkafka
+        app.kubernetes.io/component: master
+    spec:
+      containers:
+      - name: cpkafka
+        image: confluentinc/cp-kafka:7.1.9
+        env:
+          - name: KAFKA_BROKER_ID
+            value: "1"
+          - name: KAFKA_ZOOKEEPER_CONNECT
+            value: zookeeper:2181
+          - name: KAFKA_ADVERTISED_LISTENERS
+            value: INTERNAL://cpkafka.services-dev1:9092
+          - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
+            value: INTERNAL:PLAINTEXT
+          - name: KAFKA_INTER_BROKER_LISTENER_NAME
+            value: INTERNAL
+          - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
+            value: "1"
+        ports:
+        - containerPort: 9092
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: cpkafka
+  labels:
+    app.kubernetes.io/name: cpkafka
+spec:
+  type: NodePort
+  selector:
+      app.kubernetes.io/name: cpkafka
+      app.kubernetes.io/component: master
+  ports:
+  - port: 9092
+    name: cpkafka
+    targetPort: 9092
+    nodePort: 30092

e2e/k8s/kindConfig.yaml

@@ -0,0 +1,12 @@
+kind: Cluster
+name: k8se2e
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  extraPortMappings:
+  - containerPort: 30200
+    hostPort: 49200
+  - containerPort: 30678
+    hostPort: 45678
+  - containerPort: 30092
+    hostPort: 49092

e2e/k8s/masterConfig/teraslice.yaml

@@ -0,0 +1,35 @@
+terafoundation:
+    environment: 'development'
+    log_level: debug
+    connectors:
+        elasticsearch:
+            default:
+                apiVersion: "5.6"
+                host:
+                    - "elasticsearch.services-dev1:9200"
+        elasticsearch-next:
+            default:
+                node:
+                    - "http://elasticsearch.services-dev1:9200"
+        kafka:
+            default:
+                brokers:
+                    - "cpkafka.services-dev1:9092"
+teraslice:
+    worker_disconnect_timeout: 60000
+    node_disconnect_timeout: 60000
+    slicer_timeout: 60000
+    shutdown_timeout: 30000
+    assets_directory: '/app/assets/'
+    cluster_manager_type: "kubernetes"
+    master: true
+    master_hostname: "127.0.0.1"
+    kubernetes_image: "teraslice-workspace:e2e"
+    kubernetes_image_pull_secrets:
+        - "docker-tera1-secret"
+    kubernetes_namespace: "ts-dev1"
+    kubernetes_overrides_enabled: true
+    kubernetes_priority_class_name: 'high-priority'
+    name: "ts-dev1"
+    cpu: 1
+    memory: 536870912

e2e/k8s/masterDeployment.yaml

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: teraslice-master
+  labels:
+      app.kubernetes.io/name: teraslice
+      app.kubernetes.io/component: master
+spec:
+  replicas: 1
+  selector:
+      matchLabels:
+          app.kubernetes.io/name: teraslice
+          app.kubernetes.io/component: master
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: teraslice
+        app.kubernetes.io/component: master
+        app.kubernetes.io/instance: k8se2e
+    spec:
+      containers:
+      - name: teraslice-master
+        image: teraslice-workspace:e2e
+        ports:
+        - containerPort: 5678
+        volumeMounts:
+          - mountPath: /app/config  # defines the directory
+            name: config
+      volumes:
+        - name: config
+          configMap:
+            name: teraslice-master
+            items:
+              - key: teraslice.yaml
+                path: teraslice.yaml  # the filename that the configMap gets written to, inside the matching mountPath
+      imagePullSecrets:
+        - name: docker-tera1-secret
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: teraslice-master
+  labels:
+    app.kubernetes.io/name: teraslice
+spec:
+  selector:
+      app.kubernetes.io/name: teraslice
+      app.kubernetes.io/component: master
+  ports:
+  - port: 5678
+    targetPort: 5678
+    nodePort: 30678  # the external port teraslice can be accessed on
+  type: NodePort

e2e/k8s/priorityClass.yaml

@@ -0,0 +1,7 @@
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: high-priority
+value: 1000000
+globalDefault: false
+description: "This priority class is for Teraslice pods."

e2e/k8s/role.yaml

@@ -0,0 +1,9 @@
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: teraslice-all-ts-dev1
+  namespace: ts-dev1
+rules:
+  - apiGroups: ["*"]
+    resources: ["*"]
+    verbs: ["*"]

e2e/k8s/roleBinding.yaml

@@ -0,0 +1,13 @@
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: teraslice-all-ts-dev1
+  namespace: ts-dev1
+subjects:
+  - kind: ServiceAccount
+    name: default
+    namespace: ts-dev1
+roleRef:
+  kind: Role
+  name: teraslice-all-ts-dev1
+  apiGroup: "rbac.authorization.k8s.io"

e2e/k8s/services-ns.yaml

@@ -0,0 +1,4 @@
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: services-dev1

e2e/k8s/ts-ns.yaml

@@ -0,0 +1,4 @@
+kind: Namespace
+apiVersion: v1
+metadata:
+  name: ts-dev1

e2e/k8s/workerConfig/teraslice.yaml

@@ -0,0 +1,33 @@
+terafoundation:
+    environment: 'development'
+    log_level: debug
+    connectors:
+        elasticsearch:
+            default:
+                apiVersion: "5.6"
+                host:
+                    - "elasticsearch.services-dev1:9200"
+        elasticsearch-next:
+            default:
+                node:
+                    - "http://elasticsearch.services-dev1:9200"
+        kafka:
+            default:
+                brokers:
+                    - "cpkafka.services-dev1:9092"
+teraslice:
+    worker_disconnect_timeout: 60000
+    node_disconnect_timeout: 60000
+    slicer_timeout: 60000
+    shutdown_timeout: 30000
+    assets_directory: '/app/assets/'
+    cluster_manager_type: "kubernetes"
+    master: false
+    master_hostname: "teraslice-master"
+    kubernetes_image: "teraslice-workspace:e2e"
+    kubernetes_namespace: "ts-dev1"
+    kubernetes_overrides_enabled: true
+    kubernetes_priority_class_name: 'high-priority'
+    name: "ts-dev1"
+    cpu: 1
+    memory: 536870912