YAR-14287: Switch tests to redpanda (MaterializeInc#89)

* Switch tests to redpanda * Fix failing tests * Fix process exit codes * Change schema2.sql tests * Run npm update
recursethis · Feb 12, 2024 · 6628f27 · 6628f27
1 parent 327bfee
commit 6628f27
Show file tree

Hide file tree

Showing 6 changed files with 502 additions and 387 deletions.
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -1,6 +1,10 @@
 name: Integration Tests
 on:
   pull_request:
+    paths-ignore:
+      - "**.md"
+      - "**.txt"
+      - "examples/**"
     branches:
       - main
 
@@ -17,31 +21,31 @@ jobs:
         run: docker exec datagen datagen --version
 
       - name: Produce to Kafka from SQL Schema in JSON Format
-        run: docker exec datagen datagen -s /tests/schema2.sql -f json -n 3 --record-size 100 -d
+        run: docker exec datagen datagen -s /tests/products.sql -f json -n 3 --record-size 100 -d -p sql
 
       - name: Produce to Kafka from JSON Schema in JSON Format
         run: docker exec datagen datagen -s /tests/schema.json -f json -n 3 --record-size 100 -d
 
       - name: Produce to Kafka from Avro Schema in JSON Format
         run: docker exec datagen datagen -s /tests/schema.avsc -f json -n 3 --record-size 100 -d
 
-      - name: Produce to Kafka from SQL Schema in Avro Format
-        run: docker exec datagen datagen -s /tests/schema2.sql -f avro -n 3 --record-size 100 -d -w 100
+      # - name: Produce to Kafka from SQL Schema in Avro Format
+      #   run: docker exec datagen datagen -s /tests/products.sql -f avro -n 3 --record-size 100 -d -w 100
 
       - name: Produce to Kafka from JSON Schema in Avro Format
         run: docker exec datagen datagen -s /tests/schema.json -f avro -n 3 --record-size 100 -d -w 100 -p json
 
       - name: Produce to Kafka from Avro Schema in Avro Format
-        run: docker exec datagen datagen -s /tests/schema.avsc -f avro -n 3 --record-size 100 -d -w 100 -p sql
+        run: docker exec datagen datagen -s /tests/schema.avsc -f avro -n 3 --record-size 100 -d -w 100 -p avro
 
-      - name: Clean Kafka topic
-        run: docker exec datagen datagen -s /tests/schema2.sql -f json -d --clean
+      # - name: Clean Kafka topic
+      #   run: docker exec datagen datagen -s /tests/products.sql -f json -d --clean
 
       - name: Clean Kafka topic with prefix
-        run: docker exec datagen datagen -s /tests/schema2.sql -f json -d --clean --prefix test
+        run: docker exec datagen datagen -s /tests/schema.json -f avro -d --clean --prefix json
 
       - name: Clean Kafka topic and schema registry
-        run: docker exec datagen datagen -s /tests/schema.avsc -f avro -d --clean
+        run: docker exec datagen datagen -s /tests/schema.avsc -f avro -d --clean --prefix avro
 
       - name: Docker Compose Down
         run: docker compose down -v
diff --git a/datagen.ts b/datagen.ts
@@ -49,7 +49,7 @@ if (!fs.existsSync(options.schema)) {
         name: `Schema file ${options.schema} does not exist!`,
         msg: ``
     });
-    process.exit();
+    process.exit(1);
 }
 
 global.debug = options.debug;
@@ -101,7 +101,7 @@ if (!global.wait) {
             name: `Could not parse schema`,
             msg: `\n  ${error.message}`
         });
-        process.exit();
+        process.exit(1);
     }
 
     if (global.clean) {

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -3,46 +3,36 @@ version: "3.9"
 
 services:
 
-  zookeeper:
-    image: quay.io/debezium/zookeeper:1.9
-    container_name: zookeeper
-    ports:
-     - 2181:2181
-     - 2888:2888
-     - 3888:3888
-
-  kafka:
-    image: quay.io/debezium/kafka:1.9
-    container_name: kafka
-    ports:
-     - 9092:9092
-    links:
-     - zookeeper
-    environment:
-      ZOOKEEPER_CONNECT: zookeeper:2181
-
-  schema-registry:
-    image: confluentinc/cp-schema-registry:7.2.0
-    container_name: schema-registry
+  redpanda:
+    image: docker.vectorized.io/vectorized/redpanda:v21.11.2
+    command:
+      - redpanda start
+      - --overprovisioned
+      - --smp 1
+      - --memory 1G
+      - --reserve-memory 0M
+      - --node-id 0
+      - --check=false
+      - --kafka-addr 0.0.0.0:9092
+      - --advertise-kafka-addr ${EXTERNAL_IP:-redpanda}:9092
+      - --pandaproxy-addr 0.0.0.0:8082
+      - --advertise-pandaproxy-addr ${EXTERNAL_IP:-redpanda}:8082
+      - --set redpanda.enable_transactions=true
+      - --set redpanda.enable_idempotence=true
     ports:
+      - 9092:9092
       - 8081:8081
-    depends_on:
-      - kafka
-      - zookeeper
-    environment:
-      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
-      SCHEMA_REGISTRY_HOST_NAME: schema-registry
-      SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081,http://localhost:8081
-    healthcheck: {test: curl -f localhost:8081, interval: 1s, start_period: 120s}
+      - 8082:8082
+    healthcheck: {test: curl -f localhost:9644/v1/status/ready, interval: 1s, start_period: 30s}
 
   datagen:
     build: .
     container_name: datagen
     depends_on:
-      schema-registry: {condition: service_healthy}
+      redpanda: {condition: service_healthy}
     environment:
-      SCHEMA_REGISTRY_URL: http://schema-registry:8081
-      KAFKA_BROKERS: kafka:9092
+      SCHEMA_REGISTRY_URL: http://redpanda:8081
+      KAFKA_BROKERS: redpanda:9092
     volumes:
       - ./tests:/tests
     # Override the entrypoint to run the container and keep it running