YAR-14287: Convert to TypeScript (MaterializeInc#66)

* Convert to TypeScript

* Allow integration tests on PRs

* Update avro name hook

* Fix CSR prefix bug

.github/workflows/integration.yaml

@@ -1,8 +1,8 @@
 name: Integration Tests
 on:
   pull_request:
-    paths:
-      - materialize/**
+    branches:
+      - main
 
 jobs:
   integration:
@@ -17,7 +17,7 @@ jobs:
         run: docker exec datagen datagen --version
 
       - name: Produce to Kafka from SQL Schema in JSON Format
-        run: docker exec datagen datagen -s /tests/schema.sql -f json -n 3 --record-size 100 -d
+        run: docker exec datagen datagen -s /tests/schema2.sql -f json -n 3 --record-size 100 -d
 
       - name: Produce to Kafka from JSON Schema in JSON Format
         run: docker exec datagen datagen -s /tests/schema.json -f json -n 3 --record-size 100 -d
@@ -26,25 +26,19 @@ jobs:
         run: docker exec datagen datagen -s /tests/schema.avsc -f json -n 3 --record-size 100 -d
 
       - name: Produce to Kafka from SQL Schema in Avro Format
-        run: docker exec datagen datagen -s /tests/schema.sql -f avro -n 3 --record-size 100 -d -w 100
+        run: docker exec datagen datagen -s /tests/schema2.sql -f avro -n 3 --record-size 100 -d -w 100
 
       - name: Produce to Kafka from JSON Schema in Avro Format
-        run: docker exec datagen datagen -s /tests/schema.json -f avro -n 3 --record-size 100 -d -w 100
+        run: docker exec datagen datagen -s /tests/schema.json -f avro -n 3 --record-size 100 -d -w 100 -p json
 
       - name: Produce to Kafka from Avro Schema in Avro Format
-        run: docker exec datagen datagen -s /tests/schema.avsc -f avro -n 3 --record-size 100 -d -w 100
-
-      - name: Topic prefix with json
-        run: docker exec datagen datagen -s /tests/schema.sql -f json -n 3 --record-size 100 -d --prefix test
-
-      - name: Topic prefix with avro
-        run: docker exec datagen datagen -s /tests/schema.sql -f avro -n 3 --record-size 100 -d --prefix test
+        run: docker exec datagen datagen -s /tests/schema.avsc -f avro -n 3 --record-size 100 -d -w 100 -p sql
 
       - name: Clean Kafka topic
-        run: docker exec datagen datagen -s /tests/schema.sql -f json -d --clean
+        run: docker exec datagen datagen -s /tests/schema2.sql -f json -d --clean
 
       - name: Clean Kafka topic with prefix
-        run: docker exec datagen datagen -s /tests/schema.sql -f json -d --clean --prefix test
+        run: docker exec datagen datagen -s /tests/schema2.sql -f json -d --clean --prefix test
 
       - name: Clean Kafka topic and schema registry
         run: docker exec datagen datagen -s /tests/schema.avsc -f avro -d --clean

.github/workflows/tests.yaml

@@ -20,5 +20,8 @@ jobs:
       # Install dependencies
       - run: npm install
 
+      # Build
+      - run: npm run build
+
       # Run tests
       - run: npm test

.gitignore

@@ -69,3 +69,5 @@ assets/img/.DS_Store
 
 # VSCode related files #
 # .vscode
+
+dist/

.npmignore

@@ -0,0 +1,4 @@
+src
+tsconfig.json
+tslint.json
+.prettierrc

Dockerfile

@@ -1,21 +1,26 @@
 # Use the latest LTS version of Node.js as the base image
-FROM node:lts-alpine
+FROM node:lts-slim as builder
 
 # Set the working directory in the container
 WORKDIR /app
 
 # Copy the package.json and package-lock.json files to the container
-COPY package*.json ./
+COPY . .
 
 # Install the application dependencies
-RUN npm install
+RUN npm install && npm run build
+
+FROM node:lts-slim
 
 # Copy the application source code to the container
-COPY ./datagen.js ./
-COPY ./src ./src
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm install --only=production
+COPY --from=builder /app/dist ./dist
 COPY ./tests ./tests
 
-RUN npm link
+RUN npm link --only=production
 
 # Set the command to run the application when the container starts
 ENTRYPOINT [ "datagen" ]

README.md

@@ -30,6 +30,7 @@ docker pull materialize/datagen
 git clone https://github.com/MaterializeInc/datagen.git
 cd datagen
 npm install
+npm run build
 npm link
 ```
 

datagen.js → datagen.ts

@@ -6,16 +6,16 @@
  *
  * @author Bobby Iliev <https://github.com/bobbyiliev>
  */
-const end = require('./src/utils/end');
-const alert = require('cli-alerts');
+import end from './src/utils/end.js';
+import alert from 'cli-alerts';
 
-const { parseSqlSchema } = require('./src/schemas/parseSqlSchema');
-const { parseAvroSchema } = require('./src/schemas/parseAvroSchema');
-const { parseJsonSchema } = require('./src/schemas/parseJsonSchema');
-const cleanKafka  = require('./src/kafka/cleanKafka');
-const dataGenerator = require('./src/dataGenerator');
-const fs = require('fs');
-const { program, Option } = require('commander');
+import { parseSqlSchema } from './src/schemas/parseSqlSchema.js';
+import { parseAvroSchema } from './src/schemas/parseAvroSchema.js';
+import parseJsonSchema from './src/schemas/parseJsonSchema.js';
+import cleanKafka from './src/kafka/cleanKafka.js';
+import dataGenerator from './src/dataGenerator.js';
+import fs from 'fs';
+import { program, Option } from 'commander';
 
 program.name('datagen').description('Fake Data Generator').version('0.1.3');
 
@@ -59,12 +59,12 @@ global.clean = options.clean;
 global.dryRun = options.dryRun;
 global.prefix = options.prefix;
 
-if (debug) {
+if (global.debug) {
     console.log(options);
 }
 
-if (!wait) {
-    wait = 0;
+if (!global.wait) {
+    global.wait = 0;
 }
 
 (async () => {
@@ -104,12 +104,12 @@ if (!wait) {
         process.exit();
     }
 
-    if (clean) {
+    if (global.clean) {
         let topics = []
-        for (table of parsedSchema){
+        for (let table of parsedSchema) {
             topics.push(table._meta.topic)
         }
-        await cleanKafka(options.format,topics)
+        await cleanKafka(options.format, topics)
         process.exit(0);
     }
 

docker-compose.yaml

@@ -28,17 +28,18 @@ services:
       - 8081:8081
     depends_on:
       - kafka
+      - zookeeper
     environment:
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
       SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081,http://localhost:8081
+    healthcheck: {test: curl -f localhost:8081, interval: 1s, start_period: 120s}
 
   datagen:
     build: .
     container_name: datagen
     depends_on:
-      - kafka
-      - schema-registry
+      schema-registry: {condition: service_healthy}
     environment:
       SCHEMA_REGISTRY_URL: http://schema-registry:8081
       KAFKA_BROKERS: kafka:9092