Postgres

.env.example

@@ -15,3 +15,10 @@ SSL_KEY_LOCATION=
 SCHEMA_REGISTRY_URL=
 SCHEMA_REGISTRY_USERNAME=
 SCHEMA_REGISTRY_PASSWORD=
+
+# Postgres
+POSTGRES_HOST=
+POSTGRES_PORT=
+POSTGRES_DB=
+POSTGRES_USER=
+POSTGRES_PASSWORD=

.github/workflows/integration.yaml

@@ -47,5 +47,12 @@ jobs:
       - name: Clean Kafka topic and schema registry
         run: docker exec datagen datagen -s /tests/schema.avsc -f avro -d --clean --prefix avro
 
+      # Postgres tests
+      - name: Produce data to Postgres with Faker.js
+        run: docker exec datagen datagen -s /tests/products.sql -f postgres -n 3
+
+      - name: Produce data to Postgres with multiple tables
+        run: docker exec datagen datagen -s /tests/schema2.sql -f postgres -n 3
+
       - name: Docker Compose Down
         run: docker compose down -v

README.md

@@ -56,6 +56,13 @@ SSL_KEY_LOCATION=
 SCHEMA_REGISTRY_URL=
 SCHEMA_REGISTRY_USERNAME=
 SCHEMA_REGISTRY_PASSWORD=
+
+# Postgres
+POSTGRES_HOST=
+POSTGRES_PORT=
+POSTGRES_DB=
+POSTGRES_USER=
+POSTGRES_PASSWORD=
 ```
 
 The `datagen` program will read the environment variables from `.env` in the current working directory.
@@ -75,7 +82,7 @@ Fake Data Generator
 Options:
   -V, --version             output the version number
   -s, --schema <char>       Schema file to use
-  -f, --format <char>       The format of the produced data (choices: "json", "avro", default: "json")
+  -f, --format <char>       The format of the produced data (choices: "json", "avro", "postgres", default: "json")
   -n, --number <char>       Number of records to generate. For infinite records, use -1 (default: "10")
   -c, --clean               Clean (delete) Kafka topics and schema subjects previously created
   -dr, --dry-run            Dry run (no data will be produced to Kafka)
@@ -245,6 +252,30 @@ CREATE TABLE "ecommerce"."products" (
 
 This will produce the desired mock data to the topic `ecommerce.products`.
 
+#### Producing to Postgres
+
+You can also produce the data to a Postgres database. To do this, you need to specify the `-f sql` option and provide Postgres connection information in the `.env` file. Here is an example `.env` file:
+
+```
+# Postgres
+export POSTGRES_HOST=
+export POSTGRES_PORT=
+export POSTGRES_DB=
+export POSTGRES_USER=
+export POSTGRES_PASSWORD=
+```
+
+Then, you can run the following command to produce the data to Postgres:
+
+```bash
+datagen \
+    -s tests/products.sql \
+    -f postgres \
+    -n 1000
+```
+
+> :warning: You can only produce to Postgres with a SQL schema.
+
 ### Avro Schema
 
 > :construction: Avro input schema currently does not support arbitrary FakerJS methods. Instead, data is randomly generated based on the type.

datagen.ts

@@ -23,7 +23,7 @@ program
     .requiredOption('-s, --schema <char>', 'Schema file to use')
     .addOption(
         new Option('-f, --format <char>', 'The format of the produced data')
-            .choices(['json', 'avro'])
+            .choices(['json', 'avro', 'postgres'])
             .default('json')
     )
     .addOption(
@@ -113,11 +113,12 @@ if (!global.wait) {
         process.exit(0);
     }
 
-    // Generate data
+
     await dataGenerator({
         format: options.format,
         schema: parsedSchema,
-        iterations: options.number
+        iterations: options.number,
+        initialSchema: options.schema
     })
 
     await end();

docker-compose.yaml

@@ -25,6 +25,15 @@ services:
       - 8082:8082
     healthcheck: {test: curl -f localhost:9644/v1/status/ready, interval: 1s, start_period: 30s}
 
+  postgres:
+    image: postgres:14.0
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
+    ports:
+      - 5432:5432
+
   datagen:
     build: .
     container_name: datagen
@@ -33,6 +42,11 @@ services:
     environment:
       SCHEMA_REGISTRY_URL: http://redpanda:8081
       KAFKA_BROKERS: redpanda:9092
+      POSTGRES_HOST: postgres
+      POSTGRES_PORT: 5432
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
     volumes:
       - ./tests:/tests
     # Override the entrypoint to run the container and keep it running