From b8fb63edd104218f32ba8f813948e11561bb3735 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Sat, 4 Mar 2023 15:38:18 +0200 Subject: [PATCH] Change debug option to boolean (#58) * Change debug option to boolean * Change dryRun to bool * Fix merge conflicts * Change clean option to bool --- .github/workflows/integration.yaml | 12 ++++++------ CONTRIBUTING.md | 12 ++++++------ README.md | 23 ++++++++++++----------- datagen.js | 24 ++++++------------------ package.json | 2 +- src/dataGenerator.js | 8 ++++---- src/kafka/cleanKafka.js | 4 ++-- src/kafka/createTopic.js | 10 +++++----- src/schemas/parseAvroSchema.js | 2 +- src/schemas/parseJsonSchema.js | 2 +- src/schemas/parseSqlSchema.js | 2 +- tests/datagen.test.js | 2 +- 12 files changed, 46 insertions(+), 57 deletions(-) diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index df5223b..c901abb 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -17,22 +17,22 @@ jobs: run: docker exec datagen datagen --version - name: Produce to Kafka from SQL Schema in JSON Format - run: docker exec datagen datagen -s /tests/schema.sql -f json -n 3 --record-size 100 -dr false -d true + run: docker exec datagen datagen -s /tests/schema.sql -f json -n 3 --record-size 100 -d - name: Produce to Kafka from JSON Schema in JSON Format - run: docker exec datagen datagen -s /tests/schema.json -f json -n 3 --record-size 100 -dr false -d true + run: docker exec datagen datagen -s /tests/schema.json -f json -n 3 --record-size 100 -d - name: Produce to Kafka from Avro Schema in JSON Format - run: docker exec datagen datagen -s /tests/schema.avsc -f json -n 3 --record-size 100 -dr false -d true + run: docker exec datagen datagen -s /tests/schema.avsc -f json -n 3 --record-size 100 -d - name: Produce to Kafka from SQL Schema in Avro Format - run: docker exec datagen datagen -s /tests/schema.sql -f avro -n 3 --record-size 100 -dr false -d true -w 100 + run: docker exec datagen datagen -s /tests/schema.sql -f avro -n 3 --record-size 100 -d -w 100 - name: Produce to Kafka from JSON Schema in Avro Format - run: docker exec datagen datagen -s /tests/schema.json -f avro -n 3 --record-size 100 -dr false -d true -w 100 + run: docker exec datagen datagen -s /tests/schema.json -f avro -n 3 --record-size 100 -d -w 100 - name: Produce to Kafka from Avro Schema in Avro Format - run: docker exec datagen datagen -s /tests/schema.avsc -f avro -n 3 --record-size 100 -dr false -d true -w 100 + run: docker exec datagen datagen -s /tests/schema.avsc -f avro -n 3 --record-size 100 -d -w 100 - name: Docker Compose Down run: docker compose down -v diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e54c766..a219129 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,12 +8,12 @@ Once you have made the changes to your local copy of the repository, you can run Also try to run the script with different options to make sure it works as expected, examples: - datagen -s ./tests/schema.json -f json -n 10 --record-size 100 -dr true -d true - datagen -s ./tests/schema.avsc -f json -n 10 --record-size 100 -dr true -d true - datagen -s ./tests/schema.sql -f json -n 10 --record-size 100 -dr true -d true - datagen -s ./tests/schema.json -f avro -n 10 --record-size 100 -dr true -d true - datagen -s ./tests/schema.avsc -f avro -n 10 --record-size 100 -dr true -d true - datagen -s ./tests/schema.sql -f avro -n 10 --record-size 100 -dr true -d true + datagen -s ./tests/schema.json -f json -n 10 --record-size 100 --dry-run --debug + datagen -s ./tests/schema.avsc -f json -n 10 --record-size 100 --dry-run --debug + datagen -s ./tests/schema.sql -f json -n 10 --record-size 100 --dry-run --debug + datagen -s ./tests/schema.json -f avro -n 10 --record-size 100 --dry-run --debug + datagen -s ./tests/schema.avsc -f avro -n 10 --record-size 100 --dry-run --debug + datagen -s ./tests/schema.sql -f avro -n 10 --record-size 100 --dry-run --debug ## Cutting a new release diff --git a/README.md b/README.md index a59e335..caff4c7 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,16 @@ Usage: datagen [options] Fake Data Generator Options: - -V, --version output the version number - -f, --format The format of the produced data (choices: "json", "avro", default: "json") - -s, --schema Schema file to use - -n, --number Number of records to generate (default: "10", infinite records: "-1") - -d, --debug (choices: "true", "false", default: "false") - -w, --wait Wait time in ms between record production (default: "0") - -dr, --dry-run Dry run (no data will be produced (choices: "true", "false", default: "false") - -rs, --record-size Record size in bytes, eg. 1048576 for 1MB - -h, --help display help for command + -V, --version output the version number + -f, --format The format of the produced data (choices: "json", "avro", default: "json") + -s, --schema Schema file to use + -n, --number Number of records to generate. For infinite records, use -1 (default: "10") + -c, --clean Clean Kafka topic and schema registry before producing data + -dr, --dry-run Dry run (no data will be produced to Kafka) + -d, --debug Output extra debugging information + -w, --wait Wait time in ms between record production + -rs, --record-size Record size in bytes, eg. 1048576 for 1MB + -h, --help display help for command ``` ### Env variables @@ -146,7 +147,7 @@ docker run \ --rm -it \ -v ${PWD}/.env:/app/.env \ -v ${PWD}/tests/schema.json:/app/blah.json \ - datagen -s blah.json -n 1 -dr true + datagen -s blah.json -n 1 --dry-run ``` ### Generate records with sequence numbers @@ -170,5 +171,5 @@ This is particularly useful when you want to generate a small set of records wit Example: ``` -datagen -s tests/iterationIndex.json -dr true -f json -n 1000 +datagen -s tests/iterationIndex.json --dry-run -f json -n 1000 ``` diff --git a/datagen.js b/datagen.js index 91e15be..7d770f5 100755 --- a/datagen.js +++ b/datagen.js @@ -17,7 +17,7 @@ const dataGenerator = require('./src/dataGenerator'); const fs = require('fs'); const { program, Option } = require('commander'); -program.name('datagen').description('Fake Data Generator').version('0.1.1'); +program.name('datagen').description('Fake Data Generator').version('0.1.2'); program .addOption( @@ -32,21 +32,9 @@ program 'Number of records to generate. For infinite records, use -1' ).default('10') ) - .addOption( - new Option('-d, --debug ') - .choices(['true', 'false']) - .default('false') - ) - .addOption( - new Option('-c, --clean ') - .choices(['true', 'false']) - .default('false') - ) - .addOption( - new Option('-dr, --dry-run ', 'Dry run (no data will be produced') - .choices(['true', 'false']) - .default('false') - ) + .option('-c, --clean', 'Clean Kafka topic and schema registry before producing data') + .option('-dr, --dry-run', 'Dry run (no data will be produced to Kafka)') + .option('-d, --debug', 'Output extra debugging information') .option('-w, --wait ', 'Wait time in ms between record production', parseInt) .option('-rs, --record-size ', 'Record size in bytes, eg. 1048576 for 1MB', parseInt); @@ -69,7 +57,7 @@ global.wait = options.wait; global.clean = options.clean; global.dryRun = options.dryRun; -if (debug === 'true') { +if (debug) { console.log(options); } @@ -114,7 +102,7 @@ if (!wait) { process.exit(); } - if (clean == 'true') { + if (clean) { let topics = [] for (table of parsedSchema){ topics.push(table._meta.topic) diff --git a/package.json b/package.json index b1eee5c..b10ef49 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@materializeinc/datagen", "description": "Materialize Datagen CLI tool", - "version": "0.1.1", + "version": "0.1.2", "license": "Apache-2.0", "bin": { "@MaterializeInc/datagen": "datagen.js", diff --git a/src/dataGenerator.js b/src/dataGenerator.js index 9a79352..58dd802 100644 --- a/src/dataGenerator.js +++ b/src/dataGenerator.js @@ -27,7 +27,7 @@ async function* asyncGenerator(number) { } function sleep(s) { - if (debug === 'true' && wait > 0) { + if (debug && wait > 0) { alert({ type: `success`, name: `Sleeping for ${s} milliseconds...`, @@ -38,7 +38,7 @@ function sleep(s) { } async function prepareTopic(topic, dryRun) { - if (dryRun == 'true') { + if (dryRun) { alert({ type: `success`, name: `Dry run: Skipping topic creation...`, @@ -112,7 +112,7 @@ module.exports = async ({ } for (const topic in megaRecord) { await prepareTopic(topic, dryRun); - if (format == 'avro' && dryRun != 'true') { + if (format == 'avro' && dryRun !== true) { avroSchemas = await prepareSchema( megaRecord, topic, @@ -135,7 +135,7 @@ module.exports = async ({ record.recordSizePayload = payload; } - if (dryRun == 'true') { + if (dryRun) { alert({ type: `success`, name: `Dry run: Skipping record production...`, diff --git a/src/kafka/cleanKafka.js b/src/kafka/cleanKafka.js index ed32a3a..d6d862a 100644 --- a/src/kafka/cleanKafka.js +++ b/src/kafka/cleanKafka.js @@ -32,7 +32,7 @@ async function deleteSchemaSubjects(topics) { module.exports = async (format, topics) => { - if (dryRun == 'true') { + if (dryRun) { console.log("This is a dry run, so no resources will be deleted") return } @@ -55,4 +55,4 @@ module.exports = async (format, topics) => { await deleteSchemaSubjects(topics); } -}; \ No newline at end of file +}; diff --git a/src/kafka/createTopic.js b/src/kafka/createTopic.js index d39bf9c..e47869a 100644 --- a/src/kafka/createTopic.js +++ b/src/kafka/createTopic.js @@ -5,7 +5,7 @@ const dotenv = require('dotenv'); module.exports = async (topic = 'datagen_test_topic') => { const kafka = kafkaConfig(); - if (debug === 'true') { + if (debug) { console.log(`Trying to create topic: ${topic}`); } // Check if the topic exists in the Kafka cluster if not create it @@ -43,13 +43,13 @@ async function getReplicationFactor(admin) { let replicationFactor = 1; try { - if (debug === 'true') { + if (debug) { console.log(`Trying to get brokers list...`); } const brokersList = await admin.describeCluster(); const brokerId = brokersList.brokers[0].nodeId.toString(); - if (debug === 'true') { + if (debug) { console.log(`Trying to get default replication factor...`); } @@ -58,7 +58,7 @@ async function getReplicationFactor(admin) { resources: [{ type: ConfigResourceTypes.BROKER, name: brokerId, configNames: ['default.replication.factor'] }] }) .catch(err => { - if (debug === 'true') { + if (debug) { console.log(err); } }); @@ -68,7 +68,7 @@ async function getReplicationFactor(admin) { ).configValue; } catch (err) { console.log(`Error getting default replication factor, using 1`); - if (debug === 'true') { + if (debug) { console.log(err); } } diff --git a/src/schemas/parseAvroSchema.js b/src/schemas/parseAvroSchema.js index 5c4bfc7..5236bfd 100644 --- a/src/schemas/parseAvroSchema.js +++ b/src/schemas/parseAvroSchema.js @@ -10,7 +10,7 @@ async function parseAvroSchema(schemaFile) { msg: `` }); - if (debug === 'true') { + if (debug) { const parsed = avro.parse(schemaFile); console.log(parsed); } diff --git a/src/schemas/parseJsonSchema.js b/src/schemas/parseJsonSchema.js index dc076d1..743eb7d 100644 --- a/src/schemas/parseJsonSchema.js +++ b/src/schemas/parseJsonSchema.js @@ -7,7 +7,7 @@ async function parseJsonSchema(schemaFile) { msg: `` }); - if (debug === 'true') { + if (debug) { const parsed = JSON.parse(schemaFile); console.log(parsed); } diff --git a/src/schemas/parseSqlSchema.js b/src/schemas/parseSqlSchema.js index 2d23737..c88dd16 100644 --- a/src/schemas/parseSqlSchema.js +++ b/src/schemas/parseSqlSchema.js @@ -34,7 +34,7 @@ async function parseSqlSchema(schemaFile) { // Convert the schema to JSON tables = await convertSqlSchemaToJson(tables); - if (debug === 'true') { + if (debug) { console.log(tables, null, 3); } diff --git a/tests/datagen.test.js b/tests/datagen.test.js index 68fb3b0..06e7d83 100644 --- a/tests/datagen.test.js +++ b/tests/datagen.test.js @@ -2,7 +2,7 @@ const { Command } = require('commander'); const { execSync } = require('child_process'); const datagen = args => { - return execSync(`node ./datagen.js -dr true ${args}`).toString(); + return execSync(`node ./datagen.js --dry-run ${args}`).toString(); }; describe('Test datagen help', () => {