feat!: [riot-gen] Added random data-structure generator

connectors/riot-db/src/test/java/com/redis/riot/db/PostgreSQLTests.java → connectors/riot-db/src/test/java/com/redis/riot/db/PostgresTests.java

@@ -29,13 +29,13 @@
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.redis.lettucemod.api.sync.RedisModulesCommands;
 import com.redis.spring.batch.DataStructure.Type;
-import com.redis.spring.batch.support.RandomDataStructureItemReader;
+import com.redis.spring.batch.reader.RandomDataStructureItemReader;
 import com.redis.testcontainers.junit.RedisTestContext;
 import com.redis.testcontainers.junit.RedisTestContextsSource;
 
 @Testcontainers
 @SuppressWarnings({ "rawtypes", "unchecked" })
-public class PostgreSQLTests extends AbstractDatabaseTests {
+public class PostgresTests extends AbstractDatabaseTests {
 
 	private static final DockerImageName POSTGRE_DOCKER_IMAGE_NAME = DockerImageName.parse(PostgreSQLContainer.IMAGE)
 			.withTag(PostgreSQLContainer.DEFAULT_TAG);
@@ -51,7 +51,7 @@ public static void setupAll() throws SQLException, IOException {
 		scriptRunner.setAutoCommit(false);
 		scriptRunner.setStopOnError(true);
 		String file = "northwind.sql";
-		InputStream inputStream = PostgreSQLTests.class.getClassLoader().getResourceAsStream(file);
+		InputStream inputStream = PostgresTests.class.getClassLoader().getResourceAsStream(file);
 		if (inputStream == null) {
 			throw new FileNotFoundException(file);
 		}
@@ -104,10 +104,10 @@ void testExportNullValues(RedisTestContext redis) throws Exception {
 			Map<String, String> hash1 = new HashMap<>();
 			hash1.put("field1", "value1");
 			hash1.put("field2", "value2");
-			sync.hmset("generated:1", hash1);
+			sync.hmset("gen:hash:1", hash1);
 			Map<String, String> hash2 = new HashMap<>();
 			hash2.put("field2", "value2");
-			sync.hmset("generated:2", hash2);
+			sync.hmset("gen:hash:2", hash2);
 			execute("export-postgresql", redis, r -> configureExportCommand(r, POSTGRESQL));
 			statement.execute("SELECT COUNT(*) AS count FROM mytable");
 			ResultSet countResultSet = statement.getResultSet();

connectors/riot-db/src/test/resources/export-postgresql

@@ -1 +1 @@
-riot-db export "INSERT INTO mytable (id, field1, field2) VALUES (CAST(:id AS SMALLINT), :field1, :field2)" --url "jdbc:postgresql://host:port/database" --username appuser --password passwd --scan-match "generated:*" --key-regex "generated:(?<id>.*)"
+riot-db export "INSERT INTO mytable (id, field1, field2) VALUES (CAST(:id AS SMALLINT), :field1, :field2)" --url "jdbc:postgresql://host:port/database" --username appuser --password passwd --scan-match "gen:hash:*" --key-regex "gen:hash:(?<id>.*)"

connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileImportCommand.java

@@ -96,30 +96,29 @@ private TaskletStep fileImportStep(String file) throws Exception {
 				.processor(this::processDataStructure).build()).build();
 	}
 
-	@SuppressWarnings({ "unchecked", "incomplete-switch" })
+	@SuppressWarnings("unchecked")
 	private DataStructure<String> processDataStructure(DataStructure<String> item) {
-		if (item.getType() != null) {
-			switch (Type.of(item.getType())) {
-			case ZSET:
-				Collection<Map<String, Object>> zset = (Collection<Map<String, Object>>) item.getValue();
-				Collection<ScoredValue<String>> values = new ArrayList<>(zset.size());
-				for (Map<String, Object> map : zset) {
-					double score = ((Number) map.get("score")).doubleValue();
-					String value = (String) map.get("value");
-					values.add((ScoredValue<String>) ScoredValue.fromNullable(score, value));
-				}
-				item.setValue(values);
-				break;
-			case STREAM:
-				Collection<Map<String, Object>> stream = (Collection<Map<String, Object>>) item.getValue();
-				Collection<StreamMessage<String, String>> messages = new ArrayList<>(stream.size());
-				for (Map<String, Object> message : stream) {
-					messages.add(new StreamMessage<>((String) message.get("stream"), (String) message.get("id"),
-							(Map<String, String>) message.get("body")));
-				}
-				item.setValue(messages);
-				break;
+		if (item.getType() == null) {
+			return item;
+		}
+		Type type = Type.of(item.getType());
+		if (type == Type.ZSET) {
+			Collection<Map<String, Object>> zset = (Collection<Map<String, Object>>) item.getValue();
+			Collection<ScoredValue<String>> values = new ArrayList<>(zset.size());
+			for (Map<String, Object> map : zset) {
+				double score = ((Number) map.get("score")).doubleValue();
+				String value = (String) map.get("value");
+				values.add((ScoredValue<String>) ScoredValue.fromNullable(score, value));
+			}
+			item.setValue(values);
+		} else if (type == Type.STREAM) {
+			Collection<Map<String, Object>> stream = (Collection<Map<String, Object>>) item.getValue();
+			Collection<StreamMessage<String, String>> messages = new ArrayList<>(stream.size());
+			for (Map<String, Object> message : stream) {
+				messages.add(new StreamMessage<>((String) message.get("stream"), (String) message.get("id"),
+						(Map<String, String>) message.get("body")));
 			}
+			item.setValue(messages);
 		}
 		return item;
 	}

connectors/riot-gen/src/docs/asciidoc/index.adoc

@@ -2,7 +2,7 @@
 
 == Introduction
 
-{project-title} is a data generator for Redis based on https://github.com/DiUS/java-faker[Faker].
+{project-title} is a data generator for Redis.
 
 It supports https://redis.io[Redis] and https://redis.com/redis-enterprise-software/overview/[Redis Enterprise] in either standalone or https://redis.io/topics/cluster-tutorial[cluster] deployments.
 
@@ -11,15 +11,26 @@ It supports https://redis.io[Redis] and https://redis.com/redis-enterprise-softw
 
 include::{includedir}/_getting-started.adoc[leveloffset=+1]
 
-== Importing
+== Random Data-Structure Generator
 
-The `import` command generates data and writes it to Redis.
+The `ds` subcommand generates random data for Redis data-structures (set, list, zset, stream, string, hash), RedisJSON, and RedisTimeSeries.
 
 === Usage
 
 [subs=+quotes]
 ----
-[green]#riot-gen# -h <host> -p <port> import [olive]#SPEL#... [REDIS COMMAND...]
+[green]#riot-gen# -h <host> -p <port> ds [OPTIONS]
+----
+
+== Faker Generator
+
+The `faker` subcommand generates data using https://github.com/DiUS/java-faker[Faker] and writes it to Redis.
+
+=== Usage
+
+[subs=+quotes]
+----
+[green]#riot-gen# -h <host> -p <port> faker [olive]#SPEL#... [REDIS COMMAND...]
 ----
 
 where SPEL is a https://docs.spring.io/spring/docs/current/spring-framework-reference/core.html#expressions[Spring Expression Language] field in the form `field="expression"`.
@@ -28,19 +39,19 @@ To show the full usage, run:
 
 [subs="+quotes"]
 ----
-[green]#riot-gen# import --help
+[green]#riot-gen# faker --help
 ----
 
 .Hash generator example
 [source,console]
 ----
-include::{test-resources}/import-hset[]
+include::{test-resources}/faker-hset[]
 ----
 
 .Set generator example
 [source,console]
 ----
-include::{test-resources}/import-sadd[]
+include::{test-resources}/faker-sadd[]
 ----
 
 === Redis Commands
@@ -57,7 +68,7 @@ You can infer generator fields from a RediSearch index using the `--infer` optio
 
 [source,console]
 ----
-include::{test-resources}/import-infer[]
+include::{test-resources}/faker-infer[]
 ----
 
 [[_faker_fields]]
@@ -74,13 +85,13 @@ Most providers don't take any arguments and can be called directly, for example:
 
 [subs="+quotes"]
 ----
-[green]#riot-gen# import firstName="name.firstName"
+[green]#riot-gen# faker firstName="name.firstName"
 ----
 
 Some providers take parameters, for example:
 [subs="+quotes"]
 ----
-[green]#riot-gen# import lease="number.digits(2)"
+[green]#riot-gen# faker lease="number.digits(2)"
 ----
 
 Refer to the link specified for each provider for complete documentation.

connectors/riot-gen/src/main/java/com/redis/riot/gen/DataStructureGeneratorCommand.java

@@ -11,8 +11,7 @@
 import com.redis.riot.RiotStep;
 import com.redis.spring.batch.DataStructure;
 import com.redis.spring.batch.RedisItemWriter;
-import com.redis.spring.batch.support.RandomDataStructureItemReader;
-import com.redis.spring.batch.support.RandomDataStructureItemReader.Builder;
+import com.redis.spring.batch.reader.RandomDataStructureItemReader;
 import com.redis.spring.batch.support.Range;
 
 import picocli.CommandLine;
@@ -40,11 +39,12 @@ protected Job job(JobBuilder jobBuilder) throws Exception {
 				.build();
 		log.debug("Creating random data structure reader with {}", options);
 		return jobBuilder.start(step(RiotStep.reader(reader()).writer(writer).name(NAME).taskName("Generating")
-				.max(() -> (long) options.getEnd() - options.getStart()).build()).build()).build();
+				.max(() -> (long) options.getCount()).build()).build()).build();
 	}
 
 	private ItemReader<DataStructure<String>> reader() {
-		Builder reader = RandomDataStructureItemReader.builder().between(options.getStart(), options.getEnd())
+		RandomDataStructureItemReader.Builder reader = RandomDataStructureItemReader.builder().start(options.getStart())
+				.count(options.getCount())
 				.collectionCardinality(
 						Range.between(options.getMinCollectionCardinality(), options.getMaxCollectionCardinality()))
 				.keyspace(options.getKeyspace())

connectors/riot-gen/src/main/java/com/redis/riot/gen/DataStructureGeneratorOptions.java

@@ -1,14 +1,14 @@
 package com.redis.riot.gen;
 
-import static com.redis.spring.batch.support.RandomDataStructureItemReader.DEFAULT_COLLECTION_CARDINALITY;
-import static com.redis.spring.batch.support.RandomDataStructureItemReader.DEFAULT_KEYSPACE;
-import static com.redis.spring.batch.support.RandomDataStructureItemReader.DEFAULT_STRING_VALUE_SIZE;
-import static com.redis.spring.batch.support.RandomDataStructureItemReader.DEFAULT_ZSET_SCORE;
+import static com.redis.spring.batch.reader.RandomDataStructureItemReader.DEFAULT_COLLECTION_CARDINALITY;
+import static com.redis.spring.batch.reader.RandomDataStructureItemReader.DEFAULT_KEYSPACE;
+import static com.redis.spring.batch.reader.RandomDataStructureItemReader.DEFAULT_STRING_VALUE_SIZE;
+import static com.redis.spring.batch.reader.RandomDataStructureItemReader.DEFAULT_ZSET_SCORE;
 
 import java.util.OptionalInt;
 
 import com.redis.spring.batch.DataStructure.Type;
-import com.redis.spring.batch.support.RandomDataStructureItemReader;
+import com.redis.spring.batch.reader.RandomDataStructureItemReader;
 
 import picocli.CommandLine.Option;
 

connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerGeneratorCommand.java

@@ -39,14 +39,14 @@ protected Job job(JobBuilder jobBuilder) throws Exception {
 
 	@Override
 	protected Long initialMax() {
-		return (long) options.getEnd() - options.getStart();
+		return (long) options.getCount();
 	}
 
 	private ItemReader<Map<String, Object>> reader() {
 		log.debug("Creating Faker reader with {}", options);
 		FakerItemReader reader = new FakerItemReader(generator());
 		reader.setStart(options.getStart());
-		reader.setEnd(options.getEnd());
+		reader.setCount(options.getCount());
 		if (options.getSleep() > 0) {
 			return new ThrottledItemReader<>(reader, options.getSleep());
 		}

connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerItemReader.java

@@ -14,37 +14,32 @@
  */
 public class FakerItemReader extends AbstractItemCountingItemStreamItemReader<Map<String, Object>> {
 
-	public static final long DEFAULT_START = 0;
-	public static final long DEFAULT_END = 1000;
+	public static final int DEFAULT_START = 1;
+	public static final int DEFAULT_COUNT = 1000;
 
-	private long start = DEFAULT_START;
-	private long end = DEFAULT_END;
+	private int start = DEFAULT_START;
+	private int count = DEFAULT_COUNT;
 	private final Generator<Map<String, Object>> generator;
 
 	public FakerItemReader(Generator<Map<String, Object>> generator) {
 		setName(ClassUtils.getShortName(FakerItemReader.class));
 		Assert.notNull(generator, "A generator is required");
-		setMaxItemCount();
+		setMaxItemCount(count);
 		this.generator = generator;
 	}
 
-	public void setStart(long start) {
+	public void setStart(int start) {
 		this.start = start;
-		setMaxItemCount();
 	}
 
-	public void setEnd(long end) {
-		this.end = end;
-		setMaxItemCount();
-	}
-
-	private void setMaxItemCount() {
-		setMaxItemCount(Math.toIntExact(end - start));
+	public void setCount(int count) {
+		this.count = count;
+		setMaxItemCount(count);
 	}
 
 	@Override
 	protected Map<String, Object> doRead() throws Exception {
-		return generator.next(start + (getCurrentItemCount() % (end - start)));
+		return generator.next(start + ((getCurrentItemCount() - 1) % count));
 	}
 
 	@Override

connectors/riot-gen/src/main/java/com/redis/riot/gen/Generator.java

@@ -2,5 +2,5 @@
 
 public interface Generator<T> {
 
-    T next(long index);
+    T next(int index);
 }

connectors/riot-gen/src/main/java/com/redis/riot/gen/GeneratorOptions.java

@@ -5,9 +5,9 @@
 public class GeneratorOptions {
 
 	@CommandLine.Option(names = "--start", description = "Start index (default: ${DEFAULT-VALUE})", paramLabel = "<int>")
-	protected int start = 0;
-	@CommandLine.Option(names = "--end", description = "End index (default: ${DEFAULT-VALUE})", paramLabel = "<int>")
-	protected int end = 1000;
+	protected int start = 1;
+	@CommandLine.Option(names = "--count", description = "Number of items to generate (default: ${DEFAULT-VALUE})", paramLabel = "<int>")
+	protected int count = 1000;
 	@CommandLine.Option(names = "--sleep", description = "Duration in ms to sleep before each item generation (default: ${DEFAULT-VALUE})", paramLabel = "<ms>")
 	private long sleep = 0;
 
@@ -19,12 +19,12 @@ public void setStart(int start) {
 		this.start = start;
 	}
 
-	public int getEnd() {
-		return end;
+	public int getCount() {
+		return count;
 	}
 
-	public void setEnd(int end) {
-		this.end = end;
+	public void setCount(int count) {
+		this.count = count;
 	}
 
 	public long getSleep() {

connectors/riot-gen/src/main/java/com/redis/riot/gen/MapGenerator.java

@@ -26,7 +26,7 @@ public MapGenerator(EvaluationContext context, Map<String, Expression> expressio
 	}
 
 	@Override
-	public Map<String, Object> next(long index) {
+	public Map<String, Object> next(int index) {
 		Map<String, Object> map = new HashMap<>();
 		context.setVariable(FIELD_INDEX, index);
 		for (Entry<String, Expression> expression : expressions.entrySet()) {

connectors/riot-gen/src/main/java/com/redis/riot/gen/MapWithMetadataGenerator.java

@@ -13,7 +13,7 @@ public MapWithMetadataGenerator(Generator<Map<String, Object>> parent) {
 	}
 
 	@Override
-	public Map<String, Object> next(long index) {
+	public Map<String, Object> next(int index) {
 		Map<String, Object> map = parent.next(index);
 		map.put(MapGenerator.FIELD_INDEX, index);
 		map.put(FIELD_THREAD, Thread.currentThread().getId());

connectors/riot-gen/src/test/java/com/redis/riot/gen/FakerReaderTests.java

@@ -43,7 +43,7 @@ void testReader() throws Exception {
 		fields.put("firstName", "name.firstName");
 		fields.put("lastName", "name.lastName");
 		FakerItemReader reader = new FakerItemReader(MapGenerator.builder().fields(fields).build());
-		reader.setEnd(count);
+		reader.setCount(count);
 		List<Map<String, Object>> items = new ArrayList<>();
 		run("reader", reader, items::addAll);
 		Assertions.assertEquals(count, items.size());
@@ -60,11 +60,11 @@ void testIncludeMetadata() throws Exception {
 		fields.put("lastName", "name.lastName");
 		FakerItemReader reader = new FakerItemReader(
 				new MapWithMetadataGenerator(MapGenerator.builder().fields(fields).build()));
-		reader.setEnd(count);
+		reader.setCount(count);
 		List<Map<String, Object>> items = new ArrayList<>();
 		run("metadata", reader, items::addAll);
 		Assertions.assertEquals(count, items.size());
-		Assertions.assertEquals(1L, items.get(0).get(MapGenerator.FIELD_INDEX));
+		Assertions.assertEquals(1, items.get(0).get(MapGenerator.FIELD_INDEX));
 	}
 
 	private <T> void run(String name, ItemReader<T> reader, ItemWriter<T> writer) throws Exception {

connectors/riot-gen/src/test/resources/faker-sadd

@@ -1 +1 @@
-riot-gen faker name="gameOfThrones.character" --end 1000 sadd --keyspace got:characters --members name
+riot-gen faker name="gameOfThrones.character" --count 1000 sadd --keyspace got:characters --members name

connectors/riot-gen/src/test/resources/faker-tsadd

@@ -1 +1 @@
-riot-gen --info faker value="random.nextDouble" --end 10 --batch 1 --sleep 1 --skip-policy never ts.add --keyspace ts:gen --value value
+riot-gen --info faker value="random.nextDouble" --count 10 --batch 1 --sleep 1 --skip-policy never ts.add --keyspace ts:gen --value value

connectors/riot-gen/src/test/resources/faker-tsadd-options

@@ -1 +1 @@
-riot-gen --info faker value="random.nextDouble" future="backToTheFuture.character" lebowski="lebowski.character" --end 1000 --sleep 1 --skip-policy never ts.add --keyspace ts:gen --keys future lebowski --value value --labels character1=future character2=lebowski
+riot-gen --info faker value="random.nextDouble" future="backToTheFuture.character" lebowski="lebowski.character" --count 1000 --sleep 1 --skip-policy never ts.add --keyspace ts:gen --keys future lebowski --value value --labels character1=future character2=lebowski