diff --git a/connectors/riot-db/riot-db.gradle b/connectors/riot-db/riot-db.gradle index 2376e0332..5e926c71e 100644 --- a/connectors/riot-db/riot-db.gradle +++ b/connectors/riot-db/riot-db.gradle @@ -17,6 +17,7 @@ dependencies { implementation 'org.postgresql:postgresql' implementation group: 'org.xerial', name: 'sqlite-jdbc', version: sqliteVersion testImplementation project(':riot-test') + testImplementation 'org.slf4j:slf4j-simple' testImplementation group: 'org.testcontainers', name: 'postgresql', version: testcontainersVersion testImplementation group: 'org.testcontainers', name: 'oracle-xe', version: testcontainersVersion } diff --git a/connectors/riot-db/src/main/java/com/redis/riot/db/DataSourceOptions.java b/connectors/riot-db/src/main/java/com/redis/riot/db/DataSourceOptions.java index 6b3a54278..aa49240f3 100644 --- a/connectors/riot-db/src/main/java/com/redis/riot/db/DataSourceOptions.java +++ b/connectors/riot-db/src/main/java/com/redis/riot/db/DataSourceOptions.java @@ -57,4 +57,10 @@ public DataSource dataSource() { return properties.initializeDataSourceBuilder().build(); } + @Override + public String toString() { + return "DataSourceOptions [driver=" + driver + ", url=" + url + ", username=" + username + ", password=" + + password + "]"; + } + } diff --git a/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseExportCommand.java b/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseExportCommand.java index 2993c9a64..4d9b103a7 100644 --- a/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseExportCommand.java +++ b/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseExportCommand.java @@ -3,11 +3,11 @@ import java.sql.Connection; import java.util.Map; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.sql.DataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.item.ItemProcessor; @@ -18,18 +18,18 @@ import com.redis.riot.processor.DataStructureItemProcessor; import com.redis.spring.batch.DataStructure; -import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; +import picocli.CommandLine.Parameters; @Command(name = "export", description = "Export to a database") public class DatabaseExportCommand extends AbstractExportCommand> { - private static final Logger log = LoggerFactory.getLogger(DatabaseExportCommand.class); + private static Logger log = Logger.getLogger(DatabaseExportCommand.class.getName()); private static final String NAME = "db-export"; - @CommandLine.Parameters(arity = "1", description = "SQL INSERT statement.", paramLabel = "SQL") + @Parameters(arity = "1", description = "SQL INSERT statement.", paramLabel = "SQL") private String sql; @Mixin private DataSourceOptions dataSourceOptions = new DataSourceOptions(); @@ -46,11 +46,11 @@ public DataSourceOptions getDataSourceOptions() { @Override protected Job job(JobBuilder jobBuilder) throws Exception { - log.debug("Creating data source with {}", dataSourceOptions); + log.log(Level.FINE, "Creating data source with {0}", dataSourceOptions); DataSource dataSource = dataSourceOptions.dataSource(); try (Connection connection = dataSource.getConnection()) { String dbName = connection.getMetaData().getDatabaseProductName(); - log.debug("Creating writer for database {} with {}", dbName, exportOptions); + log.log(Level.FINE, "Creating writer for database {0} with {1}", new Object[] { dbName, exportOptions }); JdbcBatchItemWriterBuilder> builder = new JdbcBatchItemWriterBuilder<>(); builder.itemSqlParameterSourceProvider(NullableMapSqlParameterSource::new); builder.dataSource(dataSource); diff --git a/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseExportOptions.java b/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseExportOptions.java index 4d8a73a63..71fa07d3d 100644 --- a/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseExportOptions.java +++ b/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseExportOptions.java @@ -1,12 +1,12 @@ package com.redis.riot.db; -import picocli.CommandLine; +import picocli.CommandLine.Option; public class DatabaseExportOptions { - @CommandLine.Option(names = "--key-regex", description = "Regex for key-field extraction (default: ${DEFAULT-VALUE}).", paramLabel = "") + @Option(names = "--key-regex", description = "Regex for key-field extraction (default: ${DEFAULT-VALUE}).", paramLabel = "") private String keyRegex = "\\w+:(?.+)"; - @CommandLine.Option(names = "--no-assert-updates", description = "Confirm every insert results in update of at least one row. True by default.", negatable = true) + @Option(names = "--no-assert-updates", description = "Confirm every insert results in update of at least one row. True by default.", negatable = true) private boolean assertUpdates = true; public boolean isAssertUpdates() { @@ -24,4 +24,10 @@ public String getKeyRegex() { public void setKeyRegex(String keyRegex) { this.keyRegex = keyRegex; } + + @Override + public String toString() { + return "DatabaseExportOptions [keyRegex=" + keyRegex + ", assertUpdates=" + assertUpdates + "]"; + } + } diff --git a/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseImportCommand.java b/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseImportCommand.java index 30a10a2f0..7731447d6 100644 --- a/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseImportCommand.java +++ b/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseImportCommand.java @@ -2,11 +2,11 @@ import java.sql.Connection; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.sql.DataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.item.database.JdbcCursorItemReader; @@ -15,18 +15,18 @@ import com.redis.riot.AbstractImportCommand; -import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; +import picocli.CommandLine.Parameters; @Command(name = "import", description = "Import from a database") public class DatabaseImportCommand extends AbstractImportCommand { - private static final Logger log = LoggerFactory.getLogger(DatabaseImportCommand.class); + private static final Logger log = Logger.getLogger(DatabaseImportCommand.class.getName()); private static final String NAME = "db-import"; - @CommandLine.Parameters(arity = "1", description = "SQL SELECT statement", paramLabel = "SQL") + @Parameters(arity = "1", description = "SQL SELECT statement", paramLabel = "SQL") private String sql; @Mixin private DataSourceOptions dataSourceOptions = new DataSourceOptions(); @@ -39,11 +39,11 @@ public DataSourceOptions getDataSourceOptions() { @Override protected Job job(JobBuilder jobBuilder) throws Exception { - log.debug("Creating data source: {}", dataSourceOptions); + log.log(Level.FINE, "Creating data source: {0}", dataSourceOptions); DataSource dataSource = dataSourceOptions.dataSource(); try (Connection connection = dataSource.getConnection()) { String name = connection.getMetaData().getDatabaseProductName(); - log.debug("Creating {} database reader: {}", name, importOptions); + log.log(Level.FINE, "Creating {0} database reader: {1}", new Object[] { name, importOptions }); JdbcCursorItemReaderBuilder> builder = new JdbcCursorItemReaderBuilder<>(); builder.saveState(false); builder.dataSource(dataSource); diff --git a/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseImportOptions.java b/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseImportOptions.java index 4f139173b..d25850b60 100644 --- a/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseImportOptions.java +++ b/connectors/riot-db/src/main/java/com/redis/riot/db/DatabaseImportOptions.java @@ -5,19 +5,19 @@ import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder; -import picocli.CommandLine; +import picocli.CommandLine.Option; public class DatabaseImportOptions { - @CommandLine.Option(names = "--fetch", description = "Number of rows to return with each fetch.", paramLabel = "") + @Option(names = "--fetch", description = "Number of rows to return with each fetch.", paramLabel = "") private Optional fetchSize = Optional.empty(); - @CommandLine.Option(names = "--rows", description = "Max number of rows the ResultSet can contain.", paramLabel = "") + @Option(names = "--rows", description = "Max number of rows the ResultSet can contain.", paramLabel = "") private Optional maxRows = Optional.empty(); - @CommandLine.Option(names = "--query-timeout", description = "The time in milliseconds for the query to timeout.", paramLabel = "") + @Option(names = "--query-timeout", description = "The time in milliseconds for the query to timeout.", paramLabel = "") private Optional queryTimeout = Optional.empty(); - @CommandLine.Option(names = "--shared-connection", description = "Use same connection for cursor and other processing.", hidden = true) + @Option(names = "--shared-connection", description = "Use same connection for cursor and other processing.", hidden = true) private boolean useSharedExtendedConnection; - @CommandLine.Option(names = "--verify", description = "Verify position of result set after row mapper.", hidden = true) + @Option(names = "--verify", description = "Verify position of result set after row mapper.", hidden = true) private boolean verifyCursorPosition; public void setFetchSize(int fetchSize) { @@ -48,4 +48,11 @@ public void configure(JdbcCursorItemReaderBuilder> builder) builder.verifyCursorPosition(verifyCursorPosition); } + @Override + public String toString() { + return "DatabaseImportOptions [fetchSize=" + fetchSize + ", maxRows=" + maxRows + ", queryTimeout=" + + queryTimeout + ", useSharedExtendedConnection=" + useSharedExtendedConnection + + ", verifyCursorPosition=" + verifyCursorPosition + "]"; + } + } diff --git a/connectors/riot-file/riot-file.gradle b/connectors/riot-file/riot-file.gradle index 5e595201e..2b37d9c1f 100644 --- a/connectors/riot-file/riot-file.gradle +++ b/connectors/riot-file/riot-file.gradle @@ -15,6 +15,7 @@ dependencies { implementation (group: 'org.springframework.cloud', name: 'spring-cloud-gcp-starter-storage', version: gcpVersion) { exclude group: 'javax.annotation', module: 'javax.annotation-api' } + testImplementation 'org.slf4j:slf4j-simple' testImplementation project(':riot-test') } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileImportCommand.java b/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileImportCommand.java index 85c6fabb8..8d5efc55a 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileImportCommand.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileImportCommand.java @@ -31,9 +31,10 @@ import io.lettuce.core.ScoredValue; import io.lettuce.core.StreamMessage; -import picocli.CommandLine; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Parameters; @Command(name = "import-dump", description = "Import Redis data files into Redis") public class DumpFileImportCommand extends AbstractTransferCommand { @@ -42,10 +43,10 @@ public class DumpFileImportCommand extends AbstractTransferCommand { private static final String NAME = "dump-file-import"; - @CommandLine.Parameters(arity = "0..*", description = "One ore more files or URLs", paramLabel = "FILE") + @Parameters(arity = "0..*", description = "One ore more files or URLs", paramLabel = "FILE") private List files; - @CommandLine.Mixin - private DumpFileImportOptions options = new DumpFileImportOptions(); + @Mixin + private DumpFileOptions options = new DumpFileOptions(); @ArgGroup(exclusive = false, heading = "Writer options%n") private RedisWriterOptions writerOptions = new RedisWriterOptions(); @@ -57,7 +58,7 @@ public void setFiles(List files) { this.files = files; } - public DumpFileImportOptions getOptions() { + public DumpFileOptions getOptions() { return options; } @@ -88,9 +89,8 @@ private List fileImportSteps() throws Exception { } private TaskletStep fileImportStep(String file) throws Exception { - DumpFileType fileType = DumpFileType.of(file, options.getType()); Resource resource = options.inputResource(file); - AbstractItemStreamItemReader> reader = reader(fileType, resource); + AbstractItemStreamItemReader> reader = reader(options.type(resource), resource); reader.setName(file + "-" + NAME + "-reader"); return step(RiotStep.reader(reader).writer(writer()).name(file + "-" + NAME).taskName("Importing " + file) .processor(this::processDataStructure).build()).build(); diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileImportOptions.java b/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileImportOptions.java deleted file mode 100644 index 6b6d69445..000000000 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileImportOptions.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.redis.riot.file; - -import java.util.Optional; - -import picocli.CommandLine; - -public class DumpFileImportOptions extends FileOptions { - - @CommandLine.Option(names = { "-t", - "--filetype" }, description = "File type: ${COMPLETION-CANDIDATES}", paramLabel = "") - private Optional type = Optional.empty(); - - public Optional getType() { - return type; - } - - public void setType(DumpFileType type) { - this.type = Optional.of(type); - } - -} diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileOptions.java b/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileOptions.java new file mode 100644 index 000000000..fc272cd98 --- /dev/null +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileOptions.java @@ -0,0 +1,47 @@ +package com.redis.riot.file; + +import java.util.Optional; + +import org.springframework.core.io.Resource; + +import picocli.CommandLine.Option; + +public class DumpFileOptions extends FileOptions { + + @Option(names = { "-t", "--filetype" }, description = "File type: ${COMPLETION-CANDIDATES}", paramLabel = "") + protected Optional type = Optional.empty(); + + public Optional getType() { + return type; + } + + public void setType(DumpFileType type) { + this.type = Optional.of(type); + } + + @Override + public String toString() { + return "DumpFileOptions [type=" + type + ", encoding=" + encoding + ", gzip=" + gzip + ", s3=" + s3 + ", gcs=" + + gcs + "]"; + } + + public DumpFileType type(Resource resource) { + if (type.isPresent()) { + return type.get(); + } + Optional extension = FileUtils.extension(resource); + if (extension.isEmpty()) { + throw new UnknownFileTypeException("Unknown file extension"); + } + switch (extension.get()) { + case XML: + return DumpFileType.XML; + case JSON: + return DumpFileType.JSON; + default: + throw new UnsupportedOperationException("Unsupported file extension: " + extension.get()); + } + + } + +} diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileType.java b/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileType.java index ffdc421c7..5331cfa58 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileType.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/DumpFileType.java @@ -1,19 +1,7 @@ package com.redis.riot.file; -import java.util.Optional; - public enum DumpFileType { - JSON, XML; + JSON, XML - public static DumpFileType of(String file, Optional type) { - if (type.isPresent()) { - return type.get(); - } - Optional extension = FileUtils.extension(file); - if (extension.isPresent() && extension.get().equalsIgnoreCase(FileUtils.EXTENSION_XML)) { - return XML; - } - return JSON; - } } \ No newline at end of file diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileExportCommand.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileExportCommand.java index fd1dd8e82..6cb52cb31 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/FileExportCommand.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileExportCommand.java @@ -17,8 +17,9 @@ import com.redis.riot.file.resource.XmlResourceItemWriterBuilder; import com.redis.spring.batch.DataStructure; -import picocli.CommandLine; +import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; +import picocli.CommandLine.Parameters; @Command(name = "export", description = "Export Redis data to JSON or XML files") public class FileExportCommand extends AbstractExportCommand> { @@ -27,9 +28,9 @@ public class FileExportCommand extends AbstractExportCommand> writer(WritableResource resource) { - DumpFileType fileType = DumpFileType.of(file, options.getType()); - if (fileType == DumpFileType.XML) { + DumpFileType type = options.type(resource); + switch (type) { + case XML: XmlResourceItemWriterBuilder> xmlWriterBuilder = new XmlResourceItemWriterBuilder<>(); xmlWriterBuilder.name("xml-resource-item-writer"); xmlWriterBuilder.append(options.isAppend()); @@ -65,17 +67,20 @@ private ItemWriter> writer(WritableResource resource) { xmlWriterBuilder.saveState(false); log.debug("Creating XML writer with {} for file {}", options, file); return xmlWriterBuilder.build(); + case JSON: + JsonResourceItemWriterBuilder> jsonWriterBuilder = new JsonResourceItemWriterBuilder<>(); + jsonWriterBuilder.name("json-resource-item-writer"); + jsonWriterBuilder.append(options.isAppend()); + jsonWriterBuilder.encoding(options.getEncoding().name()); + jsonWriterBuilder.jsonObjectMarshaller(new JacksonJsonObjectMarshaller<>()); + jsonWriterBuilder.lineSeparator(options.getLineSeparator()); + jsonWriterBuilder.resource(resource); + jsonWriterBuilder.saveState(false); + log.debug("Creating JSON writer with {} for file {}", options, file); + return jsonWriterBuilder.build(); + default: + throw new UnsupportedOperationException("Unsupported file type: " + type); } - JsonResourceItemWriterBuilder> jsonWriterBuilder = new JsonResourceItemWriterBuilder<>(); - jsonWriterBuilder.name("json-resource-item-writer"); - jsonWriterBuilder.append(options.isAppend()); - jsonWriterBuilder.encoding(options.getEncoding().name()); - jsonWriterBuilder.jsonObjectMarshaller(new JacksonJsonObjectMarshaller<>()); - jsonWriterBuilder.lineSeparator(options.getLineSeparator()); - jsonWriterBuilder.resource(resource); - jsonWriterBuilder.saveState(false); - log.debug("Creating JSON writer with {} for file {}", options, file); - return jsonWriterBuilder.build(); } private JsonObjectMarshaller> xmlMarshaller() { diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileExportOptions.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileExportOptions.java index 23a3e0676..1054948ff 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/FileExportOptions.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileExportOptions.java @@ -1,36 +1,23 @@ package com.redis.riot.file; -import java.util.Optional; - import org.springframework.batch.item.support.AbstractFileItemWriter; -import picocli.CommandLine; +import picocli.CommandLine.Option; -public class FileExportOptions extends FileOptions { +public class FileExportOptions extends DumpFileOptions { public static final String DEFAULT_ELEMENT_NAME = "record"; public static final String DEFAULT_ROOT_NAME = "root"; - @CommandLine.Option(names = { "-t", - "--filetype" }, description = "File type: ${COMPLETION-CANDIDATES}", paramLabel = "") - private Optional type = Optional.empty(); - @CommandLine.Option(names = "--append", description = "Append to file if it exists") + @Option(names = "--append", description = "Append to file if it exists") private boolean append; - @CommandLine.Option(names = "--root", description = "XML root element tag name (default: ${DEFAULT-VALUE})", paramLabel = "") + @Option(names = "--root", description = "XML root element tag name (default: ${DEFAULT-VALUE})", paramLabel = "") private String rootName = DEFAULT_ROOT_NAME; - @CommandLine.Option(names = "--element", description = "XML element tag name (default: ${DEFAULT-VALUE})", paramLabel = "") + @Option(names = "--element", description = "XML element tag name (default: ${DEFAULT-VALUE})", paramLabel = "") private String elementName = DEFAULT_ELEMENT_NAME; - @CommandLine.Option(names = "--line-sep", description = "String to separate lines (default: system default)", paramLabel = "") + @Option(names = "--line-sep", description = "String to separate lines (default: system default)", paramLabel = "") private String lineSeparator = AbstractFileItemWriter.DEFAULT_LINE_SEPARATOR; - public Optional getType() { - return type; - } - - public void setType(DumpFileType type) { - this.type = Optional.of(type); - } - public boolean isAppend() { return append; } @@ -63,4 +50,11 @@ public void setLineSeparator(String lineSeparator) { this.lineSeparator = lineSeparator; } + @Override + public String toString() { + return "FileExportOptions [type=" + type + ", append=" + append + ", rootName=" + rootName + ", elementName=" + + elementName + ", lineSeparator=" + lineSeparator + ", encoding=" + encoding + ", gzip=" + gzip + + ", s3=" + s3 + ", gcs=" + gcs + "]"; + } + } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileExtension.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileExtension.java new file mode 100644 index 000000000..8c89ef16a --- /dev/null +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileExtension.java @@ -0,0 +1,7 @@ +package com.redis.riot.file; + +public enum FileExtension { + + CSV, TSV, PSV, FW, JSON, XML, GZ + +} diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileImportCommand.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileImportCommand.java index f2fb02df0..f3397fce5 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/FileImportCommand.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileImportCommand.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -35,35 +34,29 @@ import org.springframework.util.ObjectUtils; import com.redis.riot.AbstractImportCommand; +import com.redis.riot.file.FileImportOptions.FileType; import com.redis.riot.file.resource.XmlItemReader; -import picocli.CommandLine; +import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; +import picocli.CommandLine.Parameters; @Command(name = "import", description = "Import delimited, fixed-width, JSON, or XML files into Redis.") public class FileImportCommand extends AbstractImportCommand { - public enum FileType { - DELIMITED, FIXED, JSON, XML - } - private static final Logger log = LoggerFactory.getLogger(FileImportCommand.class); - private static final String NAME = "file-import"; + private static final String DELIMITER_PIPE = "|"; - @CommandLine.Parameters(arity = "0..*", description = "One ore more files or URLs", paramLabel = "FILE") + @Parameters(arity = "0..*", description = "One ore more files or URLs", paramLabel = "FILE") private List files = new ArrayList<>(); - @CommandLine.Option(names = { "-t", - "--filetype" }, description = "File type: ${COMPLETION-CANDIDATES}", paramLabel = "") - private Optional type = Optional.empty(); - @CommandLine.ArgGroup(exclusive = false, heading = "Delimited and fixed-width file options%n") + @ArgGroup(exclusive = false, heading = "Delimited and fixed-width file options%n") private FileImportOptions options = new FileImportOptions(); public FileImportCommand() { } private FileImportCommand(Builder builder) { - this.type = builder.fileType; this.options = builder.options; } @@ -75,14 +68,6 @@ public void setFiles(List files) { this.files = files; } - public Optional getType() { - return type; - } - - public void setType(FileType type) { - this.type = Optional.of(type); - } - public FileImportOptions getOptions() { return options; } @@ -120,40 +105,39 @@ private Resource resource(String file) throws IOException { return options.inputResource(file); } - private Optional type(Optional extension) { + private FileType type(Optional extension) { + Optional type = options.getType(); if (type.isPresent()) { - return type; - } - if (extension.isEmpty()) { - return Optional.empty(); + return type.get(); } - switch (extension.get().toLowerCase()) { - case FileUtils.EXTENSION_FW: - return Optional.of(FileType.FIXED); - case FileUtils.EXTENSION_JSON: - return Optional.of(FileType.JSON); - case FileUtils.EXTENSION_XML: - return Optional.of(FileType.XML); - case FileUtils.EXTENSION_CSV: - case FileUtils.EXTENSION_PSV: - case FileUtils.EXTENSION_TSV: - return Optional.of(FileType.DELIMITED); - default: - return Optional.empty(); + if (extension.isPresent()) { + switch (extension.get()) { + case FW: + return FileType.FIXED; + case JSON: + return FileType.JSON; + case XML: + return FileType.XML; + case CSV: + case PSV: + case TSV: + return FileType.DELIMITED; + default: + throw new UnknownFileTypeException("Unknown file extension: " + extension); + } } + throw new UnknownFileTypeException("Could not determine file type"); } @SuppressWarnings({ "unchecked", "rawtypes" }) private AbstractItemStreamItemReader> reader(Resource resource) { - Optional extension = FileUtils.extension(resource.getFilename()); - Optional fileType = type(extension); - if (fileType.isEmpty()) { - throw new IllegalArgumentException("Could not determine file type for " + resource); - } - switch (fileType.get()) { + Optional extension = FileUtils.extension(resource); + FileType type = type(extension); + switch (type) { case DELIMITED: DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer(); - tokenizer.setDelimiter(options.delimiter(extension)); + options.getDelimiter().ifPresentOrElse(tokenizer::setDelimiter, + () -> tokenizer.setDelimiter(delimiter(extension.get()))); tokenizer.setQuoteCharacter(options.getQuoteCharacter()); if (!ObjectUtils.isEmpty(options.getIncludedFields())) { tokenizer.setIncludedFields(options.getIncludedFields()); @@ -167,8 +151,7 @@ private AbstractItemStreamItemReader> reader(Resource resour editor.setAsText(String.join(",", options.getColumnRanges())); Range[] ranges = (Range[]) editor.getValue(); if (ranges.length == 0) { - throw new IllegalArgumentException( - "Invalid ranges specified: " + Arrays.toString(options.getColumnRanges())); + throw new IllegalArgumentException("Invalid ranges specified: " + options.getColumnRanges()); } fixedLengthTokenizer.setColumns(ranges); log.debug("Creating fixed-width reader with {} for {}", options, resource); @@ -176,15 +159,30 @@ private AbstractItemStreamItemReader> reader(Resource resour case XML: log.debug("Creating XML reader for {}", resource); return (XmlItemReader) FileUtils.xmlReader(resource, Map.class); - default: + case JSON: log.debug("Creating JSON reader for {}", resource); return (JsonItemReader) FileUtils.jsonReader(resource, Map.class); + default: + throw new UnsupportedOperationException("Unsupported file type: " + type); + } + } + + private String delimiter(FileExtension extension) { + switch (extension) { + case CSV: + return DelimitedLineTokenizer.DELIMITER_COMMA; + case PSV: + return DELIMITER_PIPE; + case TSV: + return DelimitedLineTokenizer.DELIMITER_TAB; + default: + throw new IllegalArgumentException("Unknown extension: " + extension); } } private FlatFileItemReader> flatFileReader(Resource resource, AbstractLineTokenizer tokenizer) { if (!ObjectUtils.isEmpty(options.getNames())) { - tokenizer.setNames(options.getNames()); + tokenizer.setNames(options.getNames().toArray(String[]::new)); } FlatFileItemReaderBuilder> builder = new FlatFileItemReaderBuilder<>(); builder.resource(resource); @@ -249,7 +247,6 @@ public static Builder builder() { public static class Builder { private FileImportOptions options = new FileImportOptions(); - private Optional fileType = Optional.empty(); public Builder options(FileImportOptions options) { Assert.notNull(options, "Options must not be null"); @@ -257,12 +254,6 @@ public Builder options(FileImportOptions options) { return this; } - public Builder type(FileType fileType) { - Assert.notNull(fileType, "File type must not be null"); - this.fileType = Optional.of(fileType); - return this; - } - public FileImportCommand build() { return new FileImportCommand(this); } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileImportOptions.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileImportOptions.java index 5e5001964..30c834f3c 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/FileImportOptions.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileImportOptions.java @@ -1,5 +1,8 @@ package com.redis.riot.file; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Optional; import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; @@ -8,11 +11,16 @@ public class FileImportOptions extends FileOptions { + public enum FileType { + DELIMITED, FIXED, JSON, XML + } + public static final String DEFAULT_CONTINUATION_STRING = "\\"; - private static final String DELIMITER_PIPE = "|"; + @Option(names = { "-t", "--filetype" }, description = "File type: ${COMPLETION-CANDIDATES}", paramLabel = "") + private Optional type = Optional.empty(); @Option(names = "--fields", arity = "1..*", description = "Delimited/FW field names", paramLabel = "") - private String[] names; + private List names = new ArrayList<>(); @Option(names = { "-h", "--header" }, description = "Delimited/FW first line contains field names") private boolean header; @Option(names = "--delimiter", description = "Delimiter character", paramLabel = "") @@ -22,18 +30,18 @@ public class FileImportOptions extends FileOptions { @Option(names = "--include", arity = "1..*", description = "Delimited/FW field indices to include (0-based)", paramLabel = "") private int[] includedFields; @Option(names = "--ranges", arity = "1..*", description = "Fixed-width column ranges", paramLabel = "") - private String[] columnRanges; + private List columnRanges = new ArrayList<>(); @Option(names = "--quote", description = "Escape character for delimited files (default: ${DEFAULT-VALUE})", paramLabel = "") private Character quoteCharacter = DelimitedLineTokenizer.DEFAULT_QUOTE_CHARACTER; @Option(names = "--cont", description = "Line continuation string (default: ${DEFAULT-VALUE})", paramLabel = "") private String continuationString = DEFAULT_CONTINUATION_STRING; public FileImportOptions() { - } private FileImportOptions(Builder builder) { super(builder); + this.type = builder.type; this.names = builder.names; this.header = builder.header; this.delimiter = builder.delimiter; @@ -44,12 +52,20 @@ private FileImportOptions(Builder builder) { this.continuationString = builder.continuationString; } - public String[] getNames() { + public List getNames() { return names; } - public void setNames(String[] names) { - this.names = names; + public Optional getType() { + return type; + } + + public void setType(FileType type) { + this.type = Optional.of(type); + } + + public void setNames(String... names) { + this.names = Arrays.asList(names); } public boolean isHeader() { @@ -72,16 +88,20 @@ public int[] getIncludedFields() { return includedFields; } - public void setIncludedFields(int[] includedFields) { + public void setIncludedFields(int... includedFields) { this.includedFields = includedFields; } - public String[] getColumnRanges() { + public List getColumnRanges() { return columnRanges; } - public void setColumnRanges(String[] columnRanges) { - this.columnRanges = columnRanges; + public void setColumnRanges(String... columnRanges) { + this.columnRanges = Arrays.asList(columnRanges); + } + + public Optional getDelimiter() { + return delimiter; } public Character getQuoteCharacter() { @@ -110,44 +130,31 @@ public int getLinesToSkip() { return 0; } - public String delimiter(Optional extension) { - if (delimiter.isPresent()) { - return delimiter.get(); - } - if (extension.isEmpty()) { - throw new IllegalArgumentException("Could not determine delimiter for extension " + extension); - } - switch (extension.get().toLowerCase()) { - case FileUtils.EXTENSION_CSV: - return DelimitedLineTokenizer.DELIMITER_COMMA; - case FileUtils.EXTENSION_PSV: - return DELIMITER_PIPE; - case FileUtils.EXTENSION_TSV: - return DelimitedLineTokenizer.DELIMITER_TAB; - default: - throw new IllegalArgumentException("Unknown extension: " + extension); - } - } - public static Builder builder() { return new Builder(); } public static final class Builder extends FileOptions.Builder { - private String[] names; + private Optional type = Optional.empty(); + private List names = new ArrayList<>(); private boolean header; private Optional delimiter = Optional.empty(); private Optional linesToSkip = Optional.empty(); private int[] includedFields; - private String[] columnRanges; + private List columnRanges = new ArrayList<>(); private Character quoteCharacter = DelimitedLineTokenizer.DEFAULT_QUOTE_CHARACTER; private String continuationString = DEFAULT_CONTINUATION_STRING; private Builder() { } - public Builder names(String[] names) { - this.names = names; + public Builder type(FileType type) { + this.type = Optional.of(type); + return this; + } + + public Builder names(String... names) { + this.names = Arrays.asList(names); return this; } @@ -166,13 +173,13 @@ public Builder linesToSkip(Optional linesToSkip) { return this; } - public Builder includedFields(int[] includedFields) { + public Builder includedFields(int... includedFields) { this.includedFields = includedFields; return this; } - public Builder columnRanges(String[] columnRanges) { - this.columnRanges = columnRanges; + public Builder columnRanges(String... columnRanges) { + this.columnRanges = Arrays.asList(columnRanges); return this; } @@ -191,4 +198,13 @@ public FileImportOptions build() { } } + @Override + public String toString() { + return "FileImportOptions [type=" + type + ", names=" + names + ", header=" + header + ", delimiter=" + + delimiter + ", linesToSkip=" + linesToSkip + ", includedFields=" + Arrays.toString(includedFields) + + ", columnRanges=" + columnRanges + ", quoteCharacter=" + quoteCharacter + ", continuationString=" + + continuationString + ", encoding=" + encoding + ", gzip=" + gzip + ", s3=" + s3 + ", gcs=" + gcs + + "]"; + } + } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileOptions.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileOptions.java index eec17ab48..d83c74e70 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/FileOptions.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileOptions.java @@ -24,13 +24,13 @@ public class FileOptions { public static final Charset DEFAULT_ENCODING = Charset.defaultCharset(); @Option(names = "--encoding", description = "File encoding (default: ${DEFAULT-VALUE})", paramLabel = "") - private Charset encoding = DEFAULT_ENCODING; + protected Charset encoding = DEFAULT_ENCODING; @Option(names = { "-z", "--gzip" }, description = "File is gzip compressed") - private boolean gzip; + protected boolean gzip; @ArgGroup(exclusive = false, heading = "Amazon Simple Storage Service options%n") - private S3Options s3 = new S3Options(); + protected S3Options s3 = new S3Options(); @ArgGroup(exclusive = false, heading = "Google Cloud Storage options%n") - private GcsOptions gcs = new GcsOptions(); + protected GcsOptions gcs = new GcsOptions(); public FileOptions() { } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileUtils.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileUtils.java index b5863de59..d6f61fbe6 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/FileUtils.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileUtils.java @@ -31,26 +31,29 @@ public interface FileUtils { public static final String GS_URI_PREFIX = "gs://"; public static final String S3_URI_PREFIX = "s3://"; - public static final Pattern EXTENSION_PATTERN = Pattern.compile("(?i)\\.(?\\w+)(?\\.gz)?$"); - public static final String EXTENSION_CSV = "csv"; - public static final String EXTENSION_TSV = "tsv"; - public static final String EXTENSION_PSV = "psv"; - public static final String EXTENSION_FW = "fw"; - public static final String EXTENSION_JSON = "json"; - public static final String EXTENSION_XML = "xml"; + public static final Pattern EXTENSION_PATTERN = Pattern.compile("(?i)\\.(?\\w+)(?:\\.(?gz))?$"); public static boolean isGzip(String file) { return extensionGroup(file, "gz").isPresent(); } - public static Optional extension(String file) { - return extensionGroup(file, "extension"); + public static Optional extension(Resource resource) { + return extensionGroup(resource.getFilename(), "extension"); } - public static Optional extensionGroup(String file, String group) { + public static Optional extensionGroup(String file, String group) { Matcher matcher = EXTENSION_PATTERN.matcher(file); if (matcher.find()) { - return Optional.ofNullable(matcher.group(group)); + String extensionString = matcher.group(group); + if (extensionString == null) { + return Optional.empty(); + } + try { + return Optional.of(FileExtension.valueOf(extensionString.toUpperCase())); + } catch (Exception e) { + // do nothing + } + return Optional.empty(); } return Optional.empty(); } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/GcsOptions.java b/connectors/riot-file/src/main/java/com/redis/riot/file/GcsOptions.java index 15224cba7..49bbef427 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/GcsOptions.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/GcsOptions.java @@ -54,4 +54,10 @@ public GoogleStorageResource resource(String locationUri, boolean readOnly) thro return new GoogleStorageResource(builder.build().getService(), locationUri); } + @Override + public String toString() { + return "GcsOptions [credentials=" + credentials + ", projectId=" + projectId + ", encodedKey=" + encodedKey + + "]"; + } + } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/RiotFile.java b/connectors/riot-file/src/main/java/com/redis/riot/file/RiotFile.java index 4d23dc056..3fa049f1b 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/RiotFile.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/RiotFile.java @@ -4,19 +4,21 @@ import java.util.logging.Logger; import com.redis.riot.RiotApp; + import picocli.CommandLine.Command; -@Command(name = "riot-file", subcommands = {FileImportCommand.class, DumpFileImportCommand.class, FileExportCommand.class}) +@Command(name = "riot-file", subcommands = { FileImportCommand.class, DumpFileImportCommand.class, + FileExportCommand.class }) public class RiotFile extends RiotApp { - public static void main(String[] args) { - System.exit(new RiotFile().execute(args)); - } - - @Override - protected void configureLogging() { - super.configureLogging(); - Logger.getLogger("com.amazonaws").setLevel(Level.SEVERE); - } + public static void main(String[] args) { + System.exit(new RiotFile().execute(args)); + } + + @Override + protected void configureLogging() { + super.configureLogging(); + Logger.getLogger("com.amazonaws").setLevel(Level.SEVERE); + } } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/S3Options.java b/connectors/riot-file/src/main/java/com/redis/riot/file/S3Options.java index 3faa28bc6..6d1e250a6 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/S3Options.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/S3Options.java @@ -75,4 +75,10 @@ public void refresh() { } } + + @Override + public String toString() { + return "S3Options [accessKey=" + accessKey + ", secretKey=" + secretKey + ", region=" + region + "]"; + } + } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/UnknownFileTypeException.java b/connectors/riot-file/src/main/java/com/redis/riot/file/UnknownFileTypeException.java new file mode 100644 index 000000000..c21e7be7c --- /dev/null +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/UnknownFileTypeException.java @@ -0,0 +1,11 @@ +package com.redis.riot.file; + +public class UnknownFileTypeException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public UnknownFileTypeException(String message) { + super(message); + } + +} \ No newline at end of file diff --git a/connectors/riot-file/src/test/java/com/redis/riot/file/FileIntegrationTests.java b/connectors/riot-file/src/test/java/com/redis/riot/file/FileIntegrationTests.java index 5d73bcca8..5e3c993c6 100644 --- a/connectors/riot-file/src/test/java/com/redis/riot/file/FileIntegrationTests.java +++ b/connectors/riot-file/src/test/java/com/redis/riot/file/FileIntegrationTests.java @@ -432,8 +432,8 @@ void importJsonAPI(RedisTestContext redis) throws Exception { FileImportCommand command = new FileImportCommand(); command.setFiles(Collections.singletonList(BEERS_JSON_URL)); HsetCommand hset = new HsetCommand(); - hset.setKeyspace("beer"); - hset.setKeys(new String[] { "id" }); + hset.getKeyOptions().setKeyspace("beer"); + hset.getKeyOptions().setKeys(new String[] { "id" }); command.setRedisCommands(Collections.singletonList(hset)); RiotFile riotFile = new RiotFile(); configure(riotFile, redis); diff --git a/connectors/riot-gen/src/main/java/com/redis/riot/gen/DataStructureGeneratorCommand.java b/connectors/riot-gen/src/main/java/com/redis/riot/gen/DataStructureGeneratorCommand.java index dd1af62fb..9412b5911 100644 --- a/connectors/riot-gen/src/main/java/com/redis/riot/gen/DataStructureGeneratorCommand.java +++ b/connectors/riot-gen/src/main/java/com/redis/riot/gen/DataStructureGeneratorCommand.java @@ -1,9 +1,9 @@ package com.redis.riot.gen; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.item.ItemReader; @@ -16,18 +16,18 @@ import com.redis.spring.batch.RedisItemWriter; import com.redis.spring.batch.reader.DataStructureGeneratorItemReader; -import picocli.CommandLine; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; @Command(name = "ds", description = "Import randomly-generated data structures") public class DataStructureGeneratorCommand extends AbstractTransferCommand { - private static final Logger log = LoggerFactory.getLogger(DataStructureGeneratorCommand.class); + private static final Logger log = Logger.getLogger(DataStructureGeneratorCommand.class.getName()); private static final String NAME = "random-import"; - @CommandLine.Mixin + @Mixin private DataStructureGeneratorOptions options = new DataStructureGeneratorOptions(); @ArgGroup(exclusive = false, heading = "Writer options%n") @@ -39,7 +39,7 @@ protected Job job(JobBuilder jobBuilder) throws Exception { .configureWriter( RedisItemWriter.client(getRedisOptions().client()).string().dataStructure().xaddArgs(m -> null)) .build(); - log.debug("Creating random data structure reader with {}", options); + log.log(Level.FINE, "Creating random data structure reader with {0}", options); return jobBuilder.start(step(RiotStep.reader(reader()).writer(writer).name(NAME).taskName("Generating") .max(() -> (long) options.getCount()).build()).build()).build(); } diff --git a/connectors/riot-gen/src/main/java/com/redis/riot/gen/DataStructureGeneratorOptions.java b/connectors/riot-gen/src/main/java/com/redis/riot/gen/DataStructureGeneratorOptions.java index 1b1e29a15..46157056b 100644 --- a/connectors/riot-gen/src/main/java/com/redis/riot/gen/DataStructureGeneratorOptions.java +++ b/connectors/riot-gen/src/main/java/com/redis/riot/gen/DataStructureGeneratorOptions.java @@ -191,4 +191,15 @@ public void setExpiration(Optional expiration) { this.expiration = expiration; } + @Override + public String toString() { + return "DataStructureGeneratorOptions [keyspace=" + keyspace + ", types=" + types + ", expiration=" + expiration + + ", hashSize=" + hashSize + ", hashFieldSize=" + hashFieldSize + ", jsonSize=" + jsonSize + + ", jsonFieldSize=" + jsonFieldSize + ", listSize=" + listSize + ", setSize=" + setSize + + ", streamSize=" + streamSize + ", streamFieldCount=" + streamFieldCount + ", streamFieldSize=" + + streamFieldSize + ", stringSize=" + stringSize + ", timeseriesSize=" + timeseriesSize + + ", timeseriesStartTime=" + timeseriesStartTime + ", zsetSize=" + zsetSize + ", zsetScore=" + zsetScore + + ", start=" + start + ", count=" + count + "]"; + } + } diff --git a/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerGeneratorCommand.java b/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerGeneratorCommand.java index 01e409079..a6490b2ad 100644 --- a/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerGeneratorCommand.java +++ b/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerGeneratorCommand.java @@ -3,9 +3,9 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; @@ -18,17 +18,17 @@ import com.redis.lettucemod.search.IndexInfo; import com.redis.riot.AbstractImportCommand; -import picocli.CommandLine; import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; @Command(name = "faker", description = "Import Faker data using the Spring Expression Language (SpEL)") public class FakerGeneratorCommand extends AbstractImportCommand { - private static final Logger log = LoggerFactory.getLogger(FakerGeneratorCommand.class); + private static final Logger log = Logger.getLogger(FakerGeneratorCommand.class.getName()); private static final String NAME = "faker-import"; - @CommandLine.Mixin + @Mixin private FakerGeneratorOptions options = new FakerGeneratorOptions(); @Override @@ -43,7 +43,7 @@ protected Long initialMax() { } private ItemReader> reader() { - log.debug("Creating Faker reader with {}", options); + log.log(Level.FINE, "Creating Faker reader with {0}", options); FakerItemReader reader = new FakerItemReader(generator()); reader.setStart(options.getStart()); reader.setCount(options.getCount()); diff --git a/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerGeneratorOptions.java b/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerGeneratorOptions.java index fd263c98a..0ee8934d5 100644 --- a/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerGeneratorOptions.java +++ b/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerGeneratorOptions.java @@ -4,17 +4,18 @@ import java.util.Map; import java.util.Optional; -import picocli.CommandLine; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; public class FakerGeneratorOptions extends GeneratorOptions { - @CommandLine.Parameters(arity = "0..*", description = "SpEL expressions in the form field1=\"exp\" field2=\"exp\"...", paramLabel = "SPEL") + @Parameters(arity = "0..*", description = "SpEL expressions in the form field1=\"exp\" field2=\"exp\"...", paramLabel = "SPEL") private Map fields; - @CommandLine.Option(names = "--infer", description = "Introspect given RediSearch index to infer Faker fields", paramLabel = "") + @Option(names = "--infer", description = "Introspect given RediSearch index to infer Faker fields", paramLabel = "") private Optional redisearchIndex = Optional.empty(); - @CommandLine.Option(names = "--locale", description = "Faker locale (default: ${DEFAULT-VALUE})", paramLabel = "") + @Option(names = "--locale", description = "Faker locale (default: ${DEFAULT-VALUE})", paramLabel = "") private Locale locale = Locale.ENGLISH; - @CommandLine.Option(names = "--metadata", description = "Include metadata (index, partition)") + @Option(names = "--metadata", description = "Include metadata (index, partition)") private boolean includeMetadata; public Map getFields() { @@ -49,4 +50,10 @@ public void setIncludeMetadata(boolean includeMetadata) { this.includeMetadata = includeMetadata; } + @Override + public String toString() { + return "FakerGeneratorOptions [fields=" + fields + ", redisearchIndex=" + redisearchIndex + ", locale=" + locale + + ", includeMetadata=" + includeMetadata + ", start=" + start + ", count=" + count + "]"; + } + } diff --git a/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerHelpCommand.java b/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerHelpCommand.java index efa9cacc6..558fcae49 100644 --- a/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerHelpCommand.java +++ b/connectors/riot-gen/src/main/java/com/redis/riot/gen/FakerHelpCommand.java @@ -18,11 +18,12 @@ import picocli.CommandLine.Command; import picocli.CommandLine.Help; import picocli.CommandLine.IHelpCommandInitializable2; +import picocli.CommandLine.Parameters; @Command(name = "faker-help", header = "Displays help information about Faker", synopsisHeading = "%nUsage: ", helpCommand = true) public class FakerHelpCommand implements IHelpCommandInitializable2, Runnable { - @CommandLine.Parameters(description = "Name of the Faker provider to show help for", paramLabel = "") + @Parameters(description = "Name of the Faker provider to show help for", paramLabel = "") private Optional provider = Optional.empty(); private static final List EXCLUDES = Arrays.asList("instance", "options"); diff --git a/connectors/riot-gen/src/main/java/com/redis/riot/gen/GeneratorOptions.java b/connectors/riot-gen/src/main/java/com/redis/riot/gen/GeneratorOptions.java index 42d18ab64..ae6deb381 100644 --- a/connectors/riot-gen/src/main/java/com/redis/riot/gen/GeneratorOptions.java +++ b/connectors/riot-gen/src/main/java/com/redis/riot/gen/GeneratorOptions.java @@ -2,15 +2,15 @@ import java.util.Optional; -import picocli.CommandLine; +import picocli.CommandLine.Option; public class GeneratorOptions { - @CommandLine.Option(names = "--start", description = "Start index (default: ${DEFAULT-VALUE})", paramLabel = "") + @Option(names = "--start", description = "Start index (default: ${DEFAULT-VALUE})", paramLabel = "") protected int start = 1; - @CommandLine.Option(names = "--count", description = "Number of items to generate (default: ${DEFAULT-VALUE})", paramLabel = "") + @Option(names = "--count", description = "Number of items to generate (default: ${DEFAULT-VALUE})", paramLabel = "") protected int count = 1000; - @CommandLine.Option(names = "--sleep", description = "Duration in ms to sleep before each item generation (default: ${DEFAULT-VALUE})", paramLabel = "") + @Option(names = "--sleep", description = "Duration in ms to sleep before each item generation (default: ${DEFAULT-VALUE})", paramLabel = "") private Optional sleep = Optional.empty(); public int getStart() { @@ -37,4 +37,9 @@ public void setSleep(long sleep) { this.sleep = Optional.of(sleep); } + @Override + public String toString() { + return "GeneratorOptions [start=" + start + ", count=" + count + ", sleep=" + sleep + "]"; + } + } diff --git a/connectors/riot-redis/src/main/java/com/redis/riot/redis/AbstractRedisCommandCommand.java b/connectors/riot-redis/src/main/java/com/redis/riot/redis/AbstractRedisCommandCommand.java index 3b4c0e4d5..aeaa4fa90 100644 --- a/connectors/riot-redis/src/main/java/com/redis/riot/redis/AbstractRedisCommandCommand.java +++ b/connectors/riot-redis/src/main/java/com/redis/riot/redis/AbstractRedisCommandCommand.java @@ -7,12 +7,12 @@ import com.redis.lettucemod.api.StatefulRedisModulesConnection; import com.redis.lettucemod.api.sync.RedisModulesCommands; -import com.redis.riot.AbstractRiotCommand; +import com.redis.riot.AbstractJobCommand; import picocli.CommandLine.Command; @Command -public abstract class AbstractRedisCommandCommand extends AbstractRiotCommand { +public abstract class AbstractRedisCommandCommand extends AbstractJobCommand { @Override protected Job job(JobBuilder jobBuilder) throws Exception { diff --git a/connectors/riot-redis/src/main/java/com/redis/riot/redis/AbstractTargetCommand.java b/connectors/riot-redis/src/main/java/com/redis/riot/redis/AbstractTargetCommand.java index 647a7324c..e53283246 100644 --- a/connectors/riot-redis/src/main/java/com/redis/riot/redis/AbstractTargetCommand.java +++ b/connectors/riot-redis/src/main/java/com/redis/riot/redis/AbstractTargetCommand.java @@ -1,7 +1,8 @@ package com.redis.riot.redis; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.logging.Level; +import java.util.logging.Logger; + import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.JobExecution; import org.springframework.batch.core.Step; @@ -27,21 +28,22 @@ import io.lettuce.core.codec.RedisCodec; import io.lettuce.core.codec.StringCodec; -import picocli.CommandLine; +import picocli.CommandLine.ArgGroup; +import picocli.CommandLine.Mixin; public abstract class AbstractTargetCommand extends AbstractTransferCommand { - private static final Logger log = LoggerFactory.getLogger(AbstractTargetCommand.class); + private static final Logger log = Logger.getLogger(AbstractTargetCommand.class.getName()); private static final String COMPARE_MESSAGE_ASCII = " >%,d T%,d ≠%,d ⧗%,d <%,d"; private static final String COMPARE_MESSAGE_COLOR = " \u001b[31m>%,d \u001b[33mT%,d \u001b[35m≠%,d \u001b[36m⧗%,d\u001b[0m"; private static final String VERIFICATION_NAME = "verification"; - @CommandLine.ArgGroup(exclusive = false, heading = "Target Redis connection options%n") + @ArgGroup(exclusive = false, heading = "Target Redis connection options%n") protected RedisOptions targetRedisOptions = new RedisOptions(); - @CommandLine.ArgGroup(exclusive = false, heading = "Reader options%n") + @ArgGroup(exclusive = false, heading = "Reader options%n") protected RedisReaderOptions readerOptions = new RedisReaderOptions(); - @CommandLine.Mixin + @Mixin private CompareOptions compareOptions = new CompareOptions(); public RedisOptions getTargetRedisOptions() { @@ -71,13 +73,14 @@ protected Step verificationStep() throws Exception { RedisItemReader> sourceReader = readerOptions .configureScanReader(reader(getRedisOptions(), StringCodec.UTF8).dataStructure()) .jobRunner(getJobRunner()).build(); - log.debug("Creating key comparator with TTL tolerance of {} seconds", compareOptions.getTtlTolerance()); + log.log(Level.FINE, "Creating key comparator with TTL tolerance of {0} seconds", + compareOptions.getTtlTolerance()); DataStructureValueReader targetValueReader = dataStructureValueReader(targetRedisOptions, StringCodec.UTF8); KeyComparisonItemWriter writer = KeyComparisonItemWriter.valueReader(targetValueReader) .tolerance(compareOptions.getTtlToleranceDuration()).build(); if (compareOptions.isShowDiffs()) { - writer.addListener(new KeyComparisonLogger(LoggerFactory.getLogger(getClass()))); + writer.addListener(new KeyComparisonLogger(java.util.logging.Logger.getLogger(getClass().getName()))); } Builder, DataStructure> riotStep = RiotStep.reader(sourceReader).writer(writer) .name(VERIFICATION_NAME).taskName("Verifying").max(this::initialMax) @@ -96,13 +99,14 @@ public ExitStatus afterStep(StepExecution stepExecution) { try { Thread.sleep(transferOptions.getProgressUpdateIntervalMillis()); } catch (InterruptedException e) { - log.debug("Verification interrupted"); + log.fine("Verification interrupted"); Thread.currentThread().interrupt(); return ExitStatus.STOPPED; } KeyComparisonResults results = writer.getResults(); - log.warn("Verification failed: OK={} Missing={} Values={} TTLs={} Types={}", results.getOK(), - results.getMissing(), results.getValue(), results.getTTL(), results.getType()); + log.log(Level.WARNING, "Verification failed: OK={0} Missing={1} Values={2} TTLs={3} Types={4}", + new Object[] { results.getOK(), results.getMissing(), results.getValue(), results.getTTL(), + results.getType() }); return new ExitStatus(ExitStatus.FAILED.getExitCode(), "Verification failed"); } }); @@ -116,7 +120,7 @@ protected Long initialMax() { try { return estimator.build().call(); } catch (Exception e) { - log.warn("Could not estimate scan size", e); + log.log(Level.WARNING, "Could not estimate scan size", e); return null; } } diff --git a/connectors/riot-redis/src/main/java/com/redis/riot/redis/CompareOptions.java b/connectors/riot-redis/src/main/java/com/redis/riot/redis/CompareOptions.java index 35f643e8b..8a96e042e 100644 --- a/connectors/riot-redis/src/main/java/com/redis/riot/redis/CompareOptions.java +++ b/connectors/riot-redis/src/main/java/com/redis/riot/redis/CompareOptions.java @@ -2,15 +2,15 @@ import java.time.Duration; -import picocli.CommandLine; +import picocli.CommandLine.Option; public class CompareOptions { public static final long DEFAULT_TTL_TOLERANCE_IN_SECONDS = 1; - @CommandLine.Option(names = "--ttl-tolerance", description = "Max TTL difference to use for dataset verification (default: ${DEFAULT-VALUE}).", paramLabel = "") + @Option(names = "--ttl-tolerance", description = "Max TTL difference to use for dataset verification (default: ${DEFAULT-VALUE}).", paramLabel = "") private long ttlTolerance = DEFAULT_TTL_TOLERANCE_IN_SECONDS; - @CommandLine.Option(names = "--show-diffs", description = "Print details of key mismatches during dataset verification") + @Option(names = "--show-diffs", description = "Print details of key mismatches during dataset verification") private boolean showDiffs; public Duration getTtlToleranceDuration() { diff --git a/connectors/riot-redis/src/main/java/com/redis/riot/redis/InfoCommand.java b/connectors/riot-redis/src/main/java/com/redis/riot/redis/InfoCommand.java index b914b4e7d..71aa796b7 100644 --- a/connectors/riot-redis/src/main/java/com/redis/riot/redis/InfoCommand.java +++ b/connectors/riot-redis/src/main/java/com/redis/riot/redis/InfoCommand.java @@ -1,7 +1,7 @@ package com.redis.riot.redis; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.logging.Level; +import java.util.logging.Logger; import com.redis.lettucemod.api.sync.RedisModulesCommands; @@ -10,11 +10,11 @@ @Command(name = "info", description = "Display INFO command output") public class InfoCommand extends AbstractRedisCommandCommand { - private static final Logger log = LoggerFactory.getLogger(InfoCommand.class); + private static final Logger log = Logger.getLogger(InfoCommand.class.getName()); @Override protected void execute(RedisModulesCommands commands) { - if (log.isInfoEnabled()) { + if (log.isLoggable(Level.INFO)) { log.info(commands.info()); } } diff --git a/connectors/riot-redis/src/main/java/com/redis/riot/redis/LatencyCommand.java b/connectors/riot-redis/src/main/java/com/redis/riot/redis/LatencyCommand.java index 9daf0637d..b0e5a5361 100644 --- a/connectors/riot-redis/src/main/java/com/redis/riot/redis/LatencyCommand.java +++ b/connectors/riot-redis/src/main/java/com/redis/riot/redis/LatencyCommand.java @@ -3,11 +3,11 @@ import java.util.Map; import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; import org.HdrHistogram.Histogram; import org.LatencyUtils.LatencyStats; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.redis.lettucemod.api.sync.RedisModulesCommands; @@ -19,7 +19,7 @@ @Command(name = "latency", description = "Calculate latency stats") public class LatencyCommand extends AbstractRedisCommandCommand { - private static final Logger log = LoggerFactory.getLogger(LatencyCommand.class); + private static final Logger log = Logger.getLogger(LatencyCommand.class.getName()); @Option(names = "--iterations", description = "Number of latency tests (default: ${DEFAULT-VALUE})", paramLabel = "") private int iterations = 1000; @@ -52,7 +52,9 @@ protected void execute(RedisModulesCommands commands) throws Int CommandMetrics.CommandLatency latency = new CommandMetrics.CommandLatency( unit.convert(histogram.getMinValue(), TimeUnit.NANOSECONDS), unit.convert(histogram.getMaxValue(), TimeUnit.NANOSECONDS), percentiles); - log.info(latency.toString()); + if (log.isLoggable(Level.INFO)) { + log.info(latency.toString()); + } } } diff --git a/connectors/riot-redis/src/main/java/com/redis/riot/redis/PingCommand.java b/connectors/riot-redis/src/main/java/com/redis/riot/redis/PingCommand.java index f4e8bd581..f6368fdab 100644 --- a/connectors/riot-redis/src/main/java/com/redis/riot/redis/PingCommand.java +++ b/connectors/riot-redis/src/main/java/com/redis/riot/redis/PingCommand.java @@ -1,7 +1,7 @@ package com.redis.riot.redis; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.logging.Level; +import java.util.logging.Logger; import com.redis.lettucemod.api.sync.RedisModulesCommands; @@ -10,11 +10,11 @@ @Command(name = "ping", description = "Execute PING command") public class PingCommand extends AbstractRedisCommandCommand { - private static final Logger log = LoggerFactory.getLogger(PingCommand.class); + private static final Logger log = Logger.getLogger(PingCommand.class.getName()); @Override protected void execute(RedisModulesCommands commands) { - log.info("Received ping reply: " + commands.ping()); + log.log(Level.INFO, "Received ping reply: {0}", commands.ping()); } @Override diff --git a/connectors/riot-redis/src/main/java/com/redis/riot/redis/ReplicateCommand.java b/connectors/riot-redis/src/main/java/com/redis/riot/redis/ReplicateCommand.java index b8a40b3c4..4d25046a4 100644 --- a/connectors/riot-redis/src/main/java/com/redis/riot/redis/ReplicateCommand.java +++ b/connectors/riot-redis/src/main/java/com/redis/riot/redis/ReplicateCommand.java @@ -3,9 +3,9 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.batch.core.Job; import org.springframework.batch.core.Step; import org.springframework.batch.core.job.builder.FlowBuilder; @@ -37,19 +37,20 @@ import io.lettuce.core.XAddArgs; import io.lettuce.core.codec.ByteArrayCodec; -import picocli.CommandLine; import picocli.CommandLine.ArgGroup; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; -@CommandLine.Command(name = "replicate", description = "Replicate a source Redis DB to a target Redis DB") +@Command(name = "replicate", description = "Replicate a source Redis DB to a target Redis DB") public class ReplicateCommand extends AbstractTargetCommand { - private static final Logger log = LoggerFactory.getLogger(ReplicateCommand.class); + private static final Logger log = Logger.getLogger(ReplicateCommand.class.getName()); - @CommandLine.Mixin + @Mixin private FlushingTransferOptions flushingTransferOptions = new FlushingTransferOptions(); - @CommandLine.Mixin + @Mixin private ReplicationOptions replicationOptions = new ReplicationOptions(); - @CommandLine.Mixin + @Mixin private KeyValueProcessorOptions processorOptions = new KeyValueProcessorOptions(); @ArgGroup(exclusive = false, heading = "Writer options%n") private RedisWriterOptions writerOptions = new RedisWriterOptions(); @@ -103,7 +104,7 @@ protected Optional optionalVerificationStep() throws Exception { } if (processorOptions.getKeyProcessor().isPresent()) { // Verification cannot be done if a processor is set - log.warn("Key processor enabled, verification will be skipped"); + log.warning("Key processor enabled, verification will be skipped"); return Optional.empty(); } return Optional.of(verificationStep()); @@ -151,10 +152,10 @@ private TaskletStep liveReplicationStep() throws Exception { @SuppressWarnings("unchecked") private > ItemWriter writer() { if (replicationOptions.isDryRun()) { - log.debug("Using no-op writer"); + log.fine("Using no-op writer"); return new NoOpItemWriter<>(); } - log.debug("Configuring writer with {}", targetRedisOptions); + log.log(Level.FINE, "Configuring writer with {0}", targetRedisOptions); OperationBuilder builder = writer(targetRedisOptions, ByteArrayCodec.INSTANCE); switch (replicationOptions.getType()) { case DS: diff --git a/connectors/riot-redis/src/main/java/com/redis/riot/redis/ReplicationOptions.java b/connectors/riot-redis/src/main/java/com/redis/riot/redis/ReplicationOptions.java index d346726d5..dc108710b 100644 --- a/connectors/riot-redis/src/main/java/com/redis/riot/redis/ReplicationOptions.java +++ b/connectors/riot-redis/src/main/java/com/redis/riot/redis/ReplicationOptions.java @@ -2,7 +2,7 @@ import com.redis.spring.batch.reader.AbstractKeyspaceNotificationItemReader; -import picocli.CommandLine; +import picocli.CommandLine.Option; public class ReplicationOptions { @@ -14,15 +14,15 @@ public enum ReplicationMode { SNAPSHOT, LIVE, LIVEONLY } - @CommandLine.Option(names = "--mode", description = "Replication mode: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") + @Option(names = "--mode", description = "Replication mode: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") private ReplicationMode mode = ReplicationMode.SNAPSHOT; - @CommandLine.Option(names = "--type", description = "Replication type: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") + @Option(names = "--type", description = "Replication type: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "") private ReplicationType type = ReplicationType.DUMP; - @CommandLine.Option(names = "--event-queue", description = "Capacity of the keyspace notification event queue (default: ${DEFAULT-VALUE})", paramLabel = "") + @Option(names = "--event-queue", description = "Capacity of the keyspace notification event queue (default: ${DEFAULT-VALUE})", paramLabel = "") private int notificationQueueCapacity = AbstractKeyspaceNotificationItemReader.DEFAULT_QUEUE_CAPACITY; - @CommandLine.Option(names = "--no-verify", description = "Verify target against source dataset after replication. True by default", negatable = true) + @Option(names = "--no-verify", description = "Verify target against source dataset after replication. True by default", negatable = true) private boolean verify = true; - @CommandLine.Option(names = "--dry-run", description = "Disable writes and only perform reads") + @Option(names = "--dry-run", description = "Disable writes and only perform reads") private boolean dryRun; public ReplicationMode getMode() { diff --git a/connectors/riot-stream/src/main/java/com/redis/riot/stream/StreamExportCommand.java b/connectors/riot-stream/src/main/java/com/redis/riot/stream/StreamExportCommand.java index b995f4083..6872abc22 100644 --- a/connectors/riot-stream/src/main/java/com/redis/riot/stream/StreamExportCommand.java +++ b/connectors/riot-stream/src/main/java/com/redis/riot/stream/StreamExportCommand.java @@ -28,8 +28,8 @@ import io.lettuce.core.StreamMessage; import io.lettuce.core.codec.StringCodec; -import picocli.CommandLine; import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @@ -39,11 +39,11 @@ public class StreamExportCommand extends AbstractTransferCommand { private static final Logger log = LoggerFactory.getLogger(StreamExportCommand.class); private static final String NAME = "stream-export"; - @CommandLine.Mixin + @Mixin private FlushingTransferOptions flushingTransferOptions = new FlushingTransferOptions(); @Parameters(arity = "0..*", description = "One ore more streams to read from", paramLabel = "STREAM") private List streams; - @CommandLine.Mixin + @Mixin private KafkaOptions options = new KafkaOptions(); @Option(names = "--offset", description = "XREAD offset (default: ${DEFAULT-VALUE})", paramLabel = "") private String offset = "0-0"; diff --git a/connectors/riot-stream/src/main/java/com/redis/riot/stream/StreamImportCommand.java b/connectors/riot-stream/src/main/java/com/redis/riot/stream/StreamImportCommand.java index 6bac33d99..3968c5814 100644 --- a/connectors/riot-stream/src/main/java/com/redis/riot/stream/StreamImportCommand.java +++ b/connectors/riot-stream/src/main/java/com/redis/riot/stream/StreamImportCommand.java @@ -30,9 +30,9 @@ import io.lettuce.core.XAddArgs; import io.lettuce.core.codec.StringCodec; -import picocli.CommandLine; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; import picocli.CommandLine.Parameters; @@ -42,11 +42,11 @@ public class StreamImportCommand extends AbstractTransferCommand { private static final Logger log = LoggerFactory.getLogger(StreamImportCommand.class); private static final String NAME = "stream-import"; - @CommandLine.Mixin + @Mixin private FlushingTransferOptions flushingTransferOptions = new FlushingTransferOptions(); @Parameters(arity = "0..*", description = "One ore more topics to read from", paramLabel = "TOPIC") private List topics; - @CommandLine.Mixin + @Mixin private KafkaOptions options = new KafkaOptions(); @Option(names = "--key", description = "Target stream key (default: same as topic)", paramLabel = "") private Optional key = Optional.empty(); @@ -54,7 +54,7 @@ public class StreamImportCommand extends AbstractTransferCommand { private Optional maxlen = Optional.empty(); @Option(names = "--trim", description = "Stream efficient trimming ('~' flag)") private boolean approximateTrimming; - @CommandLine.Mixin + @Mixin private FilteringOptions filteringOptions = new FilteringOptions(); @ArgGroup(exclusive = false, heading = "Writer options%n") private RedisWriterOptions writerOptions = new RedisWriterOptions(); diff --git a/core/riot-core/riot-core.gradle b/core/riot-core/riot-core.gradle index 08a154909..0a94b2dec 100644 --- a/core/riot-core/riot-core.gradle +++ b/core/riot-core/riot-core.gradle @@ -1,14 +1,12 @@ dependencies { - api 'org.slf4j:slf4j-api' api group: 'com.redis', name: 'lettucemod', version: lettucemodVersion - api 'org.apache.commons:commons-pool2' api group: 'com.redis', name: 'spring-batch-redis', version: batchRedisVersion - api 'org.springframework.batch:spring-batch-core' api group: 'info.picocli', name: 'picocli', version: picocliVersion annotationProcessor group: 'info.picocli', name: 'picocli-codegen', version: picocliVersion + api 'org.apache.commons:commons-pool2' + api 'org.springframework.batch:spring-batch-core' implementation group: 'me.tongfei', name: 'progressbar', version: progressbarVersion implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-xml' - implementation 'org.slf4j:slf4j-jdk14' implementation group: 'org.awaitility', name: 'awaitility', version: awaitilityVersion testImplementation 'org.junit.jupiter:junit-jupiter-engine' } \ No newline at end of file diff --git a/core/riot-core/src/main/java/com/redis/riot/AbstractExportCommand.java b/core/riot-core/src/main/java/com/redis/riot/AbstractExportCommand.java index 22e7050e3..222a9a10d 100644 --- a/core/riot-core/src/main/java/com/redis/riot/AbstractExportCommand.java +++ b/core/riot-core/src/main/java/com/redis/riot/AbstractExportCommand.java @@ -1,9 +1,9 @@ package com.redis.riot; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.batch.core.step.builder.AbstractTaskletStepBuilder; import org.springframework.batch.core.step.builder.SimpleStepBuilder; import org.springframework.batch.item.ItemProcessor; @@ -16,7 +16,7 @@ public abstract class AbstractExportCommand extends AbstractTransferCommand { - private static final Logger log = LoggerFactory.getLogger(AbstractExportCommand.class); + private static final Logger log = Logger.getLogger(AbstractExportCommand.class.getName()); @ArgGroup(exclusive = false, heading = "Reader options%n") private RedisReaderOptions options = new RedisReaderOptions(); @@ -31,7 +31,7 @@ private Long initialMax() { try { return estimator().build().call(); } catch (Exception e) { - log.warn("Could not estimate scan size", e); + log.log(Level.WARNING, "Could not estimate scan size", e); return null; } } diff --git a/core/riot-core/src/main/java/com/redis/riot/AbstractImportCommand.java b/core/riot-core/src/main/java/com/redis/riot/AbstractImportCommand.java index 097222a32..be2ae5281 100644 --- a/core/riot-core/src/main/java/com/redis/riot/AbstractImportCommand.java +++ b/core/riot-core/src/main/java/com/redis/riot/AbstractImportCommand.java @@ -29,7 +29,6 @@ import com.redis.spring.batch.writer.RedisOperation; import io.lettuce.core.codec.StringCodec; -import picocli.CommandLine; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; @@ -39,7 +38,7 @@ TsAddCommand.class }, subcommandsRepeatable = true, synopsisSubcommandLabel = "[REDIS COMMAND...]", commandListHeading = "Redis commands:%n") public abstract class AbstractImportCommand extends AbstractTransferCommand { - @CommandLine.ArgGroup(exclusive = false, heading = "Processor options%n") + @ArgGroup(exclusive = false, heading = "Processor options%n") private MapProcessorOptions processorOptions = new MapProcessorOptions(); @ArgGroup(exclusive = false, heading = "Writer options%n") diff --git a/core/riot-core/src/main/java/com/redis/riot/AbstractRiotCommand.java b/core/riot-core/src/main/java/com/redis/riot/AbstractJobCommand.java similarity index 89% rename from core/riot-core/src/main/java/com/redis/riot/AbstractRiotCommand.java rename to core/riot-core/src/main/java/com/redis/riot/AbstractJobCommand.java index 3a7d2d219..d64a5270a 100644 --- a/core/riot-core/src/main/java/com/redis/riot/AbstractRiotCommand.java +++ b/core/riot-core/src/main/java/com/redis/riot/AbstractJobCommand.java @@ -16,7 +16,7 @@ import picocli.CommandLine.Spec; @Command(abbreviateSynopsis = true, sortOptions = false) -public abstract class AbstractRiotCommand extends HelpCommand implements Callable { +public abstract class AbstractJobCommand extends HelpCommand implements Callable { @Spec private CommandSpec commandSpec; @@ -53,7 +53,8 @@ private int exitCode(JobExecution execution) { } public JobExecution execute() throws Exception { - return getJobRunner().run(job(configureJob(getJobRunner().job(commandName())))); + Job job = job(configureJob(getJobRunner().job(commandName()))); + return getJobRunner().run(job); } protected abstract Job job(JobBuilder jobBuilder) throws Exception; diff --git a/core/riot-core/src/main/java/com/redis/riot/AbstractTransferCommand.java b/core/riot-core/src/main/java/com/redis/riot/AbstractTransferCommand.java index c57fad8f7..4d37c725a 100644 --- a/core/riot-core/src/main/java/com/redis/riot/AbstractTransferCommand.java +++ b/core/riot-core/src/main/java/com/redis/riot/AbstractTransferCommand.java @@ -2,9 +2,9 @@ import java.time.Duration; import java.util.concurrent.ThreadPoolExecutor; +import java.util.logging.Level; +import java.util.logging.Logger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.batch.core.ItemWriteListener; import org.springframework.batch.core.StepExecutionListener; import org.springframework.batch.core.step.builder.FaultTolerantStepBuilder; @@ -27,13 +27,13 @@ import com.redis.spring.batch.reader.PollableItemReader; import io.lettuce.core.codec.RedisCodec; -import picocli.CommandLine; +import picocli.CommandLine.Mixin; -public abstract class AbstractTransferCommand extends AbstractRiotCommand { +public abstract class AbstractTransferCommand extends AbstractJobCommand { - private static final Logger log = LoggerFactory.getLogger(AbstractTransferCommand.class); + private static final Logger log = Logger.getLogger(AbstractTransferCommand.class.getName()); - @CommandLine.Mixin + @Mixin protected TransferOptions transferOptions = new TransferOptions(); protected TypeBuilder stringReader(RedisOptions redisOptions) { @@ -78,7 +78,7 @@ public FaultTolerantStepBuilder step(RiotStep riotStep) throw taskExecutor.setQueueCapacity(transferOptions.getThreads()); taskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); taskExecutor.afterPropertiesSet(); - log.debug("Created pooled task executor of size {}", taskExecutor.getCorePoolSize()); + log.log(Level.FINE, "Created pooled task executor of size {0}", taskExecutor.getCorePoolSize()); ftStep.taskExecutor(taskExecutor).throttleLimit(transferOptions.getThreads()); } else { ftStep.taskExecutor(new SyncTaskExecutor()); diff --git a/core/riot-core/src/main/java/com/redis/riot/FlushingTransferOptions.java b/core/riot-core/src/main/java/com/redis/riot/FlushingTransferOptions.java index c8771f127..91148637c 100644 --- a/core/riot-core/src/main/java/com/redis/riot/FlushingTransferOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/FlushingTransferOptions.java @@ -10,13 +10,13 @@ import com.redis.spring.batch.reader.LiveRedisItemReader; import com.redis.spring.batch.step.FlushingSimpleStepBuilder; -import picocli.CommandLine; +import picocli.CommandLine.Option; public class FlushingTransferOptions { - @CommandLine.Option(names = "--flush-interval", description = "Max duration between flushes (default: ${DEFAULT-VALUE})", paramLabel = "") + @Option(names = "--flush-interval", description = "Max duration between flushes (default: ${DEFAULT-VALUE})", paramLabel = "") private long flushInterval = 50; - @CommandLine.Option(names = "--idle-timeout", description = "Min duration of inactivity to consider transfer complete", paramLabel = "") + @Option(names = "--idle-timeout", description = "Min duration of inactivity to consider transfer complete", paramLabel = "") private Optional idleTimeout = Optional.empty(); public void setFlushInterval(Duration flushInterval) { @@ -47,4 +47,9 @@ public FlushingSimpleStepBuilder configure(FaultTolerantStepBuilder return reader; } + @Override + public String toString() { + return "FlushingTransferOptions [flushInterval=" + flushInterval + ", idleTimeout=" + idleTimeout + "]"; + } + } diff --git a/core/riot-core/src/main/java/com/redis/riot/GenerateCompletionCommand.java b/core/riot-core/src/main/java/com/redis/riot/GenerateCompletionCommand.java index 4bc38cf99..295367c2f 100644 --- a/core/riot-core/src/main/java/com/redis/riot/GenerateCompletionCommand.java +++ b/core/riot-core/src/main/java/com/redis/riot/GenerateCompletionCommand.java @@ -1,8 +1,8 @@ package com.redis.riot; import picocli.AutoComplete; -import picocli.CommandLine; +import picocli.CommandLine.Command; -@CommandLine.Command(hidden = true, name = "generate-completion", usageHelpAutoWidth = true) +@Command(hidden = true, name = "generate-completion", usageHelpAutoWidth = true) public class GenerateCompletionCommand extends AutoComplete.GenerateCompletion { } \ No newline at end of file diff --git a/core/riot-core/src/main/java/com/redis/riot/HelpCommand.java b/core/riot-core/src/main/java/com/redis/riot/HelpCommand.java index 3ec310a97..9dac14960 100644 --- a/core/riot-core/src/main/java/com/redis/riot/HelpCommand.java +++ b/core/riot-core/src/main/java/com/redis/riot/HelpCommand.java @@ -1,11 +1,12 @@ package com.redis.riot; -import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; -@CommandLine.Command(usageHelpAutoWidth = true) +@Command(usageHelpAutoWidth = true) public class HelpCommand { - @CommandLine.Option(names = { "-H", "--help" }, usageHelp = true, description = "Show this help message and exit") + @Option(names = { "-H", "--help" }, usageHelp = true, description = "Show this help message and exit") private boolean helpRequested; } diff --git a/core/riot-core/src/main/java/com/redis/riot/KeyValueProcessorOptions.java b/core/riot-core/src/main/java/com/redis/riot/KeyValueProcessorOptions.java index 5c0c0ed34..d1d5ac9e4 100644 --- a/core/riot-core/src/main/java/com/redis/riot/KeyValueProcessorOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/KeyValueProcessorOptions.java @@ -19,4 +19,9 @@ public Optional getTtlProcessor() { return ttlProcessor; } + @Override + public String toString() { + return "KeyValueProcessorOptions [keyProcessor=" + keyProcessor + ", ttlProcessor=" + ttlProcessor + "]"; + } + } diff --git a/core/riot-core/src/main/java/com/redis/riot/LoggingOptions.java b/core/riot-core/src/main/java/com/redis/riot/LoggingOptions.java index 1e4684739..9c8bf645e 100644 --- a/core/riot-core/src/main/java/com/redis/riot/LoggingOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/LoggingOptions.java @@ -1,9 +1,31 @@ package com.redis.riot; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.TimeZone; +import java.util.logging.ConsoleHandler; +import java.util.logging.Formatter; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import org.springframework.core.NestedExceptionUtils; + +import io.netty.util.internal.logging.InternalLoggerFactory; +import io.netty.util.internal.logging.JdkLoggerFactory; import picocli.CommandLine.Option; public class LoggingOptions { + private static final String ROOT_LOGGER = ""; + @Option(names = { "-q", "--quiet" }, description = "Log errors only.") private boolean quiet; @Option(names = { "-w", "--warn" }, description = "Set log level to warn.") @@ -15,40 +37,92 @@ public class LoggingOptions { @Option(names = "--stacktrace", description = "Print out the stacktrace for all exceptions.") private boolean stacktrace; - public boolean isQuiet() { - return quiet; + public Level getLogLevel() { + if (debug) { + return Level.FINE; + } + if (info) { + return Level.INFO; + } + if (warning) { + return Level.WARNING; + } + if (quiet) { + return Level.OFF; + } + return Level.SEVERE; } - public void setQuiet(boolean quiet) { - this.quiet = quiet; + public void configure() { + InternalLoggerFactory.setDefaultFactory(JdkLoggerFactory.INSTANCE); + LogManager.getLogManager().reset(); + Logger activeLogger = Logger.getLogger(ROOT_LOGGER); + ConsoleHandler handler = new ConsoleHandler(); + handler.setLevel(Level.ALL); + handler.setFormatter(stacktrace || debug ? new StackTraceOneLineLogFormat() : new OneLineLogFormat()); + activeLogger.addHandler(handler); + Logger.getLogger(ROOT_LOGGER).setLevel(getLogLevel()); } - public boolean isWarning() { - return warning; + static class OneLineLogFormat extends Formatter { + + @Override + public String format(LogRecord logRecord) { + String message = formatMessage(logRecord); + if (logRecord.getThrown() != null) { + Throwable rootCause = NestedExceptionUtils.getRootCause(logRecord.getThrown()); + if (rootCause != null && rootCause.getMessage() != null) { + return String.format("%s: %s%n", message, rootCause.getMessage()); + } + } + return String.format("%s%n", message); + } + } - public void setWarning(boolean warning) { - this.warning = warning; + static class StackTraceOneLineLogFormat extends Formatter { + + private final DateTimeFormatter d = new DateTimeFormatterBuilder().appendValue(ChronoField.HOUR_OF_DAY, 2) + .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2).optionalStart().appendLiteral(':') + .appendValue(ChronoField.SECOND_OF_MINUTE, 2).optionalStart() + .appendFraction(ChronoField.NANO_OF_SECOND, 3, 3, true).toFormatter(); + + private final ZoneId offset = TimeZone.getDefault().toZoneId(); + + @Override + public String format(LogRecord logRecord) { + String message = formatMessage(logRecord); + ZonedDateTime time = Instant.ofEpochMilli(logRecord.getMillis()).atZone(offset); + if (logRecord.getThrown() == null) { + return String.format("%s %s %s\t: %s%n", time.format(d), logRecord.getLevel().getLocalizedName(), + logRecord.getLoggerName(), message); + } + return String.format("%s %s %s\t: %s%n%s%n", time.format(d), logRecord.getLevel().getLocalizedName(), + logRecord.getLoggerName(), message, stackTrace(logRecord)); + } + + private String stackTrace(LogRecord logRecord) { + StringWriter sw = new StringWriter(4096); + PrintWriter pw = new PrintWriter(sw); + logRecord.getThrown().printStackTrace(pw); + return sw.toString(); + } } - public boolean isInfo() { - return info; + public void setDebug(boolean debug) { + this.debug = debug; } public void setInfo(boolean info) { this.info = info; } - public boolean isDebug() { - return debug; - } - - public void setDebug(boolean debug) { - this.debug = debug; + public void setQuiet(boolean quiet) { + this.quiet = quiet; } - public boolean isStacktrace() { - return stacktrace || debug; + public void setWarning(boolean warning) { + this.warning = warning; } public void setStacktrace(boolean stacktrace) { diff --git a/core/riot-core/src/main/java/com/redis/riot/MapProcessorOptions.java b/core/riot-core/src/main/java/com/redis/riot/MapProcessorOptions.java index 39aed6ed0..a17104989 100644 --- a/core/riot-core/src/main/java/com/redis/riot/MapProcessorOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/MapProcessorOptions.java @@ -3,15 +3,16 @@ import java.lang.reflect.Method; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.batch.item.ItemProcessor; import org.springframework.core.convert.converter.Converter; import org.springframework.expression.EvaluationContext; @@ -31,7 +32,7 @@ public class MapProcessorOptions { - private static final Logger log = LoggerFactory.getLogger(MapProcessorOptions.class); + private static final Logger log = Logger.getLogger(MapProcessorOptions.class.getName()); @Option(arity = "1..*", names = "--process", description = "SpEL expressions in the form field1=\"exp\" field2=\"exp\"...", paramLabel = "") private Map spelFields; @@ -114,10 +115,16 @@ private EvaluationContext context() { Method geoMethod = GeoLocation.class.getDeclaredMethod("toString", String.class, String.class); context.registerFunction("geo", geoMethod); } catch (Exception e) { - log.warn("Could not register geo function", e); + log.log(Level.WARNING, "Could not register geo function", e); } context.setPropertyAccessors(Collections.singletonList(new MapAccessor())); return context; } + @Override + public String toString() { + return "MapProcessorOptions [spelFields=" + spelFields + ", variables=" + variables + ", dateFormat=" + + dateFormat + ", filters=" + Arrays.toString(filters) + ", regexes=" + regexes + "]"; + } + } diff --git a/core/riot-core/src/main/java/com/redis/riot/OneLineLogFormat.java b/core/riot-core/src/main/java/com/redis/riot/OneLineLogFormat.java deleted file mode 100644 index 3d6dc4d5b..000000000 --- a/core/riot-core/src/main/java/com/redis/riot/OneLineLogFormat.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.redis.riot; - -import java.util.logging.Formatter; -import java.util.logging.LogRecord; - -import org.springframework.core.NestedExceptionUtils; - -public class OneLineLogFormat extends Formatter { - - @Override - public String format(LogRecord logRecord) { - String message = formatMessage(logRecord); - if (logRecord.getThrown() != null) { - Throwable rootCause = NestedExceptionUtils.getRootCause(logRecord.getThrown()); - if (rootCause != null && rootCause.getMessage() != null) { - return String.format("%s: %s%n", message, rootCause.getMessage()); - } - } - return String.format("%s%n", message); - } - -} \ No newline at end of file diff --git a/core/riot-core/src/main/java/com/redis/riot/ProgressMonitor.java b/core/riot-core/src/main/java/com/redis/riot/ProgressMonitor.java index e0f9768ec..bc5911ab4 100644 --- a/core/riot-core/src/main/java/com/redis/riot/ProgressMonitor.java +++ b/core/riot-core/src/main/java/com/redis/riot/ProgressMonitor.java @@ -5,8 +5,7 @@ import java.util.Optional; import java.util.function.Supplier; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.springframework.batch.core.BatchStatus; import org.springframework.batch.core.ExitStatus; import org.springframework.batch.core.ItemWriteListener; import org.springframework.batch.core.StepExecution; @@ -23,8 +22,6 @@ @SuppressWarnings("rawtypes") public class ProgressMonitor implements StepExecutionListener, ItemWriteListener { - private static final Logger log = LoggerFactory.getLogger(ProgressMonitor.class); - private final TransferOptions.Progress style; private final String taskName; private final Duration updateInterval; @@ -51,17 +48,13 @@ public void beforeStep(StepExecution stepExecution) { initialMax.ifPresent(m -> { Long initialMaxValue = m.get(); if (initialMaxValue != null) { - log.debug("Setting initial max to {}", initialMaxValue); builder.setInitialMax(initialMaxValue); } }); - log.debug("Setting update interval to {}", updateInterval); builder.setUpdateIntervalMillis(Math.toIntExact(updateInterval.toMillis())); - log.debug("Setting task name to {}", taskName); builder.setTaskName(taskName); builder.showSpeed(); - log.debug("Opening progress bar"); - this.progressBar = builder.build(); + progressBar = builder.build(); } private ProgressBarStyle progressBarStyle() { @@ -76,9 +69,10 @@ private ProgressBarStyle progressBarStyle() { } @Override - public ExitStatus afterStep(StepExecution stepExecution) { - log.debug("Closing progress bar"); - progressBar.close(); + public synchronized ExitStatus afterStep(StepExecution stepExecution) { + if (stepExecution.getStatus() != BatchStatus.FAILED) { + progressBar.close(); + } return null; } diff --git a/core/riot-core/src/main/java/com/redis/riot/RedisOptions.java b/core/riot-core/src/main/java/com/redis/riot/RedisOptions.java index 7eeb61d14..dc1e7affd 100644 --- a/core/riot-core/src/main/java/com/redis/riot/RedisOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/RedisOptions.java @@ -3,9 +3,8 @@ import java.io.File; import java.time.Duration; import java.util.Optional; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.util.logging.Level; +import java.util.logging.Logger; import com.redis.lettucemod.RedisModulesClient; import com.redis.lettucemod.api.StatefulRedisModulesConnection; @@ -26,7 +25,7 @@ public class RedisOptions { - private static final Logger log = LoggerFactory.getLogger(RedisOptions.class); + private static final Logger log = Logger.getLogger(RedisOptions.class.getName()); public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 6379; @@ -241,7 +240,7 @@ public AbstractRedisClient client() { public RedisModulesClusterClient redisModulesClusterClient() { if (client == null) { - log.debug("Creating Redis cluster client: {}", this); + log.log(Level.FINE, "Creating Redis cluster client: {0}", this); RedisModulesClusterClient clusterClient = RedisModulesClusterClient.create(clientResources(), uri()); clusterClient.setOptions( ClusterClientOptions.builder().autoReconnect(autoReconnect).sslOptions(sslOptions()).build()); @@ -252,7 +251,7 @@ public RedisModulesClusterClient redisModulesClusterClient() { public RedisModulesClient redisModulesClient() { if (client == null) { - log.debug("Creating Redis client: {}", this); + log.log(Level.FINE, "Creating Redis client: {0}", this); RedisModulesClient redisClient = RedisModulesClient.create(clientResources(), uri()); redisClient .setOptions(ClientOptions.builder().autoReconnect(autoReconnect).sslOptions(sslOptions()).build()); @@ -268,7 +267,7 @@ public String toString() { + ", cluster=" + cluster + ", tls=" + tls + ", insecure=" + insecure + ", keystore=" + keystore + ", keystorePassword=" + keystorePassword + ", truststore=" + truststore + ", truststorePassword=" + truststorePassword + ", cert=" + cert + ", showMetrics=" + showMetrics + ", autoReconnect=" - + autoReconnect + ", clientName=" + clientName + "]"; + + autoReconnect + ", clientName=" + clientName + ", client=" + client + "]"; } } diff --git a/core/riot-core/src/main/java/com/redis/riot/RedisReaderOptions.java b/core/riot-core/src/main/java/com/redis/riot/RedisReaderOptions.java index c07ce01a8..66190022e 100644 --- a/core/riot-core/src/main/java/com/redis/riot/RedisReaderOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/RedisReaderOptions.java @@ -1,10 +1,10 @@ package com.redis.riot; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import com.redis.spring.batch.KeyValue; import com.redis.spring.batch.RedisItemReader; @@ -15,7 +15,7 @@ public class RedisReaderOptions { - private static final Logger log = LoggerFactory.getLogger(RedisReaderOptions.class); + private static final Logger log = Logger.getLogger(RedisReaderOptions.class.getName()); @Option(names = "--scan-count", description = "SCAN COUNT option (default: ${DEFAULT-VALUE}).", paramLabel = "") private long scanCount = ScanKeyItemReader.DEFAULT_SCAN_COUNT; @@ -100,7 +100,8 @@ public void setPoolMaxTotal(int poolMaxTotal) { public > RedisItemReader.Builder configureScanReader( RedisItemReader.Builder builder) { - log.debug("Configuring scan reader with {} {} {}", scanCount, scanMatch, scanType); + log.log(Level.FINE, "Configuring scan reader with {0} {1} {2}", + new Object[] { scanCount, scanMatch, scanType }); builder.match(scanMatch); builder.count(scanCount); scanType.ifPresent(builder::type); @@ -108,8 +109,8 @@ public void setPoolMaxTotal(int poolMaxTotal) { } public > B configureReader(B builder) { - log.debug("Configuring reader with threads: {}, batch-size: {}, queue-capacity: {}", threads, batchSize, - queueCapacity); + log.log(Level.FINE, "Configuring reader with threads: {0}, batch-size: {1}, queue-capacity: {2}", + new Object[] { threads, batchSize, queueCapacity }); builder.threads(threads); builder.chunkSize(batchSize); builder.valueQueueCapacity(queueCapacity); @@ -120,8 +121,15 @@ public void setPoolMaxTotal(int poolMaxTotal) { public GenericObjectPoolConfig> poolConfig() { GenericObjectPoolConfig> config = new GenericObjectPoolConfig<>(); config.setMaxTotal(poolMaxTotal); - log.debug("Configuring reader with pool config {}", config); + log.log(Level.FINE, "Configuring reader with pool config {0}", config); return config; } + @Override + public String toString() { + return "RedisReaderOptions [scanCount=" + scanCount + ", scanMatch=" + scanMatch + ", scanType=" + scanType + + ", queueCapacity=" + queueCapacity + ", threads=" + threads + ", batchSize=" + batchSize + + ", sampleSize=" + sampleSize + ", poolMaxTotal=" + poolMaxTotal + "]"; + } + } diff --git a/core/riot-core/src/main/java/com/redis/riot/RedisWriterOptions.java b/core/riot-core/src/main/java/com/redis/riot/RedisWriterOptions.java index 8be28d09f..544d1cbd8 100644 --- a/core/riot-core/src/main/java/com/redis/riot/RedisWriterOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/RedisWriterOptions.java @@ -66,4 +66,11 @@ public void setPoolMaxTotal(int poolMaxTotal) { writer.poolConfig(poolConfig); return writer; } + + @Override + public String toString() { + return "RedisWriterOptions [multiExec=" + multiExec + ", waitReplicas=" + waitReplicas + ", waitTimeout=" + + waitTimeout + ", poolMaxTotal=" + poolMaxTotal + "]"; + } + } diff --git a/core/riot-core/src/main/java/com/redis/riot/RiotApp.java b/core/riot-core/src/main/java/com/redis/riot/RiotApp.java index 26d8fec58..64957b6a5 100644 --- a/core/riot-core/src/main/java/com/redis/riot/RiotApp.java +++ b/core/riot-core/src/main/java/com/redis/riot/RiotApp.java @@ -1,11 +1,8 @@ package com.redis.riot; -import java.util.logging.ConsoleHandler; import java.util.logging.Level; -import java.util.logging.LogManager; import java.util.logging.Logger; -import org.slf4j.LoggerFactory; import org.springframework.expression.Expression; import org.springframework.expression.spel.standard.SpelExpressionParser; @@ -13,13 +10,10 @@ import com.redis.spring.batch.support.IntRange; import io.lettuce.core.RedisURI; -import io.netty.util.internal.logging.InternalLoggerFactory; -import io.netty.util.internal.logging.JdkLoggerFactory; import picocli.CommandLine; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; import picocli.CommandLine.IExecutionStrategy; -import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; import picocli.CommandLine.ParseResult; import picocli.CommandLine.RunFirst; @@ -28,15 +22,13 @@ @Command(sortOptions = false, versionProvider = ManifestVersionProvider.class, subcommands = GenerateCompletionCommand.class, abbreviateSynopsis = true) public class RiotApp extends HelpCommand { - private static final org.slf4j.Logger log = LoggerFactory.getLogger(RiotApp.class); - - private static final String ROOT_LOGGER = ""; - @Option(names = { "-V", "--version" }, versionHelp = true, description = "Print version information and exit.") private boolean versionRequested; + @ArgGroup(heading = "Redis connection options%n", exclusive = false) private RedisOptions redisOptions = new RedisOptions(); - @Mixin + + @ArgGroup(heading = "Logging options%n", exclusive = false) private LoggingOptions loggingOptions = new LoggingOptions(); public RedisOptions getRedisOptions() { @@ -47,83 +39,16 @@ public LoggingOptions getLoggingOptions() { return loggingOptions; } - private int executionStrategy(ParseResult parseResult) { - return execute(new RunLast(), parseResult); // default execution strategy - } - - private int executionStragegyRunFirst(ParseResult parseResult) { - return execute(new RunFirst(), parseResult); - } - - private int execute(IExecutionStrategy strategy, ParseResult parseResult) { + protected int execute(IExecutionStrategy strategy, ParseResult parseResult) { configureLogging(); - log.debug("Running {} {} with {}", parseResult.commandSpec().name(), ManifestVersionProvider.getVersionString(), - parseResult.originalArgs()); + Logger log = Logger.getLogger(getClass().getName()); + log.log(Level.FINE, "Running {0} {1} with {2}", new Object[] { parseResult.commandSpec().name(), + ManifestVersionProvider.getVersionString(), parseResult.originalArgs() }); return strategy.execute(parseResult); } protected void configureLogging() { - InternalLoggerFactory.setDefaultFactory(JdkLoggerFactory.INSTANCE); - LogManager.getLogManager().reset(); - Logger activeLogger = Logger.getLogger(ROOT_LOGGER); - ConsoleHandler handler = new ConsoleHandler(); - handler.setLevel(Level.ALL); - handler.setFormatter(loggingOptions.isStacktrace() ? new StackTraceOneLineLogFormat() : new OneLineLogFormat()); - activeLogger.addHandler(handler); - Logger.getLogger(ROOT_LOGGER).setLevel(getLogLevel()); - Logger.getLogger("com.redis.riot").setLevel(getRiotLogLevel()); - Logger.getLogger("com.redis.spring.batch").setLevel(getRiotLogLevel()); - Logger.getLogger("org.springframework.batch.core.step.item.ChunkMonitor").setLevel(getSpringLevel()); - Logger.getLogger("org.springframework.batch.core.step.builder.FaultTolerantStepBuilder") - .setLevel(getSpringLevel()); - } - - private Level getSpringLevel() { - if (loggingOptions.isDebug()) { - return Level.INFO; - } - if (loggingOptions.isInfo()) { - return Level.WARNING; - } - if (loggingOptions.isWarning()) { - return Level.SEVERE; - } - if (loggingOptions.isQuiet()) { - return Level.OFF; - } - return Level.SEVERE; - } - - private Level getLogLevel() { - if (loggingOptions.isDebug()) { - return Level.FINE; - } - if (loggingOptions.isInfo()) { - return Level.INFO; - } - if (loggingOptions.isWarning()) { - return Level.SEVERE; - } - if (loggingOptions.isQuiet()) { - return Level.OFF; - } - return Level.WARNING; - } - - private Level getRiotLogLevel() { - if (loggingOptions.isDebug()) { - return Level.FINEST; - } - if (loggingOptions.isInfo()) { - return Level.FINE; - } - if (loggingOptions.isWarning()) { - return Level.WARNING; - } - if (loggingOptions.isQuiet()) { - return Level.SEVERE; - } - return Level.INFO; + loggingOptions.configure(); } public int execute(String... args) { @@ -131,8 +56,8 @@ public int execute(String... args) { } public RiotCommandLine commandLine() { - RiotCommandLine commandLine = new RiotCommandLine(this, this::executionStragegyRunFirst); - commandLine.setExecutionStrategy(this::executionStrategy); + RiotCommandLine commandLine = new RiotCommandLine(this, r -> execute(new RunFirst(), r)); + commandLine.setExecutionStrategy(r -> execute(new RunLast(), r)); commandLine.setExecutionExceptionHandler(this::handleExecutionException); registerConverters(commandLine); commandLine.setCaseInsensitiveEnumValuesAllowed(true); diff --git a/core/riot-core/src/main/java/com/redis/riot/StackTraceOneLineLogFormat.java b/core/riot-core/src/main/java/com/redis/riot/StackTraceOneLineLogFormat.java deleted file mode 100644 index 6c12a55e6..000000000 --- a/core/riot-core/src/main/java/com/redis/riot/StackTraceOneLineLogFormat.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.redis.riot; - -import java.io.PrintWriter; -import java.io.StringWriter; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.time.temporal.ChronoField; -import java.util.TimeZone; -import java.util.logging.Formatter; -import java.util.logging.LogRecord; - -public class StackTraceOneLineLogFormat extends Formatter { - - private final DateTimeFormatter d = new DateTimeFormatterBuilder().appendValue(ChronoField.HOUR_OF_DAY, 2) - .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2).optionalStart().appendLiteral(':') - .appendValue(ChronoField.SECOND_OF_MINUTE, 2).optionalStart() - .appendFraction(ChronoField.NANO_OF_SECOND, 3, 3, true).toFormatter(); - - private final ZoneId offset = TimeZone.getDefault().toZoneId(); - - @Override - public String format(LogRecord logRecord) { - String message = formatMessage(logRecord); - ZonedDateTime time = Instant.ofEpochMilli(logRecord.getMillis()).atZone(offset); - if (logRecord.getThrown() == null) { - return String.format("%s %s %s\t: %s%n", time.format(d), logRecord.getLevel().getLocalizedName(), - logRecord.getLoggerName(), message); - } - return String.format("%s %s %s\t: %s%n%s%n", time.format(d), logRecord.getLevel().getLocalizedName(), - logRecord.getLoggerName(), message, stackTrace(logRecord)); - } - - private String stackTrace(LogRecord logRecord) { - StringWriter sw = new StringWriter(4096); - PrintWriter pw = new PrintWriter(sw); - logRecord.getThrown().printStackTrace(pw); - return sw.toString(); - } -} diff --git a/core/riot-core/src/main/java/com/redis/riot/TransferOptions.java b/core/riot-core/src/main/java/com/redis/riot/TransferOptions.java index 31c25c020..5536fb5d3 100644 --- a/core/riot-core/src/main/java/com/redis/riot/TransferOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/TransferOptions.java @@ -1,6 +1,6 @@ package com.redis.riot; -import picocli.CommandLine; +import picocli.CommandLine.Option; public class TransferOptions { @@ -12,18 +12,18 @@ public enum SkipPolicy { ALWAYS, NEVER, LIMIT } - @CommandLine.Option(names = "--progress", description = "Style of progress bar: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE})", paramLabel = "