diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileDumpExport.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileDumpExport.java index e1604979e..0d5f949f9 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/FileDumpExport.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileDumpExport.java @@ -6,19 +6,19 @@ import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.ItemWriter; import org.springframework.batch.item.json.JacksonJsonObjectMarshaller; +import org.springframework.batch.item.json.JsonFileItemWriter; import org.springframework.batch.item.json.JsonObjectMarshaller; +import org.springframework.batch.item.json.builder.JsonFileItemWriterBuilder; import org.springframework.core.io.Resource; import org.springframework.core.io.WritableResource; import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.redis.riot.core.AbstractExport; import com.redis.riot.core.ExecutionException; -import com.redis.riot.file.resource.JsonResourceItemWriter; -import com.redis.riot.file.resource.JsonResourceItemWriterBuilder; -import com.redis.riot.file.resource.XmlResourceItemWriter; -import com.redis.riot.file.resource.XmlResourceItemWriterBuilder; -import com.redis.spring.batch.KeyValue; +import com.redis.riot.file.xml.XmlResourceItemWriter; +import com.redis.riot.file.xml.XmlResourceItemWriterBuilder; import com.redis.spring.batch.RedisItemReader; +import com.redis.spring.batch.reader.MemKeyValue; import io.lettuce.core.codec.StringCodec; @@ -36,35 +36,7 @@ public class FileDumpExport extends AbstractExport { private String lineSeparator = DEFAULT_LINE_SEPARATOR; private FileDumpType type; - public void setFile(String file) { - this.file = file; - } - - public void setFileOptions(FileOptions fileOptions) { - this.fileOptions = fileOptions; - } - - public void setType(FileDumpType type) { - this.type = type; - } - - public void setAppend(boolean append) { - this.append = append; - } - - public void setRootName(String rootName) { - this.rootName = rootName; - } - - public void setElementName(String elementName) { - this.elementName = elementName; - } - - public void setLineSeparator(String lineSeparator) { - this.lineSeparator = lineSeparator; - } - - private ItemWriter> writer() { + private ItemWriter> writer() { WritableResource resource; try { resource = FileUtils.outputResource(file, fileOptions); @@ -84,8 +56,8 @@ private FileDumpType dumpType(WritableResource resource) { return type; } - private JsonResourceItemWriter> jsonWriter(Resource resource) { - JsonResourceItemWriterBuilder> jsonWriterBuilder = new JsonResourceItemWriterBuilder<>(); + private JsonFileItemWriter> jsonWriter(WritableResource resource) { + JsonFileItemWriterBuilder> jsonWriterBuilder = new JsonFileItemWriterBuilder<>(); jsonWriterBuilder.name("json-resource-item-writer"); jsonWriterBuilder.append(append); jsonWriterBuilder.encoding(fileOptions.getEncoding()); @@ -96,8 +68,8 @@ private JsonResourceItemWriter> jsonWriter(Resource res return jsonWriterBuilder.build(); } - private XmlResourceItemWriter> xmlWriter(Resource resource) { - XmlResourceItemWriterBuilder> xmlWriterBuilder = new XmlResourceItemWriterBuilder<>(); + private XmlResourceItemWriter> xmlWriter(Resource resource) { + XmlResourceItemWriterBuilder> xmlWriterBuilder = new XmlResourceItemWriterBuilder<>(); xmlWriterBuilder.name("xml-resource-item-writer"); xmlWriterBuilder.append(append); xmlWriterBuilder.encoding(fileOptions.getEncoding()); @@ -109,20 +81,49 @@ private XmlResourceItemWriter> xmlWriter(Resource resou return xmlWriterBuilder.build(); } - private JsonObjectMarshaller> xmlMarshaller() { + private JsonObjectMarshaller> xmlMarshaller() { XmlMapper mapper = FileUtils.xmlMapper(); mapper.setConfig(mapper.getSerializationConfig().withRootName(elementName)); - JacksonJsonObjectMarshaller> marshaller = new JacksonJsonObjectMarshaller<>(); + JacksonJsonObjectMarshaller> marshaller = new JacksonJsonObjectMarshaller<>(); marshaller.setObjectMapper(mapper); return marshaller; } + @SuppressWarnings({ "rawtypes", "unchecked" }) @Override protected Job job() { - RedisItemReader> reader = RedisItemReader.struct(); + RedisItemReader> reader = RedisItemReader.struct(); configure(reader); - ItemProcessor, KeyValue> processor = processor(StringCodec.UTF8); + ItemProcessor processor = processor(StringCodec.UTF8); return jobBuilder().start(step(getName(), reader, writer()).processor(processor).build()).build(); } + public void setFile(String file) { + this.file = file; + } + + public void setFileOptions(FileOptions fileOptions) { + this.fileOptions = fileOptions; + } + + public void setType(FileDumpType type) { + this.type = type; + } + + public void setAppend(boolean append) { + this.append = append; + } + + public void setRootName(String rootName) { + this.rootName = rootName; + } + + public void setElementName(String elementName) { + this.elementName = elementName; + } + + public void setLineSeparator(String lineSeparator) { + this.lineSeparator = lineSeparator; + } + } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileDumpImport.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileDumpImport.java index 982ba114d..00da4abc3 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/FileDumpImport.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileDumpImport.java @@ -14,6 +14,7 @@ import com.redis.riot.core.AbstractImport; import com.redis.spring.batch.KeyValue; import com.redis.spring.batch.RedisItemWriter; +import com.redis.spring.batch.reader.MemKeyValue; public class FileDumpImport extends AbstractImport { @@ -59,11 +60,11 @@ protected Job job() { return job.build(); } - private ItemReader> reader(Resource resource) { + private ItemReader> reader(Resource resource) { if (type == FileDumpType.XML) { - return FileUtils.xmlReader(resource, KeyValue.class); + return FileUtils.xmlReader(resource, MemKeyValue.class); } - return FileUtils.jsonReader(resource, KeyValue.class); + return FileUtils.jsonReader(resource, MemKeyValue.class); } } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileOptions.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileOptions.java index 30951a342..be1ceead5 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/FileOptions.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileOptions.java @@ -1,49 +1,49 @@ package com.redis.riot.file; -import com.redis.riot.file.resource.AbstractResourceItemWriter; +import java.nio.charset.StandardCharsets; public class FileOptions { - public static final String DEFAULT_ENCODING = AbstractResourceItemWriter.DEFAULT_CHARSET; + public static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name(); - private String encoding = DEFAULT_ENCODING; + private String encoding = DEFAULT_ENCODING; - private boolean gzipped; + private boolean gzipped; - private GoogleStorageOptions googleStorageOptions = new GoogleStorageOptions(); + private GoogleStorageOptions googleStorageOptions = new GoogleStorageOptions(); - private AmazonS3Options amazonS3Options = new AmazonS3Options(); + private AmazonS3Options amazonS3Options = new AmazonS3Options(); - public String getEncoding() { - return encoding; - } + public String getEncoding() { + return encoding; + } - public void setEncoding(String encoding) { - this.encoding = encoding; - } + public void setEncoding(String encoding) { + this.encoding = encoding; + } - public boolean isGzipped() { - return gzipped; - } + public boolean isGzipped() { + return gzipped; + } - public void setGzipped(boolean gzipped) { - this.gzipped = gzipped; - } + public void setGzipped(boolean gzipped) { + this.gzipped = gzipped; + } - public GoogleStorageOptions getGoogleStorageOptions() { - return googleStorageOptions; - } + public GoogleStorageOptions getGoogleStorageOptions() { + return googleStorageOptions; + } - public void setGoogleStorageOptions(GoogleStorageOptions googleStorageOptions) { - this.googleStorageOptions = googleStorageOptions; - } + public void setGoogleStorageOptions(GoogleStorageOptions googleStorageOptions) { + this.googleStorageOptions = googleStorageOptions; + } - public AmazonS3Options getAmazonS3Options() { - return amazonS3Options; - } + public AmazonS3Options getAmazonS3Options() { + return amazonS3Options; + } - public void setAmazonS3Options(AmazonS3Options amazonS3Options) { - this.amazonS3Options = amazonS3Options; - } + public void setAmazonS3Options(AmazonS3Options amazonS3Options) { + this.amazonS3Options = amazonS3Options; + } } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/FileUtils.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FileUtils.java index 7f5779ef1..e3d413503 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/FileUtils.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FileUtils.java @@ -48,13 +48,10 @@ import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.ServiceOptions; import com.google.cloud.storage.StorageOptions; -import com.redis.riot.file.resource.FilenameInputStreamResource; -import com.redis.riot.file.resource.OutputStreamResource; -import com.redis.riot.file.resource.UncustomizedUrlResource; -import com.redis.riot.file.resource.XmlItemReader; -import com.redis.riot.file.resource.XmlItemReaderBuilder; -import com.redis.riot.file.resource.XmlObjectReader; -import com.redis.spring.batch.KeyValue; +import com.redis.riot.file.xml.XmlItemReader; +import com.redis.riot.file.xml.XmlItemReaderBuilder; +import com.redis.riot.file.xml.XmlObjectReader; +import com.redis.spring.batch.reader.MemKeyValue; public abstract class FileUtils { @@ -187,7 +184,7 @@ public static XmlMapper xmlMapper() { private static void configureMapper(ObjectMapper mapper) { mapper.configure(DeserializationFeature.USE_LONG_FOR_INTS, true); SimpleModule module = new SimpleModule(); - module.addDeserializer(KeyValue.class, new KeyValueDeserializer()); + module.addDeserializer(MemKeyValue.class, new MemKeyValueDeserializer()); mapper.registerModule(module); mapper.setSerializationInclusion(Include.NON_NULL); } diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/FilenameInputStreamResource.java b/connectors/riot-file/src/main/java/com/redis/riot/file/FilenameInputStreamResource.java similarity index 89% rename from connectors/riot-file/src/main/java/com/redis/riot/file/resource/FilenameInputStreamResource.java rename to connectors/riot-file/src/main/java/com/redis/riot/file/FilenameInputStreamResource.java index 6203a8c04..77358e1ac 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/FilenameInputStreamResource.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/FilenameInputStreamResource.java @@ -1,11 +1,11 @@ -package com.redis.riot.file.resource; +package com.redis.riot.file; import java.io.InputStream; import java.util.Objects; import org.springframework.core.io.InputStreamResource; -import com.redis.riot.file.resource.FilenameInputStreamResource; +import com.redis.riot.file.FilenameInputStreamResource; public class FilenameInputStreamResource extends InputStreamResource { diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/KeyValueDeserializer.java b/connectors/riot-file/src/main/java/com/redis/riot/file/MemKeyValueDeserializer.java similarity index 92% rename from connectors/riot-file/src/main/java/com/redis/riot/file/KeyValueDeserializer.java rename to connectors/riot-file/src/main/java/com/redis/riot/file/MemKeyValueDeserializer.java index 91ad2f9e4..4b0e95039 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/KeyValueDeserializer.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/MemKeyValueDeserializer.java @@ -16,11 +16,12 @@ import com.redis.lettucemod.timeseries.Sample; import com.redis.spring.batch.KeyValue; import com.redis.spring.batch.KeyValue.DataType; +import com.redis.spring.batch.reader.MemKeyValue; import io.lettuce.core.ScoredValue; import io.lettuce.core.StreamMessage; -public class KeyValueDeserializer extends StdDeserializer> { +public class MemKeyValueDeserializer extends StdDeserializer> { private static final long serialVersionUID = 1L; @@ -35,18 +36,18 @@ public class KeyValueDeserializer extends StdDeserializer> t) { + public MemKeyValueDeserializer(Class> t) { super(t); } @Override - public KeyValue deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + public MemKeyValue deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { JsonNode node = p.getCodec().readTree(p); - KeyValue keyValue = new KeyValue<>(); + MemKeyValue keyValue = new MemKeyValue<>(); JsonNode keyNode = node.get(KEY); if (keyNode != null) { keyValue.setKey(node.get(KEY).asText()); diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/OutputStreamResource.java b/connectors/riot-file/src/main/java/com/redis/riot/file/OutputStreamResource.java similarity index 97% rename from connectors/riot-file/src/main/java/com/redis/riot/file/resource/OutputStreamResource.java rename to connectors/riot-file/src/main/java/com/redis/riot/file/OutputStreamResource.java index 2c6a4c465..4e1f69667 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/OutputStreamResource.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/OutputStreamResource.java @@ -1,4 +1,4 @@ -package com.redis.riot.file.resource; +package com.redis.riot.file; import java.io.IOException; import java.io.InputStream; diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/UncustomizedUrlResource.java b/connectors/riot-file/src/main/java/com/redis/riot/file/UncustomizedUrlResource.java similarity index 94% rename from connectors/riot-file/src/main/java/com/redis/riot/file/resource/UncustomizedUrlResource.java rename to connectors/riot-file/src/main/java/com/redis/riot/file/UncustomizedUrlResource.java index ace4c0265..a867983e7 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/UncustomizedUrlResource.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/UncustomizedUrlResource.java @@ -1,4 +1,4 @@ -package com.redis.riot.file.resource; +package com.redis.riot.file; import org.springframework.core.io.UrlResource; diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/AbstractResourceItemWriter.java b/connectors/riot-file/src/main/java/com/redis/riot/file/resource/AbstractResourceItemWriter.java deleted file mode 100644 index 0241058b2..000000000 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/AbstractResourceItemWriter.java +++ /dev/null @@ -1,561 +0,0 @@ -/* - * Copyright 2006-2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.redis.riot.file.resource; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.Writer; -import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; -import java.nio.charset.UnsupportedCharsetException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.batch.item.Chunk; -import org.springframework.batch.item.ExecutionContext; -import org.springframework.batch.item.ItemStream; -import org.springframework.batch.item.ItemStreamException; -import org.springframework.batch.item.WriteFailedException; -import org.springframework.batch.item.WriterNotOpenException; -import org.springframework.batch.item.file.FlatFileFooterCallback; -import org.springframework.batch.item.file.FlatFileHeaderCallback; -import org.springframework.batch.item.file.ResourceAwareItemWriterItemStream; -import org.springframework.batch.item.support.AbstractItemStreamItemWriter; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.core.io.Resource; -import org.springframework.core.io.WritableResource; -import org.springframework.util.Assert; - -/** - * Base class for item writers that write data to a file or stream. This class - * provides common features like restart, force sync, append etc. The location - * of the output file is defined by a {@link Resource} which must represent a - * writable file.
- * - * Uses buffered writer to improve performance.
- * - * The implementation is not thread-safe. - * - * @author Waseem Malik - * @author Tomas Slanina - * @author Robert Kasanicky - * @author Dave Syer - * @author Michael Minella - * @author Mahmoud Ben Hassine - * - * @since 4.1 - */ -public abstract class AbstractResourceItemWriter extends AbstractItemStreamItemWriter - implements ResourceAwareItemWriterItemStream, InitializingBean { - - public static final boolean DEFAULT_TRANSACTIONAL = true; - - protected final Logger log = LoggerFactory.getLogger(AbstractResourceItemWriter.class); - - public static final String DEFAULT_LINE_SEPARATOR = System.getProperty("line.separator"); - - // default encoding for writing to output files - set to UTF-8. - public static final String DEFAULT_CHARSET = "UTF-8"; - - private static final String WRITTEN_STATISTICS_NAME = "written"; - - private static final String RESTART_DATA_NAME = "current.count"; - - private WritableResource resource; - - protected OutputState state = null; - - private boolean saveState = true; - - protected boolean shouldDeleteIfExists = true; - - private boolean shouldDeleteIfEmpty = false; - - private String encoding = DEFAULT_CHARSET; - - private FlatFileHeaderCallback headerCallback; - - private FlatFileFooterCallback footerCallback; - - protected String lineSeparator = DEFAULT_LINE_SEPARATOR; - - private boolean transactional = DEFAULT_TRANSACTIONAL; - - protected boolean append = false; - - /** - * Public setter for the line separator. Defaults to the System property - * line.separator. - * - * @param lineSeparator the line separator to set - */ - public void setLineSeparator(String lineSeparator) { - this.lineSeparator = lineSeparator; - } - - /** - * Setter for resource. Represents a file that can be written. - * - * @param resource the resource to be written to - */ - @Override - public void setResource(WritableResource resource) { - this.resource = resource; - } - - /** - * Sets encoding for output template. - * - * @param newEncoding {@link String} containing the encoding to be used for the - * writer. - */ - public void setEncoding(String newEncoding) { - this.encoding = newEncoding; - } - - /** - * Flag to indicate that the target file should be deleted if it already exists, - * otherwise it will be created. Defaults to true, so no appending except on - * restart. If set to false and {@link #setAppendAllowed(boolean) appendAllowed} - * is also false then there will be an exception when the stream is opened to - * prevent existing data being potentially corrupted. - * - * @param shouldDeleteIfExists the flag value to set - */ - public void setShouldDeleteIfExists(boolean shouldDeleteIfExists) { - this.shouldDeleteIfExists = shouldDeleteIfExists; - } - - /** - * Flag to indicate that the target file should be appended if it already - * exists. If this flag is set then the flag - * {@link #setShouldDeleteIfExists(boolean) shouldDeleteIfExists} is - * automatically set to false, so that flag should not be set explicitly. - * Defaults value is false. - * - * @param append the flag value to set - */ - public void setAppendAllowed(boolean append) { - this.append = append; - } - - /** - * Flag to indicate that the target file should be deleted if no lines have been - * written (other than header and footer) on close. Defaults to false. - * - * @param shouldDeleteIfEmpty the flag value to set - */ - public void setShouldDeleteIfEmpty(boolean shouldDeleteIfEmpty) { - this.shouldDeleteIfEmpty = shouldDeleteIfEmpty; - } - - /** - * Set the flag indicating whether or not state should be saved in the provided - * {@link ExecutionContext} during the {@link ItemStream} call to update. - * Setting this to false means that it will always start at the beginning on a - * restart. - * - * @param saveState if true, state will be persisted - */ - public void setSaveState(boolean saveState) { - this.saveState = saveState; - } - - /** - * headerCallback will be called before writing the first item to file. Newline - * will be automatically appended after the header is written. - * - * @param headerCallback {@link FlatFileHeaderCallback} to generate the header - * - */ - public void setHeaderCallback(FlatFileHeaderCallback headerCallback) { - this.headerCallback = headerCallback; - } - - /** - * footerCallback will be called after writing the last item to file, but before - * the file is closed. - * - * @param footerCallback {@link FlatFileFooterCallback} to generate the footer - * - */ - public void setFooterCallback(FlatFileFooterCallback footerCallback) { - this.footerCallback = footerCallback; - } - - /** - * Flag to indicate that writing to the buffer should be delayed if a - * transaction is active. Defaults to true. - * - * @param transactional true if writing to buffer should be delayed. - * - */ - public void setTransactional(boolean transactional) { - this.transactional = transactional; - } - - /** - * Writes out a string followed by a "new line", where the format of the new - * line separator is determined by the underlying operating system. - * - * @param items list of items to be written to output stream - * @throws Exception if an error occurs while writing items to the output stream - */ - @Override - public void write(Chunk items) throws Exception { - if (!getOutputState().isInitialized()) { - throw new WriterNotOpenException("Writer must be open before it can be written to"); - } - - log.debug("Writing to file with {} items.", items.size()); - - OutputState outputState = getOutputState(); - - String lines = doWrite(items); - try { - outputState.write(lines); - } catch (IOException e) { - throw new WriteFailedException("Could not write data. The file may be corrupt.", e); - } - outputState.setLinesWritten(outputState.getLinesWritten() + items.size()); - } - - /** - * Write out a string of items followed by a "new line", where the format of the - * new line separator is determined by the underlying operating system. - * - * @param items to be written - * @return written lines - */ - protected abstract String doWrite(Chunk items); - - /** - * @see ItemStream#close() - */ - @Override - public void close() { - if (state != null) { - try { - if (footerCallback != null && state.outputBufferedWriter != null) { - footerCallback.writeFooter(state.outputBufferedWriter); - state.outputBufferedWriter.flush(); - } - } catch (IOException e) { - throw new ItemStreamException("Failed to write footer before closing", e); - } finally { - state.close(); - state = null; - } - } - } - - /** - * Initialize the reader. This method may be called multiple times before close - * is called. - * - * @see ItemStream#open(ExecutionContext) - */ - @Override - public void open(ExecutionContext executionContext) throws ItemStreamException { - Assert.notNull(resource, "The resource must be set"); - - if (!getOutputState().isInitialized()) { - doOpen(executionContext); - } - } - - private void doOpen(ExecutionContext executionContext) throws ItemStreamException { - OutputState outputState = getOutputState(); - if (executionContext.containsKey(getExecutionContextKey(RESTART_DATA_NAME))) { - outputState.restoreFrom(executionContext); - } - try { - outputState.initializeBufferedWriter(); - } catch (IOException ioe) { - throw new ItemStreamException("Failed to initialize writer", ioe); - } - if (outputState.lastMarkedByteOffsetPosition == 0 && !outputState.appending && headerCallback != null) { - try { - headerCallback.writeHeader(outputState.outputBufferedWriter); - outputState.write(lineSeparator); - } catch (IOException e) { - throw new ItemStreamException("Could not write headers. The file may be corrupt.", e); - } - } - } - - /** - * @see ItemStream#update(ExecutionContext) - */ - @Override - public void update(ExecutionContext executionContext) { - if (state == null) { - throw new ItemStreamException("ItemStream not open or already closed."); - } - - Assert.notNull(executionContext, "ExecutionContext must not be null"); - - if (saveState) { - - try { - executionContext.putLong(getExecutionContextKey(RESTART_DATA_NAME), state.position()); - } catch (IOException e) { - throw new ItemStreamException("ItemStream does not return current position properly", e); - } - - executionContext.putLong(getExecutionContextKey(WRITTEN_STATISTICS_NAME), state.linesWritten); - } - } - - // Returns object representing state. - protected OutputState getOutputState() { - if (state == null) { - Assert.state(!resource.exists() || resource.isWritable(), "Resource is not writable: [" + resource + "]"); - state = new OutputState(); - state.setDeleteIfExists(shouldDeleteIfExists); - state.setAppendAllowed(append); - state.setEncoding(encoding); - } - return state; - } - - /** - * Encapsulates the runtime state of the writer. All state changing operations - * on the writer go through this class. - */ - protected class OutputState { - - private static final String ERROR_CLOSING = "Unable to close the the ItemWriter"; - - private CountingOutputStream os; - - // The bufferedWriter over the file channel that is actually written - Writer outputBufferedWriter; - - WritableByteChannel fileChannel; - - // this represents the charset encoding (if any is needed) for the - // output file - String encoding = DEFAULT_CHARSET; - - boolean restarted = false; - - long lastMarkedByteOffsetPosition = 0; - - long linesWritten = 0; - - boolean shouldDeleteIfExists = true; - - boolean initialized = false; - - private boolean append = false; - - private boolean appending = false; - - /** - * @return byte offset position of the cursor in the output file as a long - * @throws IOException If unable to get the offset position - */ - public long position() throws IOException { - long pos = 0; - - if (fileChannel == null) { - return 0; - } - - outputBufferedWriter.flush(); - pos = os.getCount(); - if (transactional) { - pos += ((TransactionAwareBufferedWriter) outputBufferedWriter).getBufferSize(); - } - - return pos; - - } - - /** - * @param append if true, append to previously created file - */ - public void setAppendAllowed(boolean append) { - this.append = append; - } - - /** - * @param executionContext state from which to restore writing from - */ - public void restoreFrom(ExecutionContext executionContext) { - lastMarkedByteOffsetPosition = executionContext.getLong(getExecutionContextKey(RESTART_DATA_NAME)); - linesWritten = executionContext.getLong(getExecutionContextKey(WRITTEN_STATISTICS_NAME)); - if (shouldDeleteIfEmpty && linesWritten == 0) { - // previous execution deleted the output file because no items were written - restarted = false; - lastMarkedByteOffsetPosition = 0; - } else { - restarted = true; - } - } - - /** - * @param shouldDeleteIfExists indicator - */ - public void setDeleteIfExists(boolean shouldDeleteIfExists) { - this.shouldDeleteIfExists = shouldDeleteIfExists; - } - - /** - * @param encoding file encoding - */ - public void setEncoding(String encoding) { - this.encoding = encoding; - } - - public long getLinesWritten() { - return linesWritten; - } - - public void setLinesWritten(long linesWritten) { - this.linesWritten = linesWritten; - } - - /** - * Close the open resource and reset counters. - */ - public void close() { - - initialized = false; - restarted = false; - try { - if (outputBufferedWriter != null) { - outputBufferedWriter.close(); - } - } catch (IOException ioe) { - throw new ItemStreamException(ERROR_CLOSING, ioe); - } finally { - if (!transactional) { - closeStream(); - } - } - } - - private void closeStream() { - try { - if (fileChannel != null) { - fileChannel.close(); - } - } catch (IOException ioe) { - throw new ItemStreamException(ERROR_CLOSING, ioe); - } finally { - try { - if (os != null) { - os.close(); - } - } catch (IOException ioe) { - throw new ItemStreamException(ERROR_CLOSING, ioe); - } - } - } - - /** - * @param line String to be written to the file - * @throws IOException If unable to write the String to the file - */ - public void write(String line) throws IOException { - if (!initialized) { - initializeBufferedWriter(); - } - - outputBufferedWriter.write(line); - outputBufferedWriter.flush(); - } - - /** - * Creates the buffered writer for the output file channel based on - * configuration information. - * - * @throws IOException if unable to initialize buffer - */ - private void initializeBufferedWriter() throws IOException { - - os = new CountingOutputStream(resource.getOutputStream()); - fileChannel = Channels.newChannel(os); - - outputBufferedWriter = getBufferedWriter(fileChannel, encoding); - outputBufferedWriter.flush(); - - if (append && resource.contentLength() > 0) { - // Bug in IO library? This doesn't work... - // lastMarkedByteOffsetPosition = fileChannel.position(); - appending = true; - // Don't write the headers again - } - - Assert.state(outputBufferedWriter != null, "Unable to initialize buffered writer"); - // in case of restarting reset position to last committed point - if (restarted) { - checkFileSize(); - } - - initialized = true; - } - - public boolean isInitialized() { - return initialized; - } - - /** - * Returns the buffered writer opened to the beginning of the file specified by - * the absolute path name contained in absoluteFileName. - */ - private Writer getBufferedWriter(WritableByteChannel fileChannel, String encoding) { - try { - final WritableByteChannel channel = fileChannel; - if (transactional) { - TransactionAwareBufferedWriter writer = new TransactionAwareBufferedWriter(channel, - this::closeStream); - - writer.setEncoding(encoding); - return writer; - } else { - return new BufferedWriter(Channels.newWriter(fileChannel, encoding)); - } - } catch (UnsupportedCharsetException ucse) { - throw new ItemStreamException("Bad encoding configuration for output file " + fileChannel, ucse); - } - } - - /** - * Checks (on setState) to make sure that the current output file's size is not - * smaller than the last saved commit point. If it is, then the file has been - * damaged in some way and whole task must be started over again from the - * beginning. - * - * @throws IOException if there is an IO problem - */ - private void checkFileSize() throws IOException { - long size = -1; - - outputBufferedWriter.flush(); - size = os.getCount(); - - if (size < lastMarkedByteOffsetPosition) { - throw new ItemStreamException("Current file size is smaller than size at last commit"); - } - } - - } - -} diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/CountingOutputStream.java b/connectors/riot-file/src/main/java/com/redis/riot/file/resource/CountingOutputStream.java deleted file mode 100644 index e88449b2f..000000000 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/CountingOutputStream.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2007 The Guava Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ - -package com.redis.riot.file.resource; - -import java.io.FilterOutputStream; -import java.io.IOException; -import java.io.OutputStream; - -/** - * An OutputStream that counts the number of bytes written. - * - * @author Chris Nokleberg - * @since 1.0 - */ -public final class CountingOutputStream extends FilterOutputStream { - - private long count; - - /** - * Wraps another output stream, counting the number of bytes written. - * - * @param out the output stream to be wrapped - */ - public CountingOutputStream(OutputStream out) { - super(out); - } - - /** - * - * @return number of bytes written. - */ - public long getCount() { - return count; - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - out.write(b, off, len); - count += len; - } - - @Override - public void write(int b) throws IOException { - out.write(b); - count++; - } - - // Overriding close() because FilterOutputStream's close() method pre-JDK8 has - // bad behavior: - // it silently ignores any exception thrown by flush(). Instead, just close the - // delegate stream. - // It should flush itself if necessary. - @Override - public void close() throws IOException { - out.close(); - } - -} diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/FlatResourceItemWriter.java b/connectors/riot-file/src/main/java/com/redis/riot/file/resource/FlatResourceItemWriter.java deleted file mode 100644 index 2891c9fdc..000000000 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/FlatResourceItemWriter.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2006-2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.redis.riot.file.resource; - -import org.springframework.batch.item.Chunk; -import org.springframework.batch.item.file.transform.LineAggregator; -import org.springframework.core.io.Resource; -import org.springframework.util.Assert; -import org.springframework.util.ClassUtils; - -/** - * This class is an item writer that writes data to a file or stream. The writer - * also provides restart. The location of the output file is defined by a - * {@link Resource} and must represent a writable file.
- * - * Uses buffered writer to improve performance.
- * - * The implementation is not thread-safe. - * - * @author Waseem Malik - * @author Tomas Slanina - * @author Robert Kasanicky - * @author Dave Syer - * @author Michael Minella - * @author Mahmoud Ben Hassine - */ -public class FlatResourceItemWriter extends AbstractResourceItemWriter { - - protected LineAggregator lineAggregator; - - public FlatResourceItemWriter() { - this.setExecutionContextName(ClassUtils.getShortName(FlatResourceItemWriter.class)); - } - - /** - * Assert that mandatory properties (lineAggregator) are set. - * - * @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet() - */ - @Override - public void afterPropertiesSet() throws Exception { - Assert.notNull(lineAggregator, "A LineAggregator must be provided."); - if (append) { - shouldDeleteIfExists = false; - } - } - - /** - * Public setter for the {@link LineAggregator}. This will be used to translate - * the item into a line for output. - * - * @param lineAggregator the {@link LineAggregator} to set - */ - public void setLineAggregator(LineAggregator lineAggregator) { - this.lineAggregator = lineAggregator; - } - - @Override - public String doWrite(Chunk items) { - StringBuilder lines = new StringBuilder(); - for (T item : items) { - lines.append(this.lineAggregator.aggregate(item)).append(this.lineSeparator); - } - return lines.toString(); - } - -} diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/FlatResourceItemWriterBuilder.java b/connectors/riot-file/src/main/java/com/redis/riot/file/resource/FlatResourceItemWriterBuilder.java deleted file mode 100644 index 2156137a7..000000000 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/FlatResourceItemWriterBuilder.java +++ /dev/null @@ -1,472 +0,0 @@ -package com.redis.riot.file.resource; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; - -import org.springframework.batch.item.file.FlatFileFooterCallback; -import org.springframework.batch.item.file.FlatFileHeaderCallback; -import org.springframework.batch.item.file.transform.BeanWrapperFieldExtractor; -import org.springframework.batch.item.file.transform.DelimitedLineAggregator; -import org.springframework.batch.item.file.transform.FieldExtractor; -import org.springframework.batch.item.file.transform.FormatterLineAggregator; -import org.springframework.batch.item.file.transform.LineAggregator; -import org.springframework.core.io.Resource; -import org.springframework.core.io.WritableResource; -import org.springframework.util.Assert; -import org.springframework.util.StringUtils; - -public class FlatResourceItemWriterBuilder { - - private WritableResource resource; - - private String lineSeparator = AbstractResourceItemWriter.DEFAULT_LINE_SEPARATOR; - - private LineAggregator lineAggregator; - - private String encoding = AbstractResourceItemWriter.DEFAULT_CHARSET; - - private boolean shouldDeleteIfExists = true; - - private boolean append = false; - - private boolean shouldDeleteIfEmpty = false; - - private FlatFileHeaderCallback headerCallback; - - private FlatFileFooterCallback footerCallback; - - private boolean saveState = true; - - private String name; - - private DelimitedBuilder delimitedBuilder; - - private FormattedBuilder formattedBuilder; - - /** - * Configure if the state of the - * {@link org.springframework.batch.item.ItemStreamSupport} should be persisted - * within the {@link org.springframework.batch.item.ExecutionContext} for - * restart purposes. - * - * @param saveState defaults to true - * @return The current instance of the builder. - */ - public FlatResourceItemWriterBuilder saveState(boolean saveState) { - this.saveState = saveState; - - return this; - } - - /** - * The name used to calculate the key within the - * {@link org.springframework.batch.item.ExecutionContext}. Required if - * {@link #saveState(boolean)} is set to true. - * - * @param name name of the reader instance - * @return The current instance of the builder. - * @see org.springframework.batch.item.ItemStreamSupport#setName(String) - */ - public FlatResourceItemWriterBuilder name(String name) { - this.name = name; - - return this; - } - - /** - * The {@link Resource} to be used as output. - * - * @param resource the output of the writer. - * @return The current instance of the builder. - */ - public FlatResourceItemWriterBuilder resource(Resource resource) { - Assert.isInstanceOf(WritableResource.class, resource); - this.resource = (WritableResource) resource; - - return this; - } - - /** - * String used to separate lines in output. Defaults to the System property - * line.separator. - * - * @param lineSeparator value to use for a line separator - * @return The current instance of the builder. - * @see FlatResourceItemWriter#setLineSeparator(String) - */ - public FlatResourceItemWriterBuilder lineSeparator(String lineSeparator) { - this.lineSeparator = lineSeparator; - - return this; - } - - /** - * Line aggregator used to build the String version of each item. - * - * @param lineAggregator {@link LineAggregator} implementation - * @return The current instance of the builder. - * @see FlatResourceItemWriter#setLineAggregator(LineAggregator) - */ - public FlatResourceItemWriterBuilder lineAggregator(LineAggregator lineAggregator) { - this.lineAggregator = lineAggregator; - - return this; - } - - /** - * Encoding used for output. - * - * @param encoding encoding type. - * @return The current instance of the builder. - * @see FlatResourceItemWriter#setEncoding(String) - */ - public FlatResourceItemWriterBuilder encoding(String encoding) { - this.encoding = encoding; - - return this; - } - - /** - * If set to true, once the step is complete, if the resource previously - * provided is empty, it will be deleted. - * - * @param shouldDelete defaults to false - * @return The current instance of the builder - * @see FlatResourceItemWriter#setShouldDeleteIfEmpty(boolean) - */ - public FlatResourceItemWriterBuilder shouldDeleteIfEmpty(boolean shouldDelete) { - this.shouldDeleteIfEmpty = shouldDelete; - - return this; - } - - /** - * If set to true, upon the start of the step, if the resource already exists, - * it will be deleted and recreated. - * - * @param shouldDelete defaults to true - * @return The current instance of the builder - * @see FlatResourceItemWriter#setShouldDeleteIfExists(boolean) - */ - public FlatResourceItemWriterBuilder shouldDeleteIfExists(boolean shouldDelete) { - this.shouldDeleteIfExists = shouldDelete; - - return this; - } - - /** - * If set to true and the file exists, the output will be appended to the - * existing file. - * - * @param append defaults to false - * @return The current instance of the builder - * @see FlatResourceItemWriter#setAppendAllowed(boolean) - */ - public FlatResourceItemWriterBuilder append(boolean append) { - this.append = append; - - return this; - } - - /** - * A callback for header processing. - * - * @param callback {@link FlatFileHeaderCallback} impl - * @return The current instance of the builder - * @see FlatResourceItemWriter#setHeaderCallback(FlatFileHeaderCallback) - */ - public FlatResourceItemWriterBuilder headerCallback(FlatFileHeaderCallback callback) { - this.headerCallback = callback; - - return this; - } - - /** - * A callback for footer processing - * - * @param callback {@link FlatFileFooterCallback} impl - * @return The current instance of the builder - * @see FlatResourceItemWriter#setFooterCallback(FlatFileFooterCallback) - */ - public FlatResourceItemWriterBuilder footerCallback(FlatFileFooterCallback callback) { - this.footerCallback = callback; - - return this; - } - - /** - * Returns an instance of a {@link DelimitedBuilder} for building a - * {@link DelimitedLineAggregator}. The {@link DelimitedLineAggregator} - * configured by this builder will only be used if one is not explicitly - * configured via {@link FlatResourceItemWriterBuilder#lineAggregator} - * - * @return a {@link DelimitedBuilder} - * - */ - public DelimitedBuilder delimited() { - this.delimitedBuilder = new DelimitedBuilder<>(this); - return this.delimitedBuilder; - } - - /** - * Returns an instance of a {@link FormattedBuilder} for building a - * {@link FormatterLineAggregator}. The {@link FormatterLineAggregator} - * configured by this builder will only be used if one is not explicitly - * configured via {@link FlatResourceItemWriterBuilder#lineAggregator} - * - * @return a {@link FormattedBuilder} - * - */ - public FormattedBuilder formatted() { - this.formattedBuilder = new FormattedBuilder<>(this); - return this.formattedBuilder; - } - - /** - * A builder for constructing a {@link FormatterLineAggregator}. - * - * @param the type of the parent {@link FlatResourceItemWriterBuilder} - */ - public static class FormattedBuilder { - - private FlatResourceItemWriterBuilder parent; - - private String format; - - private Locale locale = Locale.getDefault(); - - private int maximumLength = 0; - - private int minimumLength = 0; - - private FieldExtractor fieldExtractor; - - private List names = new ArrayList<>(); - - protected FormattedBuilder(FlatResourceItemWriterBuilder parent) { - this.parent = parent; - } - - /** - * Set the format string used to aggregate items - * - * @param format used to aggregate items - * @return The instance of the builder for chaining. - */ - public FormattedBuilder format(String format) { - this.format = format; - return this; - } - - /** - * Set the locale. - * - * @param locale to use - * @return The instance of the builder for chaining. - */ - public FormattedBuilder locale(Locale locale) { - this.locale = locale; - return this; - } - - /** - * Set the minimum length of the formatted string. If this is not set the - * default is to allow any length. - * - * @param minimumLength of the formatted string - * @return The instance of the builder for chaining. - */ - public FormattedBuilder minimumLength(int minimumLength) { - this.minimumLength = minimumLength; - return this; - } - - /** - * Set the maximum length of the formatted string. If this is not set the - * default is to allow any length. - * - * @param maximumLength of the formatted string - * @return The instance of the builder for chaining. - */ - public FormattedBuilder maximumLength(int maximumLength) { - this.maximumLength = maximumLength; - return this; - } - - /** - * Set the {@link FieldExtractor} to use to extract fields from each item. - * - * @param fieldExtractor to use to extract fields from each item - * @return The current instance of the builder - */ - public FlatResourceItemWriterBuilder fieldExtractor(FieldExtractor fieldExtractor) { - this.fieldExtractor = fieldExtractor; - return this.parent; - } - - /** - * Names of each of the fields within the fields that are returned in the order - * they occur within the formatted file. These names will be used to create a - * {@link BeanWrapperFieldExtractor} only if no explicit field extractor is set - * via {@link FormattedBuilder#fieldExtractor(FieldExtractor)}. - * - * @param names names of each field - * @return The parent {@link FlatResourceItemWriterBuilder} - * @see BeanWrapperFieldExtractor#setNames(String[]) - */ - public FlatResourceItemWriterBuilder names(String[] names) { - this.names.addAll(Arrays.asList(names)); - return this.parent; - } - - public FormatterLineAggregator build() { - Assert.notNull(this.format, "A format is required"); - Assert.isTrue((this.names != null && !this.names.isEmpty()) || this.fieldExtractor != null, - "A list of field names or a field extractor is required"); - - FormatterLineAggregator formatterLineAggregator = new FormatterLineAggregator<>(); - formatterLineAggregator.setFormat(this.format); - formatterLineAggregator.setLocale(this.locale); - formatterLineAggregator.setMinimumLength(this.minimumLength); - formatterLineAggregator.setMaximumLength(this.maximumLength); - - if (this.fieldExtractor == null) { - BeanWrapperFieldExtractor beanWrapperFieldExtractor = new BeanWrapperFieldExtractor<>(); - beanWrapperFieldExtractor.setNames(this.names.toArray(new String[this.names.size()])); - try { - beanWrapperFieldExtractor.afterPropertiesSet(); - } catch (Exception e) { - throw new IllegalStateException("Unable to initialize FormatterLineAggregator", e); - } - this.fieldExtractor = beanWrapperFieldExtractor; - } - - formatterLineAggregator.setFieldExtractor(this.fieldExtractor); - return formatterLineAggregator; - } - } - - /** - * A builder for constructing a {@link DelimitedLineAggregator} - * - * @param the type of the parent {@link FlatResourceItemWriterBuilder} - */ - public static class DelimitedBuilder { - - private FlatResourceItemWriterBuilder parent; - - private List names = new ArrayList<>(); - - private String delimiter = ","; - - private FieldExtractor fieldExtractor; - - protected DelimitedBuilder(FlatResourceItemWriterBuilder parent) { - this.parent = parent; - } - - /** - * Define the delimiter for the file. - * - * @param delimiter String used as a delimiter between fields. - * @return The instance of the builder for chaining. - * @see DelimitedLineAggregator#setDelimiter(String) - */ - public DelimitedBuilder delimiter(String delimiter) { - this.delimiter = delimiter; - return this; - } - - /** - * Names of each of the fields within the fields that are returned in the order - * they occur within the delimited file. These names will be used to create a - * {@link BeanWrapperFieldExtractor} only if no explicit field extractor is set - * via {@link DelimitedBuilder#fieldExtractor(FieldExtractor)}. - * - * @param names names of each field - * @return The parent {@link FlatResourceItemWriterBuilder} - * @see BeanWrapperFieldExtractor#setNames(String[]) - */ - public FlatResourceItemWriterBuilder names(String[] names) { - this.names.addAll(Arrays.asList(names)); - return this.parent; - } - - /** - * Set the {@link FieldExtractor} to use to extract fields from each item. - * - * @param fieldExtractor to use to extract fields from each item - * @return The parent {@link FlatResourceItemWriterBuilder} - */ - public FlatResourceItemWriterBuilder fieldExtractor(FieldExtractor fieldExtractor) { - this.fieldExtractor = fieldExtractor; - return this.parent; - } - - public DelimitedLineAggregator build() { - Assert.isTrue((this.names != null && !this.names.isEmpty()) || this.fieldExtractor != null, - "A list of field names or a field extractor is required"); - - DelimitedLineAggregator delimitedLineAggregator = new DelimitedLineAggregator<>(); - if (StringUtils.hasLength(this.delimiter)) { - delimitedLineAggregator.setDelimiter(this.delimiter); - } - - if (this.fieldExtractor == null) { - BeanWrapperFieldExtractor beanWrapperFieldExtractor = new BeanWrapperFieldExtractor<>(); - beanWrapperFieldExtractor.setNames(this.names.toArray(new String[this.names.size()])); - try { - beanWrapperFieldExtractor.afterPropertiesSet(); - } catch (Exception e) { - throw new IllegalStateException("Unable to initialize DelimitedLineAggregator", e); - } - this.fieldExtractor = beanWrapperFieldExtractor; - } - - delimitedLineAggregator.setFieldExtractor(this.fieldExtractor); - return delimitedLineAggregator; - } - } - - /** - * Validates and builds a {@link FlatResourceItemWriter}. - * - * @return a {@link FlatResourceItemWriter} - */ - public FlatResourceItemWriter build() { - - Assert.isTrue(this.lineAggregator != null || this.delimitedBuilder != null || this.formattedBuilder != null, - "A LineAggregator or a DelimitedBuilder or a FormattedBuilder is required"); - Assert.notNull(this.resource, "A Resource is required"); - - if (this.saveState) { - Assert.hasText(this.name, "A name is required when saveState is true"); - } - - FlatResourceItemWriter writer = new FlatResourceItemWriter<>(); - - writer.setName(this.name); - writer.setAppendAllowed(this.append); - writer.setEncoding(this.encoding); - writer.setFooterCallback(this.footerCallback); - writer.setHeaderCallback(this.headerCallback); - if (this.lineAggregator == null) { - Assert.state(this.delimitedBuilder == null || this.formattedBuilder == null, - "Either a DelimitedLineAggregator or a FormatterLineAggregator should be provided, but not both"); - if (this.delimitedBuilder != null) { - this.lineAggregator = this.delimitedBuilder.build(); - } else { - this.lineAggregator = this.formattedBuilder.build(); - } - } - writer.setLineAggregator(this.lineAggregator); - writer.setLineSeparator(this.lineSeparator); - writer.setResource(this.resource); - writer.setSaveState(this.saveState); - writer.setShouldDeleteIfEmpty(this.shouldDeleteIfEmpty); - writer.setShouldDeleteIfExists(this.shouldDeleteIfExists); - return writer; - } -} \ No newline at end of file diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/JsonResourceItemWriter.java b/connectors/riot-file/src/main/java/com/redis/riot/file/resource/JsonResourceItemWriter.java deleted file mode 100644 index 7e2fb3c0c..000000000 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/JsonResourceItemWriter.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.redis.riot.file.resource; - -import java.util.Iterator; - -import org.springframework.batch.item.Chunk; -import org.springframework.batch.item.json.GsonJsonObjectMarshaller; -import org.springframework.batch.item.json.JacksonJsonObjectMarshaller; -import org.springframework.batch.item.json.JsonObjectMarshaller; -import org.springframework.core.io.Resource; -import org.springframework.core.io.WritableResource; -import org.springframework.util.Assert; -import org.springframework.util.ClassUtils; - -/** - * Item writer that writes data in json format to an output file. The location - * of the output file is defined by a {@link Resource} and must represent a - * writable file. Items are transformed to json format using a - * {@link JsonObjectMarshaller}. Items will be enclosed in a json array as - * follows: - * - *

- * - * [ - * {json object}, - * {json object}, - * {json object} - * ] - * - *

- * - * The implementation is not thread-safe. - * - * @see GsonJsonObjectMarshaller - * @see JacksonJsonObjectMarshaller - * @param type of object to write as json representation - * @author Mahmoud Ben Hassine - * @since 4.1 - */ -public class JsonResourceItemWriter extends AbstractResourceItemWriter { - - private static final char JSON_OBJECT_SEPARATOR = ','; - private static final char JSON_ARRAY_START = '['; - private static final char JSON_ARRAY_STOP = ']'; - - private JsonObjectMarshaller jsonObjectMarshaller; - - /** - * Create a new {@link JsonResourceItemWriter} instance. - * - * @param resource to write json data to - * @param jsonObjectMarshaller used to marshal object into json representation - */ - public JsonResourceItemWriter(WritableResource resource, JsonObjectMarshaller jsonObjectMarshaller) { - Assert.notNull(resource, "resource must not be null"); - Assert.notNull(jsonObjectMarshaller, "json object marshaller must not be null"); - setResource(resource); - setJsonObjectMarshaller(jsonObjectMarshaller); - setHeaderCallback(writer -> writer.write(JSON_ARRAY_START)); - setFooterCallback(writer -> writer.write(this.lineSeparator + JSON_ARRAY_STOP + this.lineSeparator)); - setExecutionContextName(ClassUtils.getShortName(JsonResourceItemWriter.class)); - } - - /** - * Assert that mandatory properties (jsonObjectMarshaller) are set. - * - * @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet() - */ - @Override - public void afterPropertiesSet() throws Exception { - if (this.append) { - this.shouldDeleteIfExists = false; - } - } - - /** - * Set the {@link JsonObjectMarshaller} to use to marshal object to json. - * - * @param jsonObjectMarshaller the marshaller to use - */ - public void setJsonObjectMarshaller(JsonObjectMarshaller jsonObjectMarshaller) { - this.jsonObjectMarshaller = jsonObjectMarshaller; - } - - @Override - public String doWrite(Chunk items) { - StringBuilder lines = new StringBuilder(); - Iterator iterator = items.iterator(); - if (!items.isEmpty() && state.getLinesWritten() > 0) { - lines.append(JSON_OBJECT_SEPARATOR).append(this.lineSeparator); - } - while (iterator.hasNext()) { - T item = iterator.next(); - lines.append(' ').append(this.jsonObjectMarshaller.marshal(item)); - if (iterator.hasNext()) { - lines.append(JSON_OBJECT_SEPARATOR).append(this.lineSeparator); - } - } - return lines.toString(); - } - -} diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/JsonResourceItemWriterBuilder.java b/connectors/riot-file/src/main/java/com/redis/riot/file/resource/JsonResourceItemWriterBuilder.java deleted file mode 100644 index b144f6f40..000000000 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/JsonResourceItemWriterBuilder.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright 2018 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.redis.riot.file.resource; - -import org.springframework.batch.item.file.FlatFileFooterCallback; -import org.springframework.batch.item.file.FlatFileHeaderCallback; -import org.springframework.batch.item.json.JsonObjectMarshaller; -import org.springframework.core.io.Resource; -import org.springframework.core.io.WritableResource; -import org.springframework.util.Assert; - -/** - * Builder for {@link JsonResourceItemWriter}. - * - * @param type of objects to write as Json output. - * @author Mahmoud Ben Hassine - * @since 4.1 - */ -public class JsonResourceItemWriterBuilder { - - private WritableResource resource; - private JsonObjectMarshaller jsonObjectMarshaller; - private FlatFileHeaderCallback headerCallback; - private FlatFileFooterCallback footerCallback; - - private String name; - private String encoding = AbstractResourceItemWriter.DEFAULT_CHARSET; - private String lineSeparator = AbstractResourceItemWriter.DEFAULT_LINE_SEPARATOR; - - private boolean append = false; - private boolean saveState = true; - private boolean shouldDeleteIfExists = true; - private boolean shouldDeleteIfEmpty = false; - - /** - * Configure if the state of the - * {@link org.springframework.batch.item.ItemStreamSupport} should be persisted - * within the {@link org.springframework.batch.item.ExecutionContext} for - * restart purposes. - * - * @param saveState defaults to true - * @return The current instance of the builder. - */ - public JsonResourceItemWriterBuilder saveState(boolean saveState) { - this.saveState = saveState; - - return this; - } - - /** - * The name used to calculate the key within the - * {@link org.springframework.batch.item.ExecutionContext}. Required if - * {@link #saveState(boolean)} is set to true. - * - * @param name name of the reader instance - * @return The current instance of the builder. - * @see org.springframework.batch.item.ItemStreamSupport#setName(String) - */ - public JsonResourceItemWriterBuilder name(String name) { - this.name = name; - - return this; - } - - /** - * String used to separate lines in output. Defaults to the System property - * line.separator. - * - * @param lineSeparator value to use for a line separator - * @return The current instance of the builder. - * @see JsonResourceItemWriter#setLineSeparator(String) - */ - public JsonResourceItemWriterBuilder lineSeparator(String lineSeparator) { - this.lineSeparator = lineSeparator; - - return this; - } - - /** - * Set the {@link JsonObjectMarshaller} to use to marshal objects to json. - * - * @param jsonObjectMarshaller to use - * @return The current instance of the builder. - * @see JsonResourceItemWriter#setJsonObjectMarshaller(JsonObjectMarshaller) - */ - public JsonResourceItemWriterBuilder jsonObjectMarshaller(JsonObjectMarshaller jsonObjectMarshaller) { - this.jsonObjectMarshaller = jsonObjectMarshaller; - - return this; - } - - /** - * The {@link Resource} to be used as output. - * - * @param resource the output of the writer. - * @return The current instance of the builder. - */ - public JsonResourceItemWriterBuilder resource(Resource resource) { - Assert.isInstanceOf(WritableResource.class, resource); - this.resource = (WritableResource) resource; - - return this; - } - - /** - * Encoding used for output. - * - * @param encoding encoding type. - * @return The current instance of the builder. - * @see JsonResourceItemWriter#setEncoding(String) - */ - public JsonResourceItemWriterBuilder encoding(String encoding) { - this.encoding = encoding; - - return this; - } - - /** - * If set to true, once the step is complete, if the resource previously - * provided is empty, it will be deleted. - * - * @param shouldDelete defaults to false - * @return The current instance of the builder - * @see JsonResourceItemWriter#setShouldDeleteIfEmpty(boolean) - */ - public JsonResourceItemWriterBuilder shouldDeleteIfEmpty(boolean shouldDelete) { - this.shouldDeleteIfEmpty = shouldDelete; - - return this; - } - - /** - * If set to true, upon the start of the step, if the resource already exists, - * it will be deleted and recreated. - * - * @param shouldDelete defaults to true - * @return The current instance of the builder - * @see JsonResourceItemWriter#setShouldDeleteIfExists(boolean) - */ - public JsonResourceItemWriterBuilder shouldDeleteIfExists(boolean shouldDelete) { - this.shouldDeleteIfExists = shouldDelete; - - return this; - } - - /** - * If set to true and the file exists, the output will be appended to the - * existing file. - * - * @param append defaults to false - * @return The current instance of the builder - * @see JsonResourceItemWriter#setAppendAllowed(boolean) - */ - public JsonResourceItemWriterBuilder append(boolean append) { - this.append = append; - - return this; - } - - /** - * A callback for header processing. - * - * @param callback {@link FlatFileHeaderCallback} implementation - * @return The current instance of the builder - * @see JsonResourceItemWriter#setHeaderCallback(FlatFileHeaderCallback) - */ - public JsonResourceItemWriterBuilder headerCallback(FlatFileHeaderCallback callback) { - this.headerCallback = callback; - - return this; - } - - /** - * A callback for footer processing. - * - * @param callback {@link FlatFileFooterCallback} implementation - * @return The current instance of the builder - * @see JsonResourceItemWriter#setFooterCallback(FlatFileFooterCallback) - */ - public JsonResourceItemWriterBuilder footerCallback(FlatFileFooterCallback callback) { - this.footerCallback = callback; - - return this; - } - - /** - * Validate the configuration and build a new {@link JsonResourceItemWriter}. - * - * @return a new instance of the {@link JsonResourceItemWriter} - */ - public JsonResourceItemWriter build() { - Assert.notNull(this.resource, "A resource is required."); - Assert.notNull(this.jsonObjectMarshaller, "A json object marshaller is required."); - - if (this.saveState) { - Assert.hasText(this.name, "A name is required when saveState is true"); - } - - JsonResourceItemWriter jsonResourceItemWriter = new JsonResourceItemWriter<>(this.resource, - this.jsonObjectMarshaller); - - jsonResourceItemWriter.setName(this.name); - jsonResourceItemWriter.setAppendAllowed(this.append); - jsonResourceItemWriter.setEncoding(this.encoding); - if (this.headerCallback != null) { - jsonResourceItemWriter.setHeaderCallback(this.headerCallback); - } - if (this.footerCallback != null) { - jsonResourceItemWriter.setFooterCallback(this.footerCallback); - } - jsonResourceItemWriter.setLineSeparator(this.lineSeparator); - jsonResourceItemWriter.setSaveState(this.saveState); - jsonResourceItemWriter.setShouldDeleteIfEmpty(this.shouldDeleteIfEmpty); - jsonResourceItemWriter.setShouldDeleteIfExists(this.shouldDeleteIfExists); - return jsonResourceItemWriter; - } -} diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/TransactionAwareBufferedWriter.java b/connectors/riot-file/src/main/java/com/redis/riot/file/resource/TransactionAwareBufferedWriter.java deleted file mode 100644 index 531a5d0e6..000000000 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/TransactionAwareBufferedWriter.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright 2006-2012 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.redis.riot.file.resource; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.io.Writer; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; - -import org.springframework.batch.item.WriteFailedException; -import org.springframework.batch.support.transaction.FlushFailedException; -import org.springframework.transaction.support.TransactionSynchronization; -import org.springframework.transaction.support.TransactionSynchronizationManager; - -/** - * Wrapper for a {@link WritableByteChannel} that delays actually writing to or - * closing the buffer if a transaction is active. If a transaction is detected - * on the call to {@link #write(String)} the parameter is buffered and passed on - * to the underlying writer only when the transaction is committed. - * - * @author Dave Syer - * @author Michael Minella - * - */ -public class TransactionAwareBufferedWriter extends Writer { - - private final Object bufferKey; - - private final Object closeKey; - - private WritableByteChannel channel; - - private final Runnable closeCallback; - - // default encoding for writing to output - set to UTF-8. - private static final String DEFAULT_CHARSET = "UTF-8"; - - private String encoding = DEFAULT_CHARSET; - - /** - * Create a new instance with the underlying byte channel provided, and a - * callback to execute on close. The callback should clean up related resources - * like output streams or channels. - * - * @param channel channel used to do the actual IO - * @param closeCallback callback to execute on close - */ - public TransactionAwareBufferedWriter(WritableByteChannel channel, Runnable closeCallback) { - super(); - this.channel = channel; - this.closeCallback = closeCallback; - this.bufferKey = new Object(); - this.closeKey = new Object(); - } - - public void setEncoding(String encoding) { - this.encoding = encoding; - } - - /** - * @return - */ - private StringBuilder getCurrentBuffer() { - - if (!TransactionSynchronizationManager.hasResource(bufferKey)) { - - TransactionSynchronizationManager.bindResource(bufferKey, new StringBuilder()); - - TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { - @Override - public void afterCompletion(int status) { - clear(); - } - - @Override - public void beforeCommit(boolean readOnly) { - try { - if (!readOnly) { - complete(); - } - } catch (IOException e) { - throw new FlushFailedException("Could not write to output buffer", e); - } - } - - private void complete() throws IOException { - StringBuilder buffer = (StringBuilder) TransactionSynchronizationManager.getResource(bufferKey); - if (buffer != null) { - String string = buffer.toString(); - byte[] bytes = string.getBytes(encoding); - int bufferLength = bytes.length; - ByteBuffer bb = ByteBuffer.wrap(bytes); - int bytesWritten = channel.write(bb); - if (bytesWritten != bufferLength) { - throw new IOException("All bytes to be written were not successfully written"); - } - if (TransactionSynchronizationManager.hasResource(closeKey)) { - closeCallback.run(); - } - } - } - - private void clear() { - if (TransactionSynchronizationManager.hasResource(bufferKey)) { - TransactionSynchronizationManager.unbindResource(bufferKey); - } - if (TransactionSynchronizationManager.hasResource(closeKey)) { - TransactionSynchronizationManager.unbindResource(closeKey); - } - } - - }); - - } - - return (StringBuilder) TransactionSynchronizationManager.getResource(bufferKey); - - } - - /** - * Convenience method for clients to determine if there is any unflushed data. - * - * @return the current size (in bytes) of unflushed buffered data - */ - public long getBufferSize() { - if (!transactionActive()) { - return 0L; - } - StringBuilder buffer = getCurrentBuffer(); - if (buffer == null) { - return 0; - } - try { - return buffer.toString().getBytes(encoding).length; - } catch (UnsupportedEncodingException e) { - throw new WriteFailedException( - "Could not determine buffer size because of unsupported encoding: " + encoding, e); - } - } - - /** - * @return - */ - private boolean transactionActive() { - return TransactionSynchronizationManager.isActualTransactionActive(); - } - - /* - * (non-Javadoc) - * - * @see java.io.Writer#close() - */ - @Override - public void close() throws IOException { - if (transactionActive()) { - StringBuilder buffer = getCurrentBuffer(); - if (buffer != null && buffer.length() > 0) { - TransactionSynchronizationManager.bindResource(closeKey, Boolean.TRUE); - } - return; - } - closeCallback.run(); - } - - /* - * (non-Javadoc) - * - * @see java.io.Writer#flush() - */ - @Override - public void flush() throws IOException { - // do nothing - } - - /* - * (non-Javadoc) - * - * @see java.io.Writer#write(char[], int, int) - */ - @Override - public void write(char[] cbuf, int off, int len) throws IOException { - - if (!transactionActive()) { - char[] subArray = new char[len]; - System.arraycopy(cbuf, off, subArray, 0, len); - byte[] bytes = new String(subArray).getBytes(encoding); - int length = bytes.length; - ByteBuffer bb = ByteBuffer.wrap(bytes); - int bytesWritten = channel.write(bb); - if (bytesWritten != length) { - throw new IOException( - "Unable to write all data. Bytes to write: " + len + ". Bytes written: " + bytesWritten); - } - return; - } - - StringBuilder buffer = getCurrentBuffer(); - if (buffer != null) { - buffer.append(cbuf, off, len); - } - } -} diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlItemReader.java b/connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlItemReader.java similarity index 98% rename from connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlItemReader.java rename to connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlItemReader.java index fa05bcf97..80629e07a 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlItemReader.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlItemReader.java @@ -1,4 +1,4 @@ -package com.redis.riot.file.resource; +package com.redis.riot.file.xml; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlItemReaderBuilder.java b/connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlItemReaderBuilder.java similarity index 99% rename from connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlItemReaderBuilder.java rename to connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlItemReaderBuilder.java index 55f128209..a24206eb8 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlItemReaderBuilder.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlItemReaderBuilder.java @@ -1,4 +1,4 @@ -package com.redis.riot.file.resource; +package com.redis.riot.file.xml; import org.springframework.core.io.Resource; import org.springframework.util.Assert; diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlObjectReader.java b/connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlObjectReader.java similarity index 95% rename from connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlObjectReader.java rename to connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlObjectReader.java index f5c668c40..81f6c4410 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlObjectReader.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlObjectReader.java @@ -1,8 +1,8 @@ -package com.redis.riot.file.resource; +package com.redis.riot.file.xml; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.redis.riot.file.resource.XmlObjectReader; +import com.redis.riot.file.xml.XmlObjectReader; import org.springframework.core.io.Resource; import org.springframework.lang.Nullable; diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlResourceItemWriter.java b/connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlResourceItemWriter.java similarity index 95% rename from connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlResourceItemWriter.java rename to connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlResourceItemWriter.java index 3ef0c976c..3ce7aa2c8 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlResourceItemWriter.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlResourceItemWriter.java @@ -14,12 +14,13 @@ * limitations under the License. */ -package com.redis.riot.file.resource; +package com.redis.riot.file.xml; import java.util.Iterator; import org.springframework.batch.item.Chunk; import org.springframework.batch.item.json.JsonObjectMarshaller; +import org.springframework.batch.item.support.AbstractFileItemWriter; import org.springframework.core.io.Resource; import org.springframework.core.io.WritableResource; import org.springframework.util.Assert; @@ -46,7 +47,7 @@ * The implementation is not thread-safe. * */ -public class XmlResourceItemWriter extends AbstractResourceItemWriter { +public class XmlResourceItemWriter extends AbstractFileItemWriter { private JsonObjectMarshaller xmlObjectMarshaller; diff --git a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlResourceItemWriterBuilder.java b/connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlResourceItemWriterBuilder.java similarity index 99% rename from connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlResourceItemWriterBuilder.java rename to connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlResourceItemWriterBuilder.java index ed89546dc..0427b2611 100644 --- a/connectors/riot-file/src/main/java/com/redis/riot/file/resource/XmlResourceItemWriterBuilder.java +++ b/connectors/riot-file/src/main/java/com/redis/riot/file/xml/XmlResourceItemWriterBuilder.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.redis.riot.file.resource; +package com.redis.riot.file.xml; import org.springframework.batch.item.file.FlatFileFooterCallback; import org.springframework.batch.item.file.FlatFileHeaderCallback; diff --git a/connectors/riot-file/src/test/java/com/redis/riot/file/AbstractFileTests.java b/connectors/riot-file/src/test/java/com/redis/riot/file/AbstractFileTests.java index 1e01af73f..b68590adf 100644 --- a/connectors/riot-file/src/test/java/com/redis/riot/file/AbstractFileTests.java +++ b/connectors/riot-file/src/test/java/com/redis/riot/file/AbstractFileTests.java @@ -14,6 +14,7 @@ import org.junit.jupiter.api.TestInfo; import com.amazonaws.util.IOUtils; +import com.redis.riot.core.AbstractRedisCallable; import com.redis.riot.core.operation.HsetBuilder; import com.redis.spring.batch.test.AbstractTestBase; @@ -30,8 +31,7 @@ abstract class AbstractFileTests extends AbstractTestBase { @Test void fileImportJSON(TestInfo info) throws Exception { try (FileImport executable = new FileImport()) { - executable.getRedisClientOptions().setRedisURI(redisURI); - executable.getRedisClientOptions().setCluster(getRedisServer().isRedisCluster()); + configure(executable); executable.setFiles(BEERS_JSON_URL); HsetBuilder hsetBuilder = new HsetBuilder(); hsetBuilder.setKeyspace(KEYSPACE); @@ -56,8 +56,7 @@ void fileImportJSON(TestInfo info) throws Exception { @Test void fileApiImportCSV(TestInfo info) throws Exception { try (FileImport executable = new FileImport()) { - executable.getRedisClientOptions().setRedisURI(redisURI); - executable.getRedisClientOptions().setCluster(getRedisServer().isRedisCluster()); + configure(executable); executable.setFiles("https://storage.googleapis.com/jrx/beers.csv"); executable.setHeader(true); executable.setName(name(info)); @@ -77,6 +76,11 @@ void fileApiImportCSV(TestInfo info) throws Exception { } } + private void configure(AbstractRedisCallable callable) { + callable.getRedisClientOptions().getUriOptions().setUri(getRedisServer().getRedisURI()); + callable.getRedisClientOptions().setCluster(getRedisServer().isRedisCluster()); + } + @SuppressWarnings("unchecked") @Test void fileApiFileExpansion(TestInfo info) throws Exception { @@ -86,8 +90,7 @@ void fileApiFileExpansion(TestInfo info) throws Exception { File file2 = temp.resolve("beers2.csv").toFile(); IOUtils.copy(getClass().getClassLoader().getResourceAsStream("beers2.csv"), new FileOutputStream(file2)); try (FileImport executable = new FileImport()) { - executable.getRedisClientOptions().setRedisURI(redisURI); - executable.getRedisClientOptions().setCluster(getRedisServer().isRedisCluster()); + configure(executable); executable.setFiles(temp.resolve("*.csv").toFile().getPath()); executable.setHeader(true); executable.setName(name(info)); @@ -111,8 +114,7 @@ void fileApiFileExpansion(TestInfo info) throws Exception { @Test void fileImportCSVMultiThreaded(TestInfo info) throws Exception { try (FileImport executable = new FileImport()) { - executable.getRedisClientOptions().setRedisURI(redisURI); - executable.getRedisClientOptions().setCluster(getRedisServer().isRedisCluster()); + configure(executable); executable.setFiles("https://storage.googleapis.com/jrx/beers.csv"); executable.setHeader(true); executable.setThreads(3); @@ -137,8 +139,7 @@ void fileImportCSVMultiThreaded(TestInfo info) throws Exception { @Test void fileImportJSONL(TestInfo info) throws Exception { try (FileImport executable = new FileImport()) { - executable.getRedisClientOptions().setRedisURI(redisURI); - executable.getRedisClientOptions().setCluster(getRedisServer().isRedisCluster()); + configure(executable); executable.setFiles(BEERS_JSONL_URL); HsetBuilder hsetBuilder = new HsetBuilder(); hsetBuilder.setKeyspace(KEYSPACE); diff --git a/connectors/riot-file/src/test/java/com/redis/riot/file/JsonSerdeTests.java b/connectors/riot-file/src/test/java/com/redis/riot/file/JsonSerdeTests.java index 381451671..bdb36f3bf 100644 --- a/connectors/riot-file/src/test/java/com/redis/riot/file/JsonSerdeTests.java +++ b/connectors/riot-file/src/test/java/com/redis/riot/file/JsonSerdeTests.java @@ -26,6 +26,7 @@ import com.redis.spring.batch.KeyValue.DataType; import com.redis.spring.batch.gen.GeneratorItemReader; import com.redis.spring.batch.gen.ItemToKeyValueFunction; +import com.redis.spring.batch.reader.MemKeyValue; import com.redis.spring.batch.test.AbstractTestBase; @TestInstance(Lifecycle.PER_CLASS) @@ -39,14 +40,14 @@ class JsonSerdeTests { void setup() { mapper.configure(DeserializationFeature.USE_LONG_FOR_INTS, true); SimpleModule module = new SimpleModule(); - module.addDeserializer(KeyValue.class, new KeyValueDeserializer()); + module.addDeserializer(MemKeyValue.class, new MemKeyValueDeserializer()); mapper.registerModule(module); } @SuppressWarnings("unchecked") @Test void deserialize() throws JsonMappingException, JsonProcessingException { - KeyValue keyValue = mapper.readValue(timeseries, KeyValue.class); + MemKeyValue keyValue = mapper.readValue(timeseries, MemKeyValue.class); Assertions.assertEquals("gen:97", keyValue.getKey()); } @@ -55,7 +56,7 @@ void serialize() throws JsonProcessingException { String key = "ts:1"; long memoryUsage = DataSize.ofGigabytes(1).toBytes(); long ttl = Instant.now().toEpochMilli(); - KeyValue ts = new KeyValue<>(); + MemKeyValue ts = new MemKeyValue<>(); ts.setKey(key); ts.setMem(memoryUsage); ts.setTtl(ttl); @@ -77,11 +78,11 @@ void serde() throws Exception { GeneratorItemReader reader = new GeneratorItemReader(); reader.setMaxItemCount(17); reader.open(new ExecutionContext()); - List> items = AbstractTestBase.readAll(reader).stream() - .map(new ItemToKeyValueFunction()).collect(Collectors.toList()); + List> items = AbstractTestBase.readAll(reader).stream() + .map(new ItemToKeyValueFunction<>(MemKeyValue::new)).collect(Collectors.toList()); for (KeyValue item : items) { String json = mapper.writeValueAsString(item); - KeyValue result = mapper.readValue(json, KeyValue.class); + MemKeyValue result = mapper.readValue(json, MemKeyValue.class); Assertions.assertEquals(item, result); } } diff --git a/connectors/riot-file/src/test/java/com/redis/riot/file/XmlItemWriterTests.java b/connectors/riot-file/src/test/java/com/redis/riot/file/XmlItemWriterTests.java index a752bb2aa..a9a62740e 100644 --- a/connectors/riot-file/src/test/java/com/redis/riot/file/XmlItemWriterTests.java +++ b/connectors/riot-file/src/test/java/com/redis/riot/file/XmlItemWriterTests.java @@ -14,8 +14,8 @@ import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.dataformat.xml.XmlMapper; -import com.redis.riot.file.resource.XmlResourceItemWriter; -import com.redis.riot.file.resource.XmlResourceItemWriterBuilder; +import com.redis.riot.file.xml.XmlResourceItemWriter; +import com.redis.riot.file.xml.XmlResourceItemWriterBuilder; import com.redis.spring.batch.KeyValue; import com.redis.spring.batch.KeyValue.DataType; diff --git a/connectors/riot-redis/src/main/java/com/redis/riot/redis/GeneratorImport.java b/connectors/riot-redis/src/main/java/com/redis/riot/redis/GeneratorImport.java index 9c94cbc4f..3db361abe 100644 --- a/connectors/riot-redis/src/main/java/com/redis/riot/redis/GeneratorImport.java +++ b/connectors/riot-redis/src/main/java/com/redis/riot/redis/GeneratorImport.java @@ -46,8 +46,8 @@ protected Job job() { return jobBuilder().start(step(getName(), reader(), writer).processor(processor()).build()).build(); } - private ItemProcessor> processor() { - return new FunctionItemProcessor<>(new ItemToKeyValueFunction()); + private ItemProcessor> processor() { + return new FunctionItemProcessor<>(new ItemToKeyValueFunction<>(KeyValue::new)); } private GeneratorItemReader reader() { diff --git a/connectors/riot-redis/src/main/java/com/redis/riot/redis/Replication.java b/connectors/riot-redis/src/main/java/com/redis/riot/redis/Replication.java index d56a93281..8cb005789 100644 --- a/connectors/riot-redis/src/main/java/com/redis/riot/redis/Replication.java +++ b/connectors/riot-redis/src/main/java/com/redis/riot/redis/Replication.java @@ -37,6 +37,7 @@ import io.lettuce.core.AbstractRedisClient; import io.lettuce.core.ReadFrom; import io.lettuce.core.RedisException; +import io.lettuce.core.RedisURI; import io.lettuce.core.codec.ByteArrayCodec; public class Replication extends AbstractExport { @@ -73,19 +74,21 @@ public class Replication extends AbstractExport { private Duration idleTimeout = DEFAULT_IDLE_TIMEOUT; private int notificationQueueCapacity = DEFAULT_NOTIFICATION_QUEUE_CAPACITY; + private RedisURI targetRedisURI; private AbstractRedisClient targetRedisClient; @Override public void afterPropertiesSet() throws Exception { - targetRedisClient = targetRedisClientOptions.redisClient(); + targetRedisURI = targetRedisClientOptions.redisURI(); + targetRedisClient = targetRedisClientOptions.redisClient(targetRedisURI); super.afterPropertiesSet(); } @Override protected StandardEvaluationContext evaluationContext() { StandardEvaluationContext context = super.evaluationContext(); - context.setVariable(SOURCE_VAR, getRedisClientOptions().getRedisURI()); - context.setVariable(TARGET_VAR, targetRedisClientOptions.getRedisURI()); + context.setVariable(SOURCE_VAR, redisURI); + context.setVariable(TARGET_VAR, targetRedisURI); return context; } @@ -203,7 +206,7 @@ protected void configure(RedisItemReader reader) { } @SuppressWarnings({ "unchecked", "rawtypes" }) - private RedisItemReader> reader() { + private RedisItemReader> reader() { if (type == ReplicationType.STRUCT) { return RedisItemReader.struct(ByteArrayCodec.INSTANCE); } diff --git a/connectors/riot-redis/src/test/java/com/redis/riot/redis/ReplicationTests.java b/connectors/riot-redis/src/test/java/com/redis/riot/redis/ReplicationTests.java index cf3b17a89..31f0a3d17 100644 --- a/connectors/riot-redis/src/test/java/com/redis/riot/redis/ReplicationTests.java +++ b/connectors/riot-redis/src/test/java/com/redis/riot/redis/ReplicationTests.java @@ -1,5 +1,9 @@ package com.redis.riot.redis; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Executors; @@ -9,24 +13,23 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.slf4j.simple.SimpleLogger; -import org.springframework.batch.item.support.ListItemWriter; import org.testcontainers.shaded.org.bouncycastle.util.encoders.Hex; import com.redis.lettucemod.api.StatefulRedisModulesConnection; import com.redis.lettucemod.util.RedisModulesUtils; import com.redis.riot.core.KeyValueProcessorOptions; -import com.redis.riot.core.PredicateItemProcessor; import com.redis.riot.core.RedisClientOptions; import com.redis.riot.core.RiotUtils; +import com.redis.riot.core.ScanSizeEstimator; +import com.redis.riot.core.SlotRange; import com.redis.riot.redis.Replication.LoggingWriteListener; -import com.redis.spring.batch.KeyValue; -import com.redis.spring.batch.RedisItemReader; -import com.redis.spring.batch.common.FlushingStepBuilder; +import com.redis.spring.batch.KeyValue.DataType; +import com.redis.spring.batch.gen.GeneratorItemReader; +import com.redis.spring.batch.gen.Item; import com.redis.spring.batch.test.AbstractTargetTestBase; -import com.redis.spring.batch.util.Predicates; +import com.redis.spring.batch.test.KeyspaceComparison; import com.redis.testcontainers.RedisServer; -import io.lettuce.core.RedisURI; import io.lettuce.core.cluster.SlotHash; import io.lettuce.core.codec.ByteArrayCodec; @@ -63,7 +66,7 @@ protected void execute(Replication replication, TestInfo info) throws Exception private RedisClientOptions redisOptions(RedisServer redis) { RedisClientOptions options = new RedisClientOptions(); - options.setRedisURI(RedisURI.create(redis.getRedisURI())); + options.getUriOptions().setUri(redis.getRedisURI()); options.setCluster(redis.isRedisCluster()); return options; } @@ -74,7 +77,9 @@ void replication(TestInfo info) throws Throwable { Assertions.assertTrue(redisCommands.dbsize() > 0); Replication replication = new Replication(); execute(replication, info); - Assertions.assertTrue(compare(info).isOk()); + KeyspaceComparison comparison = compare(info); + Assertions.assertFalse(comparison.getAll().isEmpty()); + Assertions.assertEquals(Collections.emptyList(), comparison.mismatches()); } @Test @@ -150,7 +155,7 @@ void binaryKeyLiveReplication(TestInfo info) throws Exception { .connection(targetRedisClient, ByteArrayCodec.INSTANCE); enableKeyspaceNotifications(); Executors.newSingleThreadExecutor().execute(() -> { - awaitPubSub(); + awaitUntilSubscribers(); connection.sync().set(key, value); }); Replication replication = new Replication(); @@ -160,19 +165,30 @@ void binaryKeyLiveReplication(TestInfo info) throws Exception { Assertions.assertArrayEquals(connection.sync().get(key), targetConnection.sync().get(key)); } + @Test + void estimateScanSize(TestInfo info) throws Exception { + GeneratorItemReader gen = generator(3000, Item.Type.HASH, Item.Type.STRING); + generate(info, gen); + long expectedCount = redisCommands.dbsize(); + ScanSizeEstimator estimator = new ScanSizeEstimator(redisClient); + estimator.setKeyPattern(GeneratorItemReader.DEFAULT_KEYSPACE + ":*"); + estimator.setSamples(300); + assertEquals(expectedCount, estimator.getAsLong(), expectedCount / 10); + estimator.setKeyType(DataType.HASH.getString()); + assertEquals(expectedCount / 2, estimator.getAsLong(), expectedCount / 10); + } + @Test void filterKeySlot(TestInfo info) throws Exception { enableKeyspaceNotifications(); - RedisItemReader> reader = structReader(info); - live(reader); - reader.setKeyProcessor(new PredicateItemProcessor<>(Predicates.slotRange(0, 8000))); - ListItemWriter> writer = new ListItemWriter<>(); + Replication replication = new Replication(); + replication.setMode(ReplicationMode.LIVE); + replication.setCompareMode(CompareMode.NONE); + replication.getReaderOptions().getKeyFilterOptions().setSlots(Arrays.asList(new SlotRange(0, 8000))); generateAsync(info, generator(100)); - FlushingStepBuilder, KeyValue> step = flushingStep(info, reader, - writer); - run(job(info).start(step.build()).build()); - Assertions.assertTrue(writer.getWrittenItems().stream().map(KeyValue::getKey).map(SlotHash::getSlot) - .allMatch(between(0, 8000))); + execute(replication, info); + awaitUntilNoSubscribers(); + Assertions.assertTrue(targetRedisCommands.keys("*").stream().map(SlotHash::getSlot).allMatch(between(0, 8000))); } private Predicate between(int start, int end) { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/AbstractMapExport.java b/core/riot-core/src/main/java/com/redis/riot/core/AbstractMapExport.java index 139fb989a..f7c3916f4 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/AbstractMapExport.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/AbstractMapExport.java @@ -12,6 +12,7 @@ import com.redis.riot.core.function.StructToMapFunction; import com.redis.spring.batch.KeyValue; import com.redis.spring.batch.RedisItemReader; +import com.redis.spring.batch.reader.MemKeyValue; import io.lettuce.core.codec.StringCodec; @@ -30,8 +31,8 @@ protected Job job() { return jobBuilder().start(step(getName(), reader(), writer()).processor(processor()).build()).build(); } - protected RedisItemReader> reader() { - RedisItemReader> reader = RedisItemReader.struct(); + protected RedisItemReader> reader() { + RedisItemReader> reader = RedisItemReader.struct(); configure(reader); return reader; } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/AbstractMapImport.java b/core/riot-core/src/main/java/com/redis/riot/core/AbstractMapImport.java index 61315951f..53b726bf4 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/AbstractMapImport.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/AbstractMapImport.java @@ -11,24 +11,24 @@ import org.springframework.util.Assert; import com.redis.spring.batch.RedisItemWriter; -import com.redis.spring.batch.operation.Operation; +import com.redis.spring.batch.writer.WriteOperation; public abstract class AbstractMapImport extends AbstractImport { private EvaluationContextOptions evaluationContextOptions = new EvaluationContextOptions(); private ImportProcessorOptions processorOptions = new ImportProcessorOptions(); - private List, Object>> operations; + private List>> operations; @SuppressWarnings("unchecked") - public void setOperations(Operation, Object>... operations) { + public void setOperations(WriteOperation>... operations) { setOperations(Arrays.asList(operations)); } - public List, Object>> getOperations() { + public List>> getOperations() { return operations; } - public void setOperations(List, Object>> operations) { + public void setOperations(List>> operations) { this.operations = operations; } @@ -53,7 +53,7 @@ protected ItemWriter> writer() { return RiotUtils.writer(operations.stream().map(this::writer).collect(Collectors.toList())); } - private ItemWriter writer(Operation operation) { + private ItemWriter writer(WriteOperation operation) { RedisItemWriter writer = RedisItemWriter.operation(operation); configure(writer); return writer; diff --git a/core/riot-core/src/main/java/com/redis/riot/core/AbstractRedisCallable.java b/core/riot-core/src/main/java/com/redis/riot/core/AbstractRedisCallable.java index fa69d4cec..e8b9e6f8f 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/AbstractRedisCallable.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/AbstractRedisCallable.java @@ -9,15 +9,16 @@ import com.redis.spring.batch.RedisItemWriter; import io.lettuce.core.AbstractRedisClient; +import io.lettuce.core.RedisURI; public abstract class AbstractRedisCallable extends AbstractRiotCallable { private static final String CONTEXT_VAR_REDIS = "redis"; private EvaluationContextOptions evaluationContextOptions = new EvaluationContextOptions(); - private RedisClientOptions redisClientOptions = new RedisClientOptions(); + protected RedisURI redisURI; private AbstractRedisClient redisClient; private StatefulRedisModulesConnection redisConnection; protected RedisModulesCommands redisCommands; @@ -25,7 +26,8 @@ public abstract class AbstractRedisCallable extends AbstractRiotCallable { @Override public void afterPropertiesSet() throws Exception { - redisClient = redisClientOptions.redisClient(); + redisURI = redisClientOptions.redisURI(); + redisClient = redisClientOptions.redisClient(redisURI); redisConnection = RedisModulesUtils.connection(redisClient); redisCommands = redisConnection.sync(); evaluationContext = evaluationContext(); @@ -62,7 +64,7 @@ public void setEvaluationContextOptions(EvaluationContextOptions spelProcessorOp protected void configure(RedisItemReader reader) { reader.setClient(redisClient); - reader.setDatabase(redisClientOptions.getRedisURI().getDatabase()); + reader.setDatabase(redisURI.getDatabase()); } protected void configure(RedisItemWriter writer) { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/AbstractRiotCallable.java b/core/riot-core/src/main/java/com/redis/riot/core/AbstractRiotCallable.java index 480f05aa8..11942d6b5 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/AbstractRiotCallable.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/AbstractRiotCallable.java @@ -22,6 +22,7 @@ import org.springframework.batch.item.support.SynchronizedItemStreamReader; import org.springframework.beans.factory.InitializingBean; import org.springframework.retry.policy.MaxAttemptsRetryPolicy; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.util.ClassUtils; import com.redis.spring.batch.RedisItemReader; @@ -98,7 +99,12 @@ protected FaultTolerantStepBuilder step(String name, ItemReader } if (isMultiThreaded()) { builder.reader(synchronize(reader)); - builder.taskExecutor(JobFactory.threadPoolTaskExecutor(threads)); + ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor(); + taskExecutor.setMaxPoolSize(threads); + taskExecutor.setCorePoolSize(threads); + taskExecutor.setQueueCapacity(threads); + taskExecutor.initialize(); + builder.taskExecutor(taskExecutor); } else { builder.reader(reader); } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/KeyFilter.java b/core/riot-core/src/main/java/com/redis/riot/core/KeyFilter.java new file mode 100644 index 000000000..88e1f347c --- /dev/null +++ b/core/riot-core/src/main/java/com/redis/riot/core/KeyFilter.java @@ -0,0 +1,102 @@ +package com.redis.riot.core; + +import java.util.List; +import java.util.function.Function; +import java.util.function.IntPredicate; +import java.util.function.Predicate; + +import org.springframework.beans.factory.InitializingBean; + +import com.hrakaroo.glob.GlobPattern; +import com.redis.spring.batch.util.BatchUtils; + +import io.lettuce.core.cluster.SlotHash; +import io.lettuce.core.codec.RedisCodec; + +public class KeyFilter implements Predicate, InitializingBean { + + private final RedisCodec codec; + private final Function toString; + private KeyFilterOptions options = new KeyFilterOptions(); + private Predicate predicate; + + public KeyFilter(RedisCodec codec) { + this.codec = codec; + this.toString = BatchUtils.toStringKeyFunction(codec); + } + + @Override + public void afterPropertiesSet() { + this.predicate = predicate(); + } + + @Override + public boolean test(K t) { + return predicate.test(t); + } + + public Predicate predicate() { + if (options.isEmptyIncludes() && options.isEmptyExcludes()) { + return slotsPredicate(); + } + Predicate stringGlobPredicate = stringGlobPredicate(); + Predicate globPredicate = k -> stringGlobPredicate.test(toString.apply(k)); + if (options.isEmptySlots()) { + return globPredicate; + } + return slotsPredicate().and(globPredicate); + } + + private Predicate slotsPredicate() { + return options.getSlots().stream().map(r -> slotRangePredicate(r.getStart(), r.getEnd())).reduce(k -> false, + Predicate::or); + } + + private Predicate stringGlobPredicate() { + if (options.isEmptyIncludes()) { + return excludesPredicate(); + } + if (options.isEmptyExcludes()) { + return includesPredicate(); + } + return includesPredicate().and(excludesPredicate()); + } + + private Predicate includesPredicate() { + return globPredicate(options.getIncludes()); + } + + private Predicate excludesPredicate() { + return globPredicate(options.getExcludes()).negate(); + } + + private Predicate globPredicate(List patterns) { + return patterns.stream().map(this::globPredicate).reduce(k -> false, Predicate::or); + } + + private Predicate globPredicate(String glob) { + return GlobPattern.compile(glob)::matches; + } + + public static IntPredicate between(int start, int end) { + return i -> i >= start && i <= end; + } + + private int slot(K key) { + return SlotHash.getSlot(codec.encodeKey(key)); + } + + public Predicate slotRangePredicate(int start, int end) { + IntPredicate rangePredicate = between(start, end); + return k -> rangePredicate.test(slot(k)); + } + + public KeyFilterOptions getOptions() { + return options; + } + + public void setOptions(KeyFilterOptions options) { + this.options = options; + } + +} diff --git a/core/riot-core/src/main/java/com/redis/riot/core/KeyFilterOptions.java b/core/riot-core/src/main/java/com/redis/riot/core/KeyFilterOptions.java index f3b6bf102..e31f9eb73 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/KeyFilterOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/KeyFilterOptions.java @@ -1,17 +1,9 @@ package com.redis.riot.core; import java.util.List; -import java.util.function.Predicate; -import java.util.stream.Stream; -import org.springframework.batch.item.ItemProcessor; import org.springframework.util.CollectionUtils; -import com.redis.spring.batch.util.BatchUtils; -import com.redis.spring.batch.util.Predicates; - -import io.lettuce.core.codec.RedisCodec; - public class KeyFilterOptions { private List includes; @@ -43,39 +35,19 @@ public void setSlots(List ranges) { } public boolean isEmpty() { - return CollectionUtils.isEmpty(includes) && CollectionUtils.isEmpty(excludes) && CollectionUtils.isEmpty(slots); - } - - public Predicate predicate(RedisCodec codec) { - return slotsPredicate(codec).and(globPredicate(codec)); - } - - private Predicate slotsPredicate(RedisCodec codec) { - if (CollectionUtils.isEmpty(slots)) { - return Predicates.isTrue(); - } - Stream> predicates = slots.stream() - .map(r -> Predicates.slotRange(codec, r.getStart(), r.getEnd())); - return Predicates.or(predicates); + return isEmptyIncludes() && isEmptyExcludes() && isEmptySlots(); } - private Predicate globPredicate(RedisCodec codec) { - return Predicates.map(BatchUtils.toStringKeyFunction(codec), globPredicate()); + public boolean isEmptySlots() { + return CollectionUtils.isEmpty(slots); } - private Predicate globPredicate() { - Predicate include = RiotUtils.globPredicate(includes); - if (CollectionUtils.isEmpty(excludes)) { - return include; - } - return include.and(RiotUtils.globPredicate(excludes).negate()); + public boolean isEmptyIncludes() { + return CollectionUtils.isEmpty(includes); } - public ItemProcessor processor(RedisCodec codec) { - if (isEmpty()) { - return null; - } - return new PredicateItemProcessor<>(predicate(codec)); + public boolean isEmptyExcludes() { + return CollectionUtils.isEmpty(excludes); } } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/RedisClientOptions.java b/core/riot-core/src/main/java/com/redis/riot/core/RedisClientOptions.java index e249c8193..37459ebfd 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/RedisClientOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/RedisClientOptions.java @@ -6,55 +6,52 @@ import io.lettuce.core.AbstractRedisClient; import io.lettuce.core.ClientOptions; import io.lettuce.core.RedisURI; +import io.lettuce.core.SslOptions; import io.lettuce.core.cluster.ClusterClientOptions; -import io.lettuce.core.resource.ClientResources; +import io.lettuce.core.protocol.ProtocolVersion; public class RedisClientOptions { - public static final String DEFAULT_REDIS_HOST = "127.0.0.1"; - public static final int DEFAULT_REDIS_PORT = RedisURI.DEFAULT_REDIS_PORT; - public static final RedisURI DEFAULT_REDIS_URI = RedisURI.create(DEFAULT_REDIS_HOST, DEFAULT_REDIS_PORT); + public static final ProtocolVersion DEFAULT_PROTOCOL_VERSION = ClientOptions.DEFAULT_PROTOCOL_VERSION; + public static final boolean DEFAULT_AUTO_RECONNECT = ClientOptions.DEFAULT_AUTO_RECONNECT; - private RedisURI redisURI = DEFAULT_REDIS_URI; + private RedisUriOptions uriOptions = new RedisUriOptions(); private boolean cluster; - private ClientOptions options; - private ClientResources resources; + private boolean autoReconnect = DEFAULT_AUTO_RECONNECT; + private ProtocolVersion protocolVersion = DEFAULT_PROTOCOL_VERSION; + private SslOptions sslOptions = SslOptions.builder().build(); - public AbstractRedisClient redisClient() { + public RedisURI redisURI() { + return uriOptions.redisURI(); + } + + public AbstractRedisClient redisClient(RedisURI redisURI) { if (cluster) { - RedisModulesClusterClient client = clusterClient(); - if (options != null) { - client.setOptions((ClusterClientOptions) options); - } + RedisModulesClusterClient client = RedisModulesClusterClient.create(redisURI); + ClusterClientOptions.Builder options = ClusterClientOptions.builder(); + configure(options); + client.setOptions(options.build()); return client; } - RedisModulesClient client = client(); - if (options != null) { - client.setOptions(options); - } + RedisModulesClient client = RedisModulesClient.create(redisURI); + ClientOptions.Builder options = ClientOptions.builder(); + configure(options); + client.setOptions(options.build()); return client; } - private RedisModulesClient client() { - if (resources == null) { - return RedisModulesClient.create(redisURI); - } - return RedisModulesClient.create(resources, redisURI); + private void configure(ClientOptions.Builder builder) { + builder.autoReconnect(autoReconnect); + builder.protocolVersion(protocolVersion); + builder.sslOptions(sslOptions); } - private RedisModulesClusterClient clusterClient() { - if (resources == null) { - return RedisModulesClusterClient.create(redisURI); - } - return RedisModulesClusterClient.create(resources, redisURI); + public RedisUriOptions getUriOptions() { + return uriOptions; } - public RedisURI getRedisURI() { - return redisURI; - } - - public void setRedisURI(RedisURI uri) { - this.redisURI = uri; + public void setUriOptions(RedisUriOptions uriOptions) { + this.uriOptions = uriOptions; } public boolean isCluster() { @@ -65,20 +62,28 @@ public void setCluster(boolean cluster) { this.cluster = cluster; } - public ClientOptions getOptions() { - return options; + public boolean isAutoReconnect() { + return autoReconnect; + } + + public void setAutoReconnect(boolean autoReconnect) { + this.autoReconnect = autoReconnect; + } + + public ProtocolVersion getProtocolVersion() { + return protocolVersion; } - public void setOptions(ClientOptions options) { - this.options = options; + public void setProtocolVersion(ProtocolVersion protocolVersion) { + this.protocolVersion = protocolVersion; } - public ClientResources getResources() { - return resources; + public SslOptions getSslOptions() { + return sslOptions; } - public void setResources(ClientResources resources) { - this.resources = resources; + public void setSslOptions(SslOptions options) { + this.sslOptions = options; } } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/RedisReaderOptions.java b/core/riot-core/src/main/java/com/redis/riot/core/RedisReaderOptions.java index 65549385c..13beb574b 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/RedisReaderOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/RedisReaderOptions.java @@ -2,13 +2,15 @@ import java.time.Duration; +import org.springframework.batch.item.ItemProcessor; import org.springframework.util.unit.DataSize; import com.redis.spring.batch.RedisItemReader; -import com.redis.spring.batch.operation.KeyValueRead; import com.redis.spring.batch.reader.AbstractPollableItemReader; +import com.redis.spring.batch.reader.MemKeyValueRead; import io.lettuce.core.ReadFrom; +import io.lettuce.core.codec.RedisCodec; public class RedisReaderOptions { @@ -17,8 +19,8 @@ public class RedisReaderOptions { public static final int DEFAULT_THREADS = RedisItemReader.DEFAULT_THREADS; public static final int DEFAULT_CHUNK_SIZE = RedisItemReader.DEFAULT_CHUNK_SIZE; public static final int DEFAULT_POOL_SIZE = RedisItemReader.DEFAULT_POOL_SIZE; - public static final DataSize DEFAULT_MEMORY_USAGE_LIMIT = KeyValueRead.DEFAULT_MEM_USAGE_LIMIT; - public static final int DEFAULT_MEMORY_USAGE_SAMPLES = KeyValueRead.DEFAULT_MEM_USAGE_SAMPLES; + public static final DataSize DEFAULT_MEMORY_USAGE_LIMIT = MemKeyValueRead.DEFAULT_MEM_USAGE_LIMIT; + public static final int DEFAULT_MEMORY_USAGE_SAMPLES = MemKeyValueRead.DEFAULT_MEM_USAGE_SAMPLES; public static final long DEFAULT_SCAN_COUNT = 1000; private String keyPattern; @@ -43,13 +45,24 @@ public void configure(RedisItemReader reader) { reader.setReadFrom(readFrom); reader.setScanCount(scanCount); reader.setThreads(threads); - if (reader.getOperation() instanceof KeyValueRead) { - KeyValueRead operation = (KeyValueRead) reader.getOperation(); + if (reader.getOperation() instanceof MemKeyValueRead) { + MemKeyValueRead operation = (MemKeyValueRead) reader.getOperation(); operation.setMemUsageLimit(memoryUsageLimit); operation.setMemUsageSamples(memoryUsageSamples); } reader.setPoolSize(poolSize); - reader.setKeyProcessor(keyFilterOptions.processor(reader.getCodec())); + reader.setKeyProcessor(keyProcessor(reader.getCodec())); + } + + private ItemProcessor keyProcessor(RedisCodec codec) { + if (keyFilterOptions.isEmpty()) { + return null; + } + KeyFilter filter = new KeyFilter<>(codec); + filter.setOptions(keyFilterOptions); + filter.afterPropertiesSet(); + return new PredicateItemProcessor<>(filter); + } public String getKeyPattern() { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/RedisUriOptions.java b/core/riot-core/src/main/java/com/redis/riot/core/RedisUriOptions.java new file mode 100644 index 000000000..1f039f37b --- /dev/null +++ b/core/riot-core/src/main/java/com/redis/riot/core/RedisUriOptions.java @@ -0,0 +1,153 @@ +package com.redis.riot.core; + +import java.time.Duration; + +import org.springframework.util.ObjectUtils; +import org.springframework.util.StringUtils; + +import io.lettuce.core.RedisURI; +import io.lettuce.core.SslVerifyMode; + +public class RedisUriOptions { + + public static final String DEFAULT_HOST = "127.0.0.1"; + public static final int DEFAULT_PORT = RedisURI.DEFAULT_REDIS_PORT; + + private String uri; + private String host = DEFAULT_HOST; + private int port = DEFAULT_PORT; + private String socket; + private String username; + private char[] password; + private long timeout; + private int database; + private String clientName = RiotVersion.riotVersion(); + private boolean tls; + private boolean insecure; + + public RedisURI redisURI() { + RedisURI.Builder builder = redisURIBuilder(); + if (!ObjectUtils.isEmpty(password)) { + if (StringUtils.hasLength(username)) { + builder.withAuthentication(username, password); + } else { + builder.withPassword(password); + } + } + if (StringUtils.hasLength(clientName)) { + builder.withClientName(clientName); + } + if (database > 0) { + builder.withDatabase(database); + } + if (tls) { + builder.withSsl(tls); + } + if (insecure) { + builder.withVerifyPeer(SslVerifyMode.NONE); + } + if (timeout > 0) { + builder.withTimeout(Duration.ofSeconds(timeout)); + } + return builder.build(); + } + + private RedisURI.Builder redisURIBuilder() { + if (StringUtils.hasLength(uri)) { + return RedisURI.builder(RedisURI.create(uri)); + } + if (StringUtils.hasLength(socket)) { + return RedisURI.Builder.socket(socket); + } + return RedisURI.Builder.redis(host, port); + } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getSocket() { + return socket; + } + + public void setSocket(String socket) { + this.socket = socket; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public char[] getPassword() { + return password; + } + + public void setPassword(char[] password) { + this.password = password; + } + + public long getTimeout() { + return timeout; + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + public int getDatabase() { + return database; + } + + public void setDatabase(int database) { + this.database = database; + } + + public String getClientName() { + return clientName; + } + + public void setClientName(String clientName) { + this.clientName = clientName; + } + + public boolean isTls() { + return tls; + } + + public void setTls(boolean tls) { + this.tls = tls; + } + + public boolean isInsecure() { + return insecure; + } + + public void setInsecure(boolean insecure) { + this.insecure = insecure; + } + +} diff --git a/core/riot-core/src/main/java/com/redis/riot/core/RedisWriterOptions.java b/core/riot-core/src/main/java/com/redis/riot/core/RedisWriterOptions.java index cabbaa3ef..592ca093d 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/RedisWriterOptions.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/RedisWriterOptions.java @@ -3,8 +3,8 @@ import java.time.Duration; import com.redis.spring.batch.RedisItemWriter; -import com.redis.spring.batch.operation.KeyValueWrite; -import com.redis.spring.batch.operation.KeyValueWrite.WriteMode; +import com.redis.spring.batch.writer.KeyValueWrite; +import com.redis.spring.batch.writer.KeyValueWrite.WriteMode; public class RedisWriterOptions { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/RiotUtils.java b/core/riot-core/src/main/java/com/redis/riot/core/RiotUtils.java index a91111f15..56981d1c0 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/RiotUtils.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/RiotUtils.java @@ -13,7 +13,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.List; import java.util.Objects; import java.util.function.Predicate; @@ -25,9 +24,6 @@ import org.springframework.expression.Expression; import org.springframework.expression.common.TemplateParserContext; import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.util.CollectionUtils; - -import com.redis.spring.batch.util.Predicates; public abstract class RiotUtils { @@ -50,13 +46,6 @@ public static Predicate predicate(EvaluationContext context, Expression e return t -> expression.getValue(context, t, Boolean.class); } - public static Predicate globPredicate(List patterns) { - if (CollectionUtils.isEmpty(patterns)) { - return Predicates.isTrue(); - } - return Predicates.or(patterns.stream().map(Predicates::glob)); - } - public static ItemProcessor processor(ItemProcessor... processors) { return processor(new ArrayList<>(Arrays.asList(processors))); } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/ScanSizeEstimator.java b/core/riot-core/src/main/java/com/redis/riot/core/ScanSizeEstimator.java new file mode 100644 index 000000000..515fe03aa --- /dev/null +++ b/core/riot-core/src/main/java/com/redis/riot/core/ScanSizeEstimator.java @@ -0,0 +1,136 @@ +package com.redis.riot.core; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.function.LongSupplier; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.springframework.util.StringUtils; + +import com.hrakaroo.glob.GlobPattern; +import com.hrakaroo.glob.MatchingEngine; +import com.redis.lettucemod.api.StatefulRedisModulesConnection; +import com.redis.lettucemod.api.async.RedisModulesAsyncCommands; +import com.redis.lettucemod.util.RedisModulesUtils; +import com.redis.spring.batch.OperationExecutor; + +import io.lettuce.core.AbstractRedisClient; +import io.lettuce.core.RedisFuture; + +public class ScanSizeEstimator implements LongSupplier { + + public static final long UNKNOWN_SIZE = -1; + public static final int DEFAULT_SAMPLES = 100; + + private final AbstractRedisClient client; + + private int samples = DEFAULT_SAMPLES; + private String keyPattern; + private String keyType; + + public ScanSizeEstimator(AbstractRedisClient client) { + this.client = client; + } + + public String getKeyPattern() { + return keyPattern; + } + + public void setKeyPattern(String keyPattern) { + this.keyPattern = keyPattern; + } + + public String getKeyType() { + return keyType; + } + + public void setKeyType(String keyType) { + this.keyType = keyType; + } + + public int getSamples() { + return samples; + } + + public void setSamples(int samples) { + this.samples = samples; + } + + /** + * Estimates the number of keys that match the given pattern and type. + * + * @return Estimated number of keys matching the given pattern and type. Returns + * null if database is empty or any error occurs + * @throws IOException if script execution exception happens during estimation + */ + @Override + public long getAsLong() { + try (StatefulRedisModulesConnection connection = RedisModulesUtils.connection(client)) { + Long dbsize = connection.sync().dbsize(); + if (dbsize == null) { + return UNKNOWN_SIZE; + } + if (!StringUtils.hasLength(keyPattern) && !StringUtils.hasLength(keyType)) { + return dbsize; + } + RedisModulesAsyncCommands commands = connection.async(); + try { + connection.setAutoFlushCommands(false); + List> keyFutures = new ArrayList<>(); + for (int index = 0; index < samples; index++) { + keyFutures.add(commands.randomkey()); + } + connection.flushCommands(); + List keys = OperationExecutor.getAll(connection.getTimeout(), keyFutures); + List> typeFutures = keys.stream().map(commands::type).collect(Collectors.toList()); + connection.flushCommands(); + List types = OperationExecutor.getAll(connection.getTimeout(), typeFutures); + Predicate matchPredicate = matchPredicate(); + Predicate typePredicate = typePredicate(); + int total = 0; + int matchCount = 0; + Iterator keyIterator = keys.iterator(); + Iterator typeIterator = types.iterator(); + while (keyIterator.hasNext()) { + String key = keyIterator.next(); + if (!typeIterator.hasNext()) { + throw new IllegalStateException("Could not find type for key " + key); + } + String type = typeIterator.next(); + total++; + if (matchPredicate.test(key) && typePredicate.test(type)) { + matchCount++; + } + } + double matchRate = total == 0 ? 0 : (double) matchCount / total; + return Math.round(dbsize * matchRate); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (Exception e) { + // Ignore and return unknown size + } finally { + connection.setAutoFlushCommands(true); + } + } + return UNKNOWN_SIZE; + } + + private Predicate matchPredicate() { + if (StringUtils.hasLength(keyPattern)) { + MatchingEngine engine = GlobPattern.compile(keyPattern); + return engine::matches; + } + return s -> true; + } + + private Predicate typePredicate() { + if (StringUtils.hasLength(keyType)) { + return keyType::equalsIgnoreCase; + } + return s -> true; + } + +} diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/AbstractMapOperationBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/AbstractMapOperationBuilder.java index 3f77217c3..8a4dea0ea 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/AbstractMapOperationBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/AbstractMapOperationBuilder.java @@ -9,8 +9,8 @@ import com.redis.riot.core.function.FieldExtractorFactory; import com.redis.riot.core.function.IdFunctionBuilder; -import com.redis.spring.batch.operation.AbstractKeyOperation; -import com.redis.spring.batch.operation.Operation; +import com.redis.spring.batch.writer.AbstractKeyOperation; +import com.redis.spring.batch.writer.WriteOperation; public abstract class AbstractMapOperationBuilder { @@ -54,7 +54,7 @@ protected Function, String> idFunction(String prefix, List, Object> build() { + public WriteOperation> build() { Function, String> keyFunction = idFunction(keyspace, keyFields); return operation(keyFunction); } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/CompositeBatchWriteOperation.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/CompositeBatchWriteOperation.java index 847dd6b43..ae1b7ff3a 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/CompositeBatchWriteOperation.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/CompositeBatchWriteOperation.java @@ -2,23 +2,23 @@ import java.util.List; -import com.redis.spring.batch.operation.Operation; +import com.redis.spring.batch.writer.WriteOperation; import io.lettuce.core.RedisFuture; import io.lettuce.core.api.async.BaseRedisAsyncCommands; -public class CompositeBatchWriteOperation implements Operation { +public class CompositeBatchWriteOperation implements WriteOperation { - private final List> delegates; + private final List> delegates; - public CompositeBatchWriteOperation(List> delegates) { + public CompositeBatchWriteOperation(List> delegates) { this.delegates = delegates; } @Override public void execute(BaseRedisAsyncCommands commands, Iterable inputs, List> outputs) { - for (Operation delegate : delegates) { + for (WriteOperation delegate : delegates) { delegate.execute(commands, inputs, outputs); } } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/DelBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/DelBuilder.java index 713611395..9f1e0d6b2 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/DelBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/DelBuilder.java @@ -3,7 +3,7 @@ import java.util.Map; import java.util.function.Function; -import com.redis.spring.batch.operation.Del; +import com.redis.spring.batch.writer.Del; public class DelBuilder extends AbstractMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/ExpireAtBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/ExpireAtBuilder.java index 70a80accf..cd0bc4f2d 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/ExpireAtBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/ExpireAtBuilder.java @@ -3,7 +3,7 @@ import java.util.Map; import java.util.function.Function; -import com.redis.spring.batch.operation.ExpireAt; +import com.redis.spring.batch.writer.ExpireAt; public class ExpireAtBuilder extends AbstractMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/ExpireBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/ExpireBuilder.java index 02abc2069..0810b43df 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/ExpireBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/ExpireBuilder.java @@ -5,7 +5,7 @@ import java.util.function.Function; import java.util.function.ToLongFunction; -import com.redis.spring.batch.operation.Expire; +import com.redis.spring.batch.writer.Expire; public class ExpireBuilder extends AbstractMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/GeoaddBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/GeoaddBuilder.java index 38505e6ed..744d5b5bc 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/GeoaddBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/GeoaddBuilder.java @@ -4,8 +4,8 @@ import java.util.function.Function; import java.util.function.ToDoubleFunction; -import com.redis.spring.batch.operation.Geoadd; import com.redis.spring.batch.util.ToGeoValueFunction; +import com.redis.spring.batch.writer.Geoadd; public class GeoaddBuilder extends AbstractCollectionMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/HsetBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/HsetBuilder.java index 1d8f545c7..3291923b3 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/HsetBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/HsetBuilder.java @@ -3,7 +3,7 @@ import java.util.Map; import java.util.function.Function; -import com.redis.spring.batch.operation.Hset; +import com.redis.spring.batch.writer.Hset; public class HsetBuilder extends AbstractFilterMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/JsonSetBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/JsonSetBuilder.java index 9dd46a95b..153e4ec3b 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/JsonSetBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/JsonSetBuilder.java @@ -8,7 +8,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import com.redis.spring.batch.operation.JsonSet; +import com.redis.spring.batch.writer.JsonSet; public class JsonSetBuilder extends AbstractMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/LpushBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/LpushBuilder.java index 51371273f..dd2a1e5d5 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/LpushBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/LpushBuilder.java @@ -3,7 +3,7 @@ import java.util.Map; import java.util.function.Function; -import com.redis.spring.batch.operation.Lpush; +import com.redis.spring.batch.writer.Lpush; public class LpushBuilder extends AbstractCollectionMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/RpushBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/RpushBuilder.java index bd3b7612c..48c7a4f78 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/RpushBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/RpushBuilder.java @@ -3,7 +3,7 @@ import java.util.Map; import java.util.function.Function; -import com.redis.spring.batch.operation.Rpush; +import com.redis.spring.batch.writer.Rpush; public class RpushBuilder extends AbstractCollectionMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/SaddBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/SaddBuilder.java index 9e3103c40..52e0389b4 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/SaddBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/SaddBuilder.java @@ -3,7 +3,7 @@ import java.util.Map; import java.util.function.Function; -import com.redis.spring.batch.operation.Sadd; +import com.redis.spring.batch.writer.Sadd; public class SaddBuilder extends AbstractCollectionMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/SetBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/SetBuilder.java index 265f11956..1da86b284 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/SetBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/SetBuilder.java @@ -8,7 +8,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.redis.riot.core.function.ObjectMapperFunction; -import com.redis.spring.batch.operation.Set; +import com.redis.spring.batch.writer.Set; public class SetBuilder extends AbstractMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/SugaddBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/SugaddBuilder.java index f348ac8af..2bf4d120f 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/SugaddBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/SugaddBuilder.java @@ -5,8 +5,8 @@ import java.util.function.ToDoubleFunction; import com.redis.lettucemod.search.Suggestion; -import com.redis.spring.batch.operation.Sugadd; import com.redis.spring.batch.util.ToSuggestionFunction; +import com.redis.spring.batch.writer.Sugadd; public class SugaddBuilder extends AbstractMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/TsAddBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/TsAddBuilder.java index de43dca4e..e743600b9 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/TsAddBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/TsAddBuilder.java @@ -15,8 +15,8 @@ import com.redis.lettucemod.timeseries.DuplicatePolicy; import com.redis.lettucemod.timeseries.Label; import com.redis.lettucemod.timeseries.Sample; -import com.redis.spring.batch.operation.TsAdd; import com.redis.spring.batch.util.ToSampleFunction; +import com.redis.spring.batch.writer.TsAdd; public class TsAddBuilder extends AbstractMapOperationBuilder { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/XaddSupplier.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/XaddSupplier.java index daefced77..0e3d2f389 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/XaddSupplier.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/XaddSupplier.java @@ -3,7 +3,7 @@ import java.util.Map; import java.util.function.Function; -import com.redis.spring.batch.operation.Xadd; +import com.redis.spring.batch.writer.Xadd; import io.lettuce.core.XAddArgs; diff --git a/core/riot-core/src/main/java/com/redis/riot/core/operation/ZaddSupplier.java b/core/riot-core/src/main/java/com/redis/riot/core/operation/ZaddSupplier.java index 8760ca9d8..3f6794101 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/operation/ZaddSupplier.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/operation/ZaddSupplier.java @@ -4,8 +4,8 @@ import java.util.function.Function; import java.util.function.ToDoubleFunction; -import com.redis.spring.batch.operation.Zadd; import com.redis.spring.batch.util.ToScoredValueFunction; +import com.redis.spring.batch.writer.Zadd; public class ZaddSupplier extends AbstractCollectionMapOperationBuilder { diff --git a/core/riot-core/src/test/java/com/redis/riot/core/ProcessorTests.java b/core/riot-core/src/test/java/com/redis/riot/core/ProcessorTests.java index 64a9ce0a4..9fddf4098 100644 --- a/core/riot-core/src/test/java/com/redis/riot/core/ProcessorTests.java +++ b/core/riot-core/src/test/java/com/redis/riot/core/ProcessorTests.java @@ -1,10 +1,13 @@ package com.redis.riot.core; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.function.Predicate; @@ -14,15 +17,25 @@ import org.springframework.expression.Expression; import org.springframework.expression.spel.support.StandardEvaluationContext; +import com.hrakaroo.glob.GlobPattern; + +import io.lettuce.core.cluster.SlotHash; import io.lettuce.core.codec.StringCodec; class ProcessorTests { + private KeyFilter keyFilter(KeyFilterOptions options) { + KeyFilter filter = new KeyFilter<>(StringCodec.UTF8); + filter.setOptions(options); + filter.afterPropertiesSet(); + return filter; + } + @Test void keyFilter() { KeyFilterOptions options = new KeyFilterOptions(); options.setIncludes(Arrays.asList("foo*", "bar*")); - Predicate predicate = options.predicate(StringCodec.UTF8); + KeyFilter predicate = keyFilter(options); Assertions.assertTrue(predicate.test("foobar")); Assertions.assertTrue(predicate.test("barfoo")); Assertions.assertFalse(predicate.test("key")); @@ -80,4 +93,77 @@ void processorFilter() throws Exception { } } + @Test + void slotExact() { + KeyFilterOptions options = new KeyFilterOptions(); + options.setSlots(Arrays.asList(new SlotRange(7638, 7638))); + KeyFilter predicate = keyFilter(options); + assertTrue(predicate.test("abc")); + assertFalse(predicate.test("abcd")); + } + + @Test + void slotRange() { + KeyFilterOptions options = new KeyFilterOptions(); + options.setSlots(slotRangeList(0, SlotHash.SLOT_COUNT)); + KeyFilter unbounded = keyFilter(options); + assertTrue(unbounded.test("foo")); + assertTrue(unbounded.test("foo1")); + options.setSlots(slotRangeList(999999, 99999)); + Predicate is999999 = keyFilter(options); + assertFalse(is999999.test("foo")); + } + + private List slotRangeList(int start, int end) { + return Arrays.asList(new SlotRange(start, end)); + } + + @Test + void kitchenSink() { + KeyFilterOptions options = new KeyFilterOptions(); + options.setExcludes(Arrays.asList("foo")); + options.setIncludes(Arrays.asList("foo1")); + options.setSlots(Arrays.asList(new SlotRange(0, SlotHash.SLOT_COUNT))); + Predicate predicate = keyFilter(options); + assertFalse(predicate.test("foo")); + assertFalse(predicate.test("bar")); + assertTrue(predicate.test("foo1")); + } + + private Predicate globPredicate(String match) { + return GlobPattern.compile(match)::matches; + } + + @Test + void include() { + Predicate foo = globPredicate("foo"); + assertTrue(foo.test("foo")); + assertFalse(foo.test("bar")); + Predicate fooStar = globPredicate("foo*"); + assertTrue(fooStar.test("foobar")); + assertFalse(fooStar.test("barfoo")); + } + + @Test + void exclude() { + Predicate foo = globPredicate("foo").negate(); + assertFalse(foo.test("foo")); + assertTrue(foo.test("foa")); + Predicate fooStar = globPredicate("foo*").negate(); + assertFalse(fooStar.test("foobar")); + assertTrue(fooStar.test("barfoo")); + } + + @Test + void includeAndExclude() { + Predicate foo1 = globPredicate("foo1").and(globPredicate("foo").negate()); + assertFalse(foo1.test("foo")); + assertFalse(foo1.test("bar")); + assertTrue(foo1.test("foo1")); + Predicate foo1Star = globPredicate("foo").and(globPredicate("foo1*").negate()); + assertTrue(foo1Star.test("foo")); + assertFalse(foo1Star.test("bar")); + assertFalse(foo1Star.test("foo1")); + } + } diff --git a/docs/guide/src/docs/asciidoc/quickstart.adoc b/docs/guide/src/docs/asciidoc/quickstart.adoc index a95dfa463..7c1997e87 100644 --- a/docs/guide/src/docs/asciidoc/quickstart.adoc +++ b/docs/guide/src/docs/asciidoc/quickstart.adoc @@ -14,14 +14,14 @@ It is not required to run locally on a Redis server. .Homebrew (macOS & Linux) [source] ---- -brew install redis-developer/tap/riot +brew install redis/tap/riot ---- [[_scoop_install]] .Scoop (Windows) [source] ---- -scoop bucket add redis-developer https://github.com/redis-developer/scoop.git +scoop bucket add redis https://github.com/redis/scoop.git scoop install riot ---- diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/AbstractExportCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/AbstractExportCommand.java index a501c1b47..51d7713dd 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/AbstractExportCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/AbstractExportCommand.java @@ -6,47 +6,40 @@ public abstract class AbstractExportCommand extends AbstractRiotCommand { + @ArgGroup(exclusive = false, heading = "Redis client options%n") + private RedisClientArgs redisClientArgs = new RedisClientArgs(); + @ArgGroup(exclusive = false, heading = "Redis reader options%n") - private ExportArgs exportArgs = new ExportArgs(); + private RedisReaderArgs redisReaderArgs = new RedisReaderArgs(); - @ArgGroup(exclusive = false) + @ArgGroup(exclusive = false, heading = "Processor options%n") private KeyValueProcessorArgs processorArgs = new KeyValueProcessorArgs(); @Override protected AbstractExport callable() { AbstractExport export = exportCallable(); - export.setRedisClientOptions(exportArgs.getRedisClientArgs().redisClientOptions()); - export.setReaderOptions(exportArgs.getRedisReaderArgs().redisReaderOptions()); + export.setRedisClientOptions(redisClientArgs.redisClientOptions()); + export.setReaderOptions(redisReaderArgs.redisReaderOptions()); export.setProcessorOptions(processorArgs.processorOptions()); return export; } protected abstract AbstractExport exportCallable(); - public static class ExportArgs { - - @ArgGroup(exclusive = false) - private RedisClientArgs redisClientArgs = new RedisClientArgs(); - - @ArgGroup(exclusive = false) - private RedisReaderArgs redisReaderArgs = new RedisReaderArgs(); - - public RedisClientArgs getRedisClientArgs() { - return redisClientArgs; - } - - public void setRedisClientArgs(RedisClientArgs redisClientArgs) { - this.redisClientArgs = redisClientArgs; - } + public RedisClientArgs getRedisClientArgs() { + return redisClientArgs; + } - public RedisReaderArgs getRedisReaderArgs() { - return redisReaderArgs; - } + public void setRedisClientArgs(RedisClientArgs redisClientArgs) { + this.redisClientArgs = redisClientArgs; + } - public void setRedisReaderArgs(RedisReaderArgs redisReaderArgs) { - this.redisReaderArgs = redisReaderArgs; - } + public RedisReaderArgs getRedisReaderArgs() { + return redisReaderArgs; + } + public void setRedisReaderArgs(RedisReaderArgs redisReaderArgs) { + this.redisReaderArgs = redisReaderArgs; } public KeyValueProcessorArgs getProcessorArgs() { @@ -57,12 +50,4 @@ public void setProcessorArgs(KeyValueProcessorArgs args) { this.processorArgs = args; } - public ExportArgs getExportArgs() { - return exportArgs; - } - - public void setExportArgs(ExportArgs exportArgs) { - this.exportArgs = exportArgs; - } - } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/AbstractImportCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/AbstractImportCommand.java index 73322cbc4..9162f4f41 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/AbstractImportCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/AbstractImportCommand.java @@ -6,51 +6,36 @@ public abstract class AbstractImportCommand extends AbstractRiotCommand { + @ArgGroup(exclusive = false, heading = "Redis client options%n") + private RedisClientArgs redisClientArgs = new RedisClientArgs(); + @ArgGroup(exclusive = false, heading = "Redis writer options%n") - private ImportArgs importArgs = new ImportArgs(); + private RedisWriterArgs redisWriterArgs = new RedisWriterArgs(); @Override protected AbstractImport callable() { AbstractImport callable = importCallable(); - callable.setRedisClientOptions(importArgs.getRedisClientArgs().redisClientOptions()); - callable.setWriterOptions(importArgs.getRedisWriterArgs().writerOptions()); + callable.setRedisClientOptions(redisClientArgs.redisClientOptions()); + callable.setWriterOptions(redisWriterArgs.writerOptions()); return callable; } protected abstract AbstractImport importCallable(); - public static class ImportArgs { - - @ArgGroup(exclusive = false) - private RedisWriterArgs redisWriterArgs = new RedisWriterArgs(); - - @ArgGroup(exclusive = false) - private RedisClientArgs redisClientArgs = new RedisClientArgs(); - - public RedisWriterArgs getRedisWriterArgs() { - return redisWriterArgs; - } - - public void setRedisWriterArgs(RedisWriterArgs args) { - this.redisWriterArgs = args; - } - - public RedisClientArgs getRedisClientArgs() { - return redisClientArgs; - } - - public void setRedisClientArgs(RedisClientArgs args) { - this.redisClientArgs = args; - } + public RedisClientArgs getRedisClientArgs() { + return redisClientArgs; + } + public void setRedisClientArgs(RedisClientArgs redisClientArgs) { + this.redisClientArgs = redisClientArgs; } - public ImportArgs getImportArgs() { - return importArgs; + public RedisWriterArgs getRedisWriterArgs() { + return redisWriterArgs; } - public void setImportArgs(ImportArgs args) { - this.importArgs = args; + public void setRedisWriterArgs(RedisWriterArgs redisWriterArgs) { + this.redisWriterArgs = redisWriterArgs; } } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/AbstractMainCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/AbstractMainCommand.java index 8e4b2e72d..dd5bfbfc1 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/AbstractMainCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/AbstractMainCommand.java @@ -90,7 +90,7 @@ private static int executionStrategy(ParseResult parseResult) { if (redisCommand.isUsageHelpRequested()) { return new RunLast().execute(redisCommand); } - importCommand.getCommands().add((RedisOperationCommand) redisCommand.commandSpec().userObject()); + importCommand.getCommands().add((WriteOperationCommand) redisCommand.commandSpec().userObject()); } return new RunFirst().execute(subcommand); } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/AbstractMapImportCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/AbstractMapImportCommand.java index ba2041ecd..08a141a1d 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/AbstractMapImportCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/AbstractMapImportCommand.java @@ -22,7 +22,7 @@ import com.redis.riot.cli.redis.ZaddCommand; import com.redis.riot.core.AbstractMapImport; import com.redis.riot.core.ImportProcessorOptions; -import com.redis.spring.batch.operation.Operation; +import com.redis.spring.batch.writer.WriteOperation; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; @@ -46,18 +46,18 @@ public abstract class AbstractMapImportCommand extends AbstractImportCommand { /** * Initialized manually during command parsing */ - private List commands = new ArrayList<>(); + private List commands = new ArrayList<>(); - public List getCommands() { + public List getCommands() { return commands; } - public void setCommands(List commands) { + public void setCommands(List commands) { this.commands = commands; } - protected List, Object>> operations() { - return commands.stream().map(RedisOperationCommand::operation).collect(Collectors.toList()); + protected List>> operations() { + return commands.stream().map(WriteOperationCommand::operation).collect(Collectors.toList()); } @Override diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/AbstractRiotCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/AbstractRiotCommand.java index 1efe5ad39..8cc10d9d3 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/AbstractRiotCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/AbstractRiotCommand.java @@ -14,11 +14,11 @@ import org.springframework.util.ClassUtils; import com.redis.riot.core.AbstractRiotCallable; +import com.redis.riot.core.ScanSizeEstimator; import com.redis.riot.faker.FakerItemReader; import com.redis.spring.batch.RedisItemReader; import com.redis.spring.batch.RedisItemReader.ReaderMode; import com.redis.spring.batch.gen.GeneratorItemReader; -import com.redis.spring.batch.reader.ScanSizeEstimator; import me.tongfei.progressbar.DelegatingProgressBarConsumer; import me.tongfei.progressbar.ProgressBarBuilder; diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/KeyValueProcessorArgs.java b/plugins/riot/src/main/java/com/redis/riot/cli/KeyValueProcessorArgs.java index 6330f1581..ed0933546 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/KeyValueProcessorArgs.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/KeyValueProcessorArgs.java @@ -9,7 +9,7 @@ public class KeyValueProcessorArgs { - @Option(names = "--key-proc", description = "SpEL template expression to transform the name of each key. E.g. \"#{#source.database}:#{key}\" with 'abc' returns '0:abc'", paramLabel = "") + @Option(names = "--key-proc", description = "SpEL template expression for key names, e.g. \"#{#source.database}:#{key}\" for 'abc' returns '0:abc'", paramLabel = "") private TemplateExpression keyExpression; @Option(names = "--type-proc", description = "SpEL expression to transform the type of each key.", paramLabel = "") diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/PingCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/PingCommand.java index c326b2064..e188c490d 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/PingCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/PingCommand.java @@ -15,7 +15,7 @@ public class PingCommand extends AbstractRiotCommand { @ParentCommand private AbstractMainCommand parent; - @ArgGroup(exclusive = false) + @ArgGroup(exclusive = false, heading = "Redis client options%n") private RedisClientArgs redisClientArgs = new RedisClientArgs(); @Option(names = "--iterations", description = "Number of test iterations. Use a negative value to test endlessly. (default: ${DEFAULT-VALUE}).", paramLabel = "") diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/RedisClientArgs.java b/plugins/riot/src/main/java/com/redis/riot/cli/RedisClientArgs.java index 41d020727..92ef4743f 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/RedisClientArgs.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/RedisClientArgs.java @@ -1,60 +1,48 @@ package com.redis.riot.cli; import com.redis.riot.core.RedisClientOptions; +import com.redis.riot.core.RedisUriOptions; -import io.lettuce.core.ClientOptions; -import io.lettuce.core.cluster.ClusterClientOptions; import io.lettuce.core.protocol.ProtocolVersion; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Option; public class RedisClientArgs { - @ArgGroup(exclusive = false) - private RedisURIArgs uriArgs = new RedisURIArgs(); + @ArgGroup + private RedisUriArgs uriArgs = new RedisUriArgs(); @Option(names = { "-c", "--cluster" }, description = "Enable cluster mode.") private boolean cluster; @Option(names = "--auto-reconnect", defaultValue = "true", fallbackValue = "true", negatable = true, description = "Automatically reconnect on connection loss. True by default.", hidden = true) - private boolean autoReconnect = true; + private boolean autoReconnect = RedisClientOptions.DEFAULT_AUTO_RECONNECT; @Option(names = "--resp", description = "Redis protocol version used to connect to Redis: ${COMPLETION-CANDIDATES}.", paramLabel = "") - private ProtocolVersion protocolVersion; + private ProtocolVersion protocolVersion = RedisClientOptions.DEFAULT_PROTOCOL_VERSION; @ArgGroup(exclusive = false) private SslArgs sslArgs = new SslArgs(); + public RedisUriOptions redisUriOptions() { + return uriArgs.redisUriOptions(); + } + public RedisClientOptions redisClientOptions() { RedisClientOptions options = new RedisClientOptions(); - options.setRedisURI(uriArgs.redisURI()); + options.setUriOptions(uriArgs.redisUriOptions()); options.setCluster(cluster); - options.setOptions(clientOptions()); + options.setAutoReconnect(autoReconnect); + options.setProtocolVersion(protocolVersion); + options.setSslOptions(sslArgs.sslOptions()); return options; } - private ClientOptions clientOptions() { - if (cluster) { - ClusterClientOptions.Builder options = ClusterClientOptions.builder(); - configure(options); - return options.build(); - } - ClientOptions.Builder options = ClientOptions.builder(); - configure(options); - return options.build(); - } - - private void configure(ClientOptions.Builder builder) { - builder.autoReconnect(autoReconnect); - builder.protocolVersion(protocolVersion); - builder.sslOptions(sslArgs.sslOptions()); - } - - public RedisURIArgs getUriArgs() { + public RedisUriArgs getUriArgs() { return uriArgs; } - public void setUriArgs(RedisURIArgs args) { + public void setUriArgs(RedisUriArgs args) { this.uriArgs = args; } @@ -78,16 +66,16 @@ public ProtocolVersion getProtocolVersion() { return protocolVersion; } - public void setProtocolVersion(ProtocolVersion version) { - this.protocolVersion = version; + public void setProtocolVersion(ProtocolVersion protocolVersion) { + this.protocolVersion = protocolVersion; } public SslArgs getSslArgs() { return sslArgs; } - public void setSslArgs(SslArgs args) { - this.sslArgs = args; + public void setSslArgs(SslArgs sslArgs) { + this.sslArgs = sslArgs; } -} \ No newline at end of file +} diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/RedisOperationCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/RedisOperationCommand.java deleted file mode 100644 index b0d0d229e..000000000 --- a/plugins/riot/src/main/java/com/redis/riot/cli/RedisOperationCommand.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.redis.riot.cli; - -import java.util.Map; - -import com.redis.spring.batch.operation.Operation; - -public interface RedisOperationCommand { - - Operation, Object> operation(); - -} diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/RedisURIArgs.java b/plugins/riot/src/main/java/com/redis/riot/cli/RedisUriArgs.java similarity index 69% rename from plugins/riot/src/main/java/com/redis/riot/cli/RedisURIArgs.java rename to plugins/riot/src/main/java/com/redis/riot/cli/RedisUriArgs.java index 6807435b0..90b3d7bac 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/RedisURIArgs.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/RedisUriArgs.java @@ -1,28 +1,21 @@ package com.redis.riot.cli; -import java.time.Duration; - -import org.springframework.util.ObjectUtils; -import org.springframework.util.StringUtils; - -import com.redis.riot.core.RedisClientOptions; +import com.redis.riot.core.RedisUriOptions; import com.redis.riot.core.RiotVersion; -import io.lettuce.core.RedisURI; -import io.lettuce.core.SslVerifyMode; import picocli.CommandLine.Option; -public class RedisURIArgs { +public class RedisUriArgs { @Option(names = { "-u", "--uri" }, description = "Server URI.", paramLabel = "") private String uri; @Option(names = { "-h", "--host" }, description = "Server hostname (default: ${DEFAULT-VALUE}).", paramLabel = "") - private String host = RedisClientOptions.DEFAULT_REDIS_HOST; + private String host = RedisUriOptions.DEFAULT_HOST; @Option(names = { "-p", "--port" }, description = "Server port (default: ${DEFAULT-VALUE}).", paramLabel = "") - private int port = RedisClientOptions.DEFAULT_REDIS_PORT; + private int port = RedisUriOptions.DEFAULT_PORT; @Option(names = { "-s", "--socket" }, description = "Server socket (overrides hostname and port).", paramLabel = "") @@ -50,51 +43,6 @@ public class RedisURIArgs { @Option(names = "--insecure", description = "Allow insecure TLS connection by skipping cert validation.") private boolean insecure; - public RedisURI redisURI() { - RedisURI.Builder builder = redisURIBuilder(); - if (database > 0) { - builder.withDatabase(database); - } - if (StringUtils.hasLength(clientName)) { - builder.withClientName(clientName); - } - if (!ObjectUtils.isEmpty(password)) { - if (StringUtils.hasLength(username)) { - builder.withAuthentication(username, password); - } else { - builder.withPassword(password); - } - } - if (tls) { - builder.withSsl(tls); - } - if (insecure) { - builder.withVerifyPeer(SslVerifyMode.NONE); - } - if (timeout > 0) { - builder.withTimeout(Duration.ofSeconds(timeout)); - } - return builder.build(); - } - - private RedisURI.Builder redisURIBuilder() { - if (StringUtils.hasLength(uri)) { - return RedisURI.builder(RedisURI.create(uri)); - } - if (StringUtils.hasLength(socket)) { - return RedisURI.Builder.socket(socket); - } - return RedisURI.Builder.redis(host, port); - } - - public String getUri() { - return uri; - } - - public void setUri(String uri) { - this.uri = uri; - } - public String getHost() { return host; } @@ -175,4 +123,28 @@ public void setInsecure(boolean insecure) { this.insecure = insecure; } -} + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public RedisUriOptions redisUriOptions() { + RedisUriOptions options = new RedisUriOptions(); + options.setClientName(clientName); + options.setDatabase(database); + options.setHost(host); + options.setInsecure(insecure); + options.setPassword(password); + options.setPort(port); + options.setSocket(socket); + options.setTimeout(timeout); + options.setTls(tls); + options.setUri(uri); + options.setUsername(username); + return options; + } + +} \ No newline at end of file diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateCommand.java index 3e837f32c..fa6efcb80 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateCommand.java @@ -19,8 +19,6 @@ import com.redis.spring.batch.reader.KeyComparison.Status; import com.redis.spring.batch.reader.KeyNotificationItemReader; -import io.lettuce.core.ClientOptions; -import io.lettuce.core.cluster.ClusterClientOptions; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -59,15 +57,21 @@ public class ReplicateCommand extends AbstractRiotCommand { @Option(names = "--idle-timeout", description = "Min number of millis to consider transfer complete (default: no timeout).", paramLabel = "") private long idleTimeout = Replication.DEFAULT_IDLE_TIMEOUT.toMillis(); - @ArgGroup(exclusive = false, heading = "Source Redis options%n") - private ReplicateSourceArgs sourceArgs = new ReplicateSourceArgs(); + @ArgGroup(exclusive = false, heading = "Source Redis client options%n") + private RedisClientArgs sourceRedisClientArgs = new RedisClientArgs(); - @ArgGroup(exclusive = false, heading = "Target Redis options%n") - private ReplicateTargetArgs targetArgs = new ReplicateTargetArgs(); + @ArgGroup(exclusive = false, heading = "Source Redis reader options%n") + private ReplicateRedisReaderArgs sourceRedisReaderArgs = new ReplicateRedisReaderArgs(); @ArgGroup(exclusive = false, heading = "Processor options%n") private KeyValueProcessorArgs processorArgs = new KeyValueProcessorArgs(); + @ArgGroup(exclusive = false, heading = "Target Redis client options%n") + private ReplicateTargetRedisClientArgs targetRedisClientArgs = new ReplicateTargetRedisClientArgs(); + + @ArgGroup(exclusive = false, heading = "Target Redis writer options%n") + private RedisWriterArgs targetRedisWriterArgs = new RedisWriterArgs(); + private static Map taskNames() { Map map = new HashMap<>(); map.put(Replication.STEP_SCAN, "Scanning"); @@ -82,47 +86,28 @@ protected Replication callable() { replication.setCompareMode(compareMode); replication.setMode(mode); replication.setShowDiffs(showDiffs); - if (targetArgs.getReadFrom() != null) { - replication.setTargetReadFrom(targetArgs.getReadFrom().getReadFrom()); + if (targetRedisClientArgs.getReadFrom() != null) { + replication.setTargetReadFrom(targetRedisClientArgs.getReadFrom().getReadFrom()); } replication.setTargetRedisClientOptions(targetRedisClientOptions()); replication.setTtlTolerance(Duration.ofMillis(ttlTolerance)); replication.setType(type ? ReplicationType.STRUCT : ReplicationType.DUMP); - replication.setWriterOptions(targetArgs.getWriterArgs().writerOptions()); + replication.setWriterOptions(targetRedisWriterArgs.writerOptions()); replication.setFlushInterval(Duration.ofMillis(flushInterval)); replication.setIdleTimeout(Duration.ofMillis(idleTimeout)); - replication.setNotificationQueueCapacity(sourceArgs.getNotificationQueueCapacity()); - replication.setRedisClientOptions(sourceArgs.getRedisClientArgs().redisClientOptions()); - replication.setReaderOptions(sourceArgs.getRedisReaderArgs().redisReaderOptions()); + replication.setNotificationQueueCapacity(sourceRedisReaderArgs.getNotificationQueueCapacity()); + replication.setRedisClientOptions(sourceRedisClientArgs.redisClientOptions()); + replication.setReaderOptions(sourceRedisReaderArgs.redisReaderOptions()); replication.setProcessorOptions(processorArgs.processorOptions()); return replication; } private RedisClientOptions targetRedisClientOptions() { - RedisClientOptions options = new RedisClientOptions(); - options.setRedisURI(targetArgs.redisURI()); - options.setCluster(targetArgs.isCluster()); - options.setOptions(targetClientOptions()); + RedisClientOptions options = targetRedisClientArgs.redisClientOptions(); + options.setSslOptions(sourceRedisClientArgs.getSslArgs().sslOptions()); return options; } - private ClientOptions targetClientOptions() { - if (targetArgs.isCluster()) { - ClusterClientOptions.Builder options = ClusterClientOptions.builder(); - configure(options); - return options.build(); - } - ClientOptions.Builder options = ClientOptions.builder(); - configure(options); - return options.build(); - } - - private void configure(ClientOptions.Builder builder) { - builder.autoReconnect(targetArgs.isAutoReconnect()); - builder.protocolVersion(targetArgs.getProtocolVersion()); - builder.sslOptions(sourceArgs.getRedisClientArgs().getSslArgs().sslOptions()); - } - @Override protected String taskName(String stepName) { return taskNames.getOrDefault(stepName, "Unknown"); @@ -185,8 +170,8 @@ public long getTtlTolerance() { return ttlTolerance; } - public void setTtlTolerance(long ttlTolerance) { - this.ttlTolerance = ttlTolerance; + public void setTtlTolerance(long tolerance) { + this.ttlTolerance = tolerance; } public boolean isShowDiffs() { @@ -201,44 +186,60 @@ public CompareMode getCompareMode() { return compareMode; } - public void setCompareMode(CompareMode compareMode) { - this.compareMode = compareMode; + public void setCompareMode(CompareMode mode) { + this.compareMode = mode; } public long getFlushInterval() { return flushInterval; } - public void setFlushInterval(long flushInterval) { - this.flushInterval = flushInterval; + public void setFlushInterval(long interval) { + this.flushInterval = interval; } public long getIdleTimeout() { return idleTimeout; } - public ReplicateSourceArgs getSourceArgs() { - return sourceArgs; + public KeyValueProcessorArgs getProcessorArgs() { + return processorArgs; } - public void setSourceArgs(ReplicateSourceArgs sourceArgs) { - this.sourceArgs = sourceArgs; + public void setProcessorArgs(KeyValueProcessorArgs args) { + this.processorArgs = args; } - public ReplicateTargetArgs getTargetArgs() { - return targetArgs; + public RedisWriterArgs getTargetRedisWriterArgs() { + return targetRedisWriterArgs; } - public void setTargetArgs(ReplicateTargetArgs targetArgs) { - this.targetArgs = targetArgs; + public void setTargetRedisWriterArgs(RedisWriterArgs args) { + this.targetRedisWriterArgs = args; } - public KeyValueProcessorArgs getProcessorArgs() { - return processorArgs; + public RedisClientArgs getSourceRedisClientArgs() { + return sourceRedisClientArgs; + } + + public void setSourceRedisClientArgs(RedisClientArgs args) { + this.sourceRedisClientArgs = args; + } + + public ReplicateRedisReaderArgs getSourceRedisReaderArgs() { + return sourceRedisReaderArgs; + } + + public void setSourceRedisReaderArgs(ReplicateRedisReaderArgs args) { + this.sourceRedisReaderArgs = args; + } + + public ReplicateTargetRedisClientArgs getTargetRedisClientArgs() { + return targetRedisClientArgs; } - public void setProcessorArgs(KeyValueProcessorArgs processorArgs) { - this.processorArgs = processorArgs; + public void setTargetRedisClientArgs(ReplicateTargetRedisClientArgs args) { + this.targetRedisClientArgs = args; } } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateRedisReaderArgs.java b/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateRedisReaderArgs.java new file mode 100644 index 000000000..3516e9839 --- /dev/null +++ b/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateRedisReaderArgs.java @@ -0,0 +1,20 @@ +package com.redis.riot.cli; + +import com.redis.riot.redis.Replication; + +import picocli.CommandLine.Option; + +public class ReplicateRedisReaderArgs extends RedisReaderArgs { + + @Option(names = "--event-queue", description = "Capacity of the keyspace notification event queue (default: ${DEFAULT-VALUE}).", paramLabel = "") + private int notificationQueueCapacity = Replication.DEFAULT_NOTIFICATION_QUEUE_CAPACITY; + + public int getNotificationQueueCapacity() { + return notificationQueueCapacity; + } + + public void setNotificationQueueCapacity(int capacity) { + this.notificationQueueCapacity = capacity; + } + +} diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateSourceArgs.java b/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateSourceArgs.java deleted file mode 100644 index c9dd58bb2..000000000 --- a/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateSourceArgs.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.redis.riot.cli; - -import com.redis.riot.redis.Replication; - -import picocli.CommandLine.ArgGroup; -import picocli.CommandLine.Option; - -public class ReplicateSourceArgs { - - @ArgGroup(exclusive = false) - private RedisClientArgs redisClientArgs = new RedisClientArgs(); - - @ArgGroup(exclusive = false) - private RedisReaderArgs redisReaderArgs = new RedisReaderArgs(); - - @Option(names = "--event-queue", description = "Capacity of the keyspace notification event queue (default: ${DEFAULT-VALUE}).", paramLabel = "") - private int notificationQueueCapacity = Replication.DEFAULT_NOTIFICATION_QUEUE_CAPACITY; - - public RedisClientArgs getRedisClientArgs() { - return redisClientArgs; - } - - public void setRedisClientArgs(RedisClientArgs args) { - this.redisClientArgs = args; - } - - public RedisReaderArgs getRedisReaderArgs() { - return redisReaderArgs; - } - - public void setRedisReaderArgs(RedisReaderArgs args) { - this.redisReaderArgs = args; - } - - public int getNotificationQueueCapacity() { - return notificationQueueCapacity; - } - - public void setNotificationQueueCapacity(int capacity) { - this.notificationQueueCapacity = capacity; - } - -} diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateTargetArgs.java b/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateTargetRedisClientArgs.java similarity index 52% rename from plugins/riot/src/main/java/com/redis/riot/cli/ReplicateTargetArgs.java rename to plugins/riot/src/main/java/com/redis/riot/cli/ReplicateTargetRedisClientArgs.java index 563602311..321f67922 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateTargetArgs.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateTargetRedisClientArgs.java @@ -1,32 +1,16 @@ package com.redis.riot.cli; -import org.springframework.util.ObjectUtils; -import org.springframework.util.StringUtils; - import com.redis.riot.cli.RedisReaderArgs.ReadFromEnum; -import com.redis.riot.core.RiotVersion; +import com.redis.riot.core.RedisClientOptions; -import io.lettuce.core.RedisURI; import io.lettuce.core.protocol.ProtocolVersion; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Option; -public class ReplicateTargetArgs { +public class ReplicateTargetRedisClientArgs { @ArgGroup(exclusive = false) - private RedisWriterArgs writerArgs = new RedisWriterArgs(); - - @Option(names = "--target-read-from", description = "Which target cluster nodes to read data from: ${COMPLETION-CANDIDATES}.", paramLabel = "") - private ReadFromEnum readFrom; - - @Option(names = "--target-uri", description = "Target server URI.", paramLabel = "") - private String uri; - - @Option(names = "--target-pass", arity = "0..1", interactive = true, description = "Password to use when connecting to the target server.", paramLabel = "") - private char[] password; - - @Option(names = "--target-client", description = "Client name used to connect to target Redis (default: ${DEFAULT-VALUE}).", paramLabel = "") - private String clientName = RiotVersion.riotVersion(); + private ReplicateTargetRedisUriArgs uriArgs = new ReplicateTargetRedisUriArgs(); @Option(names = "--target-cluster", description = "Enable target cluster mode.") private boolean cluster; @@ -37,12 +21,15 @@ public class ReplicateTargetArgs { @Option(names = "--target-resp", description = "Redis protocol version used to connect to target Redis: ${COMPLETION-CANDIDATES}.", paramLabel = "") private ProtocolVersion protocolVersion; - public RedisWriterArgs getWriterArgs() { - return writerArgs; + @Option(names = "--target-read-from", description = "Which target cluster nodes to read data from: ${COMPLETION-CANDIDATES}.", paramLabel = "") + private ReadFromEnum readFrom; + + public ReplicateTargetRedisUriArgs getUriArgs() { + return uriArgs; } - public void setWriterArgs(RedisWriterArgs writerArgs) { - this.writerArgs = writerArgs; + public void setUriArgs(ReplicateTargetRedisUriArgs uriArgs) { + this.uriArgs = uriArgs; } public ReadFromEnum getReadFrom() { @@ -53,14 +40,6 @@ public void setReadFrom(ReadFromEnum readFrom) { this.readFrom = readFrom; } - public String getUri() { - return uri; - } - - public void setUri(String uri) { - this.uri = uri; - } - public boolean isCluster() { return cluster; } @@ -85,23 +64,13 @@ public void setProtocolVersion(ProtocolVersion protocolVersion) { this.protocolVersion = protocolVersion; } - public char[] getPassword() { - return password; - } - - public void setPassword(char[] password) { - this.password = password; - } - - public RedisURI redisURI() { - RedisURI.Builder builder = RedisURI.builder(RedisURI.create(uri)); - if (!ObjectUtils.isEmpty(password)) { - builder.withPassword(password); - } - if (StringUtils.hasLength(clientName)) { - builder.withClientName(clientName); - } - return builder.build(); + public RedisClientOptions redisClientOptions() { + RedisClientOptions options = new RedisClientOptions(); + options.setAutoReconnect(autoReconnect); + options.setCluster(cluster); + options.setProtocolVersion(protocolVersion); + options.setUriOptions(uriArgs.redisUriOptions()); + return options; } } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateTargetRedisUriArgs.java b/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateTargetRedisUriArgs.java new file mode 100644 index 000000000..7ffd11453 --- /dev/null +++ b/plugins/riot/src/main/java/com/redis/riot/cli/ReplicateTargetRedisUriArgs.java @@ -0,0 +1,87 @@ +package com.redis.riot.cli; + +import com.redis.riot.core.RedisUriOptions; +import com.redis.riot.core.RiotVersion; + +import picocli.CommandLine.Option; + +public class ReplicateTargetRedisUriArgs { + + @Option(names = "--target-uri", description = "Target server URI.", paramLabel = "") + private String uri; + + @Option(names = "--target-host", description = "Target server hostname (default: ${DEFAULT-VALUE}).", paramLabel = "") + private String host = RedisUriOptions.DEFAULT_HOST; + + @Option(names = "--target-port", description = "Target server port (default: ${DEFAULT-VALUE}).", paramLabel = "") + private int port = RedisUriOptions.DEFAULT_PORT; + + @Option(names = "--target-user", description = "Target ACL style 'AUTH username pass'. Needs password.", paramLabel = "") + private String username; + + @Option(names = "--target-pass", arity = "0..1", interactive = true, description = "Password to use when connecting to the target server.", paramLabel = "") + private char[] password; + + @Option(names = "--target-client", description = "Client name used to connect to target Redis (default: ${DEFAULT-VALUE}).", paramLabel = "") + private String clientName = RiotVersion.riotVersion(); + + public RedisUriOptions redisUriOptions() { + RedisUriOptions options = new RedisUriOptions(); + options.setClientName(clientName); + options.setHost(host); + options.setPort(port); + options.setPassword(password); + options.setUri(uri); + options.setUsername(username); + return options; + } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public char[] getPassword() { + return password; + } + + public void setPassword(char[] password) { + this.password = password; + } + + public String getClientName() { + return clientName; + } + + public void setClientName(String clientName) { + this.clientName = clientName; + } + +} diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/SslArgs.java b/plugins/riot/src/main/java/com/redis/riot/cli/SslArgs.java index 3fb2bad64..5c5156ba2 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/SslArgs.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/SslArgs.java @@ -9,28 +9,28 @@ public class SslArgs { - @Option(names = "--ks", description = "Path to keystore.", paramLabel = "", hidden = true) + @Option(names = "--keystore", description = "Path to keystore.", paramLabel = "", hidden = true) private File keystore; - @Option(names = "--ks-pwd", arity = "0..1", interactive = true, description = "Keystore password.", paramLabel = "", hidden = true) + @Option(names = "--keystore-pass", arity = "0..1", interactive = true, description = "Keystore password.", paramLabel = "", hidden = true) private char[] keystorePassword; - @Option(names = "--ts", description = "Path to truststore.", paramLabel = "", hidden = true) + @Option(names = "--trust", description = "Path to truststore.", paramLabel = "", hidden = true) private File truststore; - @Option(names = "--ts-pwd", arity = "0..1", interactive = true, description = "Truststore password.", paramLabel = "", hidden = true) + @Option(names = "--trust-pass", arity = "0..1", interactive = true, description = "Truststore password.", paramLabel = "", hidden = true) private char[] truststorePassword; - @Option(names = "--cert", description = "X.509 cert chain file to authenticate (PEM).", paramLabel = "") + @Option(names = "--cert", description = "Client certificate to authenticate with (X.509 PEM).", paramLabel = "") private File keyCert; - @Option(names = "--key", description = "PKCS#8 private key file to authenticate (PEM).", paramLabel = "") + @Option(names = "--key", description = "Private key file to authenticate with (PKCS#8 PEM).", paramLabel = "") private File key; - @Option(names = "--key-pwd", arity = "0..1", interactive = true, description = "Private key password.", paramLabel = "") + @Option(names = "--key-pass", arity = "0..1", interactive = true, description = "Private key password.", paramLabel = "") private char[] keyPassword; - @Option(names = "--cacert", description = "X.509 CA certificate file to verify with.", paramLabel = "") + @Option(names = "--cacert", description = "CA Certificate file to verify with (X.509).", paramLabel = "") private File trustedCerts; public SslOptions sslOptions() { diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/WriteOperationCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/WriteOperationCommand.java new file mode 100644 index 000000000..4a88bb8ef --- /dev/null +++ b/plugins/riot/src/main/java/com/redis/riot/cli/WriteOperationCommand.java @@ -0,0 +1,11 @@ +package com.redis.riot.cli; + +import java.util.Map; + +import com.redis.spring.batch.writer.WriteOperation; + +public interface WriteOperationCommand { + + WriteOperation> operation(); + +} diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/redis/AbstractCollectionOperationCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/redis/AbstractCollectionOperationCommand.java index b0766ac11..afb4f45ca 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/redis/AbstractCollectionOperationCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/redis/AbstractCollectionOperationCommand.java @@ -7,7 +7,7 @@ import picocli.CommandLine.Option; -abstract class AbstractCollectionOperationCommand extends AbstractRedisOperationCommand { +abstract class AbstractCollectionOperationCommand extends AbstractWriteOperationCommand { @Option(names = "--member-space", description = "Keyspace prefix for member IDs.", paramLabel = "") private String memberSpace; diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/redis/AbstractRedisOperationCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/redis/AbstractWriteOperationCommand.java similarity index 86% rename from plugins/riot/src/main/java/com/redis/riot/cli/redis/AbstractRedisOperationCommand.java rename to plugins/riot/src/main/java/com/redis/riot/cli/redis/AbstractWriteOperationCommand.java index b238b96f2..006199781 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/redis/AbstractRedisOperationCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/redis/AbstractWriteOperationCommand.java @@ -3,15 +3,15 @@ import java.util.List; import java.util.Map; -import com.redis.riot.cli.RedisOperationCommand; +import com.redis.riot.cli.WriteOperationCommand; import com.redis.riot.core.operation.AbstractMapOperationBuilder; -import com.redis.spring.batch.operation.Operation; +import com.redis.spring.batch.writer.WriteOperation; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @Command(usageHelpAutoWidth = true, abbreviateSynopsis = true, mixinStandardHelpOptions = true) -abstract class AbstractRedisOperationCommand implements RedisOperationCommand { +abstract class AbstractWriteOperationCommand implements WriteOperationCommand { @Option(names = { "-p", "--keyspace" }, description = "Keyspace prefix.", paramLabel = "") private String keyspace; @@ -30,7 +30,7 @@ abstract class AbstractRedisOperationCommand implements RedisOperationCommand { private boolean ignoreMissingFields = AbstractMapOperationBuilder.DEFAULT_IGNORE_MISSING_FIELDS; @Override - public Operation, Object> operation() { + public WriteOperation> operation() { AbstractMapOperationBuilder builder = operationBuilder(); builder.setIgnoreMissingFields(ignoreMissingFields); builder.setKeyFields(keys); diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/redis/DelCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/redis/DelCommand.java index e029a7641..e46e063a9 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/redis/DelCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/redis/DelCommand.java @@ -5,7 +5,7 @@ import picocli.CommandLine.Command; @Command(name = "del", description = "Delete keys") -public class DelCommand extends AbstractRedisOperationCommand { +public class DelCommand extends AbstractWriteOperationCommand { @Override protected DelBuilder operationBuilder() { diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/redis/ExpireCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/redis/ExpireCommand.java index 73580a142..54dc7fd0d 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/redis/ExpireCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/redis/ExpireCommand.java @@ -8,7 +8,7 @@ import picocli.CommandLine.Option; @Command(name = "expire", description = "Set timeouts on keys") -public class ExpireCommand extends AbstractRedisOperationCommand { +public class ExpireCommand extends AbstractWriteOperationCommand { public static final long DEFAULT_TTL = 60; diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/redis/HsetCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/redis/HsetCommand.java index c51fc20d0..88590306a 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/redis/HsetCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/redis/HsetCommand.java @@ -6,7 +6,7 @@ import picocli.CommandLine.Mixin; @Command(name = "hset", aliases = "hmset", description = "Set hashes from input") -public class HsetCommand extends AbstractRedisOperationCommand { +public class HsetCommand extends AbstractWriteOperationCommand { @Mixin private FieldFilteringArgs filteringArgs = new FieldFilteringArgs(); diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/redis/JsonSetCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/redis/JsonSetCommand.java index 932fa6eb5..9d677eaff 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/redis/JsonSetCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/redis/JsonSetCommand.java @@ -6,7 +6,7 @@ import picocli.CommandLine.Option; @Command(name = "json.set", description = "Add JSON documents to RedisJSON") -public class JsonSetCommand extends AbstractRedisOperationCommand { +public class JsonSetCommand extends AbstractWriteOperationCommand { @Option(names = "--path", description = "Path field.", paramLabel = "") private String path; diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/redis/SetCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/redis/SetCommand.java index 6575fc6b8..72a3dd02c 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/redis/SetCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/redis/SetCommand.java @@ -7,7 +7,7 @@ import picocli.CommandLine.Option; @Command(name = "set", description = "Set strings from input") -public class SetCommand extends AbstractRedisOperationCommand { +public class SetCommand extends AbstractWriteOperationCommand { public static final StringFormat DEFAULT_FORMAT = StringFormat.JSON; diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/redis/SugaddCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/redis/SugaddCommand.java index 3fc7664f1..c33ee056c 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/redis/SugaddCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/redis/SugaddCommand.java @@ -6,7 +6,7 @@ import picocli.CommandLine.Option; @Command(name = "ft.sugadd", description = "Add suggestion strings to a RediSearch auto-complete dictionary") -public class SugaddCommand extends AbstractRedisOperationCommand { +public class SugaddCommand extends AbstractWriteOperationCommand { public static final double DEFAULT_SCORE = 1; diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/redis/TsAddCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/redis/TsAddCommand.java index 6ddacb10c..83c69189b 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/redis/TsAddCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/redis/TsAddCommand.java @@ -10,7 +10,7 @@ import picocli.CommandLine.Option; @Command(name = "ts.add", description = "Add samples to RedisTimeSeries") -public class TsAddCommand extends AbstractRedisOperationCommand { +public class TsAddCommand extends AbstractWriteOperationCommand { @Option(names = "--timestamp", description = "Name of the field to use for timestamps. If unset, uses auto-timestamping.", paramLabel = "") private String timestampField; diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/redis/XaddCommand.java b/plugins/riot/src/main/java/com/redis/riot/cli/redis/XaddCommand.java index 2d82b2fca..63bfe5376 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/redis/XaddCommand.java +++ b/plugins/riot/src/main/java/com/redis/riot/cli/redis/XaddCommand.java @@ -7,7 +7,7 @@ import picocli.CommandLine.Option; @Command(name = "xadd", description = "Append entries to a stream") -public class XaddCommand extends AbstractRedisOperationCommand { +public class XaddCommand extends AbstractWriteOperationCommand { @Mixin private FieldFilteringArgs filteringOptions = new FieldFilteringArgs(); diff --git a/plugins/riot/src/main/resources/com/redis/riot/cli/Banner.properties b/plugins/riot/src/main/resources/com/redis/riot/cli/Banner.properties deleted file mode 100644 index 557b0f5ba..000000000 --- a/plugins/riot/src/main/resources/com/redis/riot/cli/Banner.properties +++ /dev/null @@ -1,4 +0,0 @@ -product.version=$version -product.id=$id -product.name=$name -product.banner={0} {1} diff --git a/plugins/riot/src/test/java/com/redis/riot/cli/AbstractDbTests.java b/plugins/riot/src/test/java/com/redis/riot/cli/AbstractDbTests.java index 923f22628..4bd489323 100644 --- a/plugins/riot/src/test/java/com/redis/riot/cli/AbstractDbTests.java +++ b/plugins/riot/src/test/java/com/redis/riot/cli/AbstractDbTests.java @@ -26,12 +26,11 @@ abstract class AbstractDbTests extends AbstractRiotTestBase { private static final RedisStackContainer redis = new RedisStackContainer( RedisStackContainer.DEFAULT_IMAGE_NAME.withTag(RedisStackContainer.DEFAULT_TAG)); - protected abstract JdbcDatabaseContainer getJdbcDatabaseContainer(); - protected Connection dbConnection; - protected DataSource dataSource; + protected abstract JdbcDatabaseContainer getJdbcDatabaseContainer(); + @BeforeAll public void setupContainers() throws SQLException { JdbcDatabaseContainer container = getJdbcDatabaseContainer(); diff --git a/plugins/riot/src/test/java/com/redis/riot/cli/AbstractReplicationTests.java b/plugins/riot/src/test/java/com/redis/riot/cli/AbstractReplicationTests.java index b6ae34e38..2995049f9 100644 --- a/plugins/riot/src/test/java/com/redis/riot/cli/AbstractReplicationTests.java +++ b/plugins/riot/src/test/java/com/redis/riot/cli/AbstractReplicationTests.java @@ -1,6 +1,7 @@ package com.redis.riot.cli; import java.time.Duration; +import java.util.Collections; import java.util.List; import org.junit.jupiter.api.Assertions; @@ -45,7 +46,9 @@ void hll(TestInfo info) throws Throwable { String value = "http://www.google.com/"; redisCommands.pfadd(key, value); Assertions.assertEquals(0, execute(info, "replicate-hll")); - Assertions.assertTrue(compare(info).isOk()); + KeyspaceComparison comparison = compare(info); + Assertions.assertFalse(comparison.getAll().isEmpty()); + Assertions.assertEquals(Collections.emptyList(), comparison.mismatches()); } @Test @@ -115,7 +118,8 @@ void liveOnlyStruct(TestInfo info) throws Exception { generateAsync(testInfo(info, "async"), generator); execute(info, "replicate-live-only-struct"); KeyspaceComparison comparison = compare(info); - Assertions.assertTrue(comparison.isOk()); + Assertions.assertFalse(comparison.getAll().isEmpty()); + Assertions.assertEquals(Collections.emptyList(), comparison.mismatches()); } @Test @@ -150,7 +154,8 @@ protected void runLiveReplication(TestInfo info, String filename) throws Excepti generateAsync(testInfo(info, "async"), generator); execute(info, filename); KeyspaceComparison comparison = compare(info); - Assertions.assertTrue(comparison.isOk()); + Assertions.assertFalse(comparison.getAll().isEmpty()); + Assertions.assertEquals(Collections.emptyList(), comparison.mismatches()); } } diff --git a/plugins/riot/src/test/java/com/redis/riot/cli/AbstractRiotTestBase.java b/plugins/riot/src/test/java/com/redis/riot/cli/AbstractRiotTestBase.java index 6b7e6dd2e..a1c65cc78 100644 --- a/plugins/riot/src/test/java/com/redis/riot/cli/AbstractRiotTestBase.java +++ b/plugins/riot/src/test/java/com/redis/riot/cli/AbstractRiotTestBase.java @@ -15,7 +15,6 @@ import com.redis.riot.redis.Replication.LoggingWriteListener; import com.redis.riot.redis.ReplicationMode; import com.redis.spring.batch.test.AbstractTargetTestBase; -import com.redis.testcontainers.RedisServer; import io.micrometer.core.instrument.util.IOUtils; import picocli.CommandLine.ExitCode; @@ -59,7 +58,7 @@ private IExecutionStrategy executionStrategy(TestInfo info, IExecutionStrategy.. private int execute(TestInfo info, ParseResult parseResult) { for (ParseResult subParseResult : parseResult.subcommands()) { Object command = subParseResult.commandSpec().commandLine().getCommand(); - if (command instanceof RedisOperationCommand) { + if (command instanceof WriteOperationCommand) { command = subParseResult.commandSpec().parent().commandLine().getCommand(); } if (command instanceof AbstractRiotCommand) { @@ -68,34 +67,33 @@ private int execute(TestInfo info, ParseResult parseResult) { riotCommand.setName(name(info)); } if (command instanceof AbstractImportCommand) { - AbstractImportCommand importCommand = ((AbstractImportCommand) command); - RedisServer server = getRedisServer(); - importCommand.getImportArgs().getRedisClientArgs().getUriArgs().setUri(server.getRedisURI()); - importCommand.getImportArgs().getRedisClientArgs().setCluster(server.isRedisCluster()); + configure(((AbstractImportCommand) command).getRedisClientArgs()); } if (command instanceof AbstractExportCommand) { - AbstractExportCommand exportCommand = ((AbstractExportCommand) command); - exportCommand.getExportArgs().getRedisClientArgs().getUriArgs().setUri(getRedisServer().getRedisURI()); - exportCommand.getExportArgs().getRedisClientArgs().setCluster(getRedisServer().isRedisCluster()); + configure(((AbstractExportCommand) command).getRedisClientArgs()); } if (command instanceof ReplicateCommand) { ReplicateCommand replicateCommand = (ReplicateCommand) command; replicateCommand.setCompareMode(CompareMode.NONE); - replicateCommand.getSourceArgs().getRedisClientArgs().getUriArgs() - .setUri(getRedisServer().getRedisURI()); - replicateCommand.getSourceArgs().getRedisClientArgs().setCluster(getRedisServer().isRedisCluster()); - replicateCommand.getTargetArgs().setUri(getTargetRedisServer().getRedisURI()); - replicateCommand.getTargetArgs().setCluster(getTargetRedisServer().isRedisCluster()); + configure(replicateCommand.getSourceRedisClientArgs()); + replicateCommand.getTargetRedisClientArgs().getUriArgs().setUri(getTargetRedisServer().getRedisURI()); + replicateCommand.getTargetRedisClientArgs().setCluster(getTargetRedisServer().isRedisCluster()); if (replicateCommand.getMode() == ReplicationMode.LIVE || replicateCommand.getMode() == ReplicationMode.LIVEONLY) { replicateCommand.setIdleTimeout(getIdleTimeout().toMillis()); - replicateCommand.getSourceArgs().setNotificationQueueCapacity(DEFAULT_NOTIFICATION_QUEUE_CAPACITY); + replicateCommand.getSourceRedisReaderArgs() + .setNotificationQueueCapacity(DEFAULT_NOTIFICATION_QUEUE_CAPACITY); } } } return ExitCode.OK; } + private void configure(RedisClientArgs redisClientArgs) { + redisClientArgs.getUriArgs().setUri(getRedisServer().getRedisURI()); + redisClientArgs.setCluster(getRedisServer().isRedisCluster()); + } + private static String[] args(String filename) throws Exception { try (InputStream inputStream = AbstractMainCommand.class.getResourceAsStream("/" + filename)) { String command = IOUtils.toString(inputStream, Charset.defaultCharset()); diff --git a/plugins/riot/src/test/java/com/redis/riot/cli/StackToStackIntegrationTests.java b/plugins/riot/src/test/java/com/redis/riot/cli/StackToStackIntegrationTests.java index 138eab236..f50ec0eaf 100644 --- a/plugins/riot/src/test/java/com/redis/riot/cli/StackToStackIntegrationTests.java +++ b/plugins/riot/src/test/java/com/redis/riot/cli/StackToStackIntegrationTests.java @@ -32,13 +32,14 @@ import com.redis.lettucemod.timeseries.MRangeOptions; import com.redis.lettucemod.timeseries.RangeResult; import com.redis.lettucemod.timeseries.TimeRange; -import com.redis.riot.file.resource.XmlItemReader; -import com.redis.riot.file.resource.XmlItemReaderBuilder; -import com.redis.riot.file.resource.XmlObjectReader; +import com.redis.riot.file.xml.XmlItemReader; +import com.redis.riot.file.xml.XmlItemReaderBuilder; +import com.redis.riot.file.xml.XmlObjectReader; import com.redis.riot.redis.GeneratorImport; import com.redis.spring.batch.KeyValue; import com.redis.spring.batch.KeyValue.DataType; import com.redis.spring.batch.gen.GeneratorItemReader; +import com.redis.spring.batch.reader.MemKeyValue; import com.redis.testcontainers.RedisStackContainer; import io.lettuce.core.GeoArgs; @@ -100,7 +101,7 @@ protected void testImport(TestInfo info, String filename, String pattern, int co @SuppressWarnings("rawtypes") @Test void fileDumpImport(TestInfo info) throws Exception { - List records = exportToJsonFile(info); + List records = exportToJsonFile(info); redisCommands.flushall(); execute(info, "dump-import", this::executeFileDumpImport); awaitUntil(() -> records.size() == Math.toIntExact(redisCommands.dbsize())); @@ -109,23 +110,23 @@ void fileDumpImport(TestInfo info) throws Exception { @SuppressWarnings("rawtypes") @Test void fileExportJSON(TestInfo info) throws Exception { - List records = exportToJsonFile(info); + List records = exportToJsonFile(info); Assertions.assertEquals(redisCommands.dbsize(), records.size()); } @SuppressWarnings("rawtypes") - private List exportToJsonFile(TestInfo info) throws Exception { + private List exportToJsonFile(TestInfo info) throws Exception { String filename = "file-export-json"; Path file = tempFile("redis.json"); generate(info, generator(73)); execute(info, filename, r -> executeFileDumpExport(r, info)); - JsonItemReaderBuilder builder = new JsonItemReaderBuilder<>(); + JsonItemReaderBuilder builder = new JsonItemReaderBuilder<>(); builder.name("json-reader"); builder.resource(new FileSystemResource(file)); - JacksonJsonObjectReader objectReader = new JacksonJsonObjectReader<>(KeyValue.class); + JacksonJsonObjectReader objectReader = new JacksonJsonObjectReader<>(MemKeyValue.class); objectReader.setMapper(new ObjectMapper()); builder.jsonObjectReader(objectReader); - JsonItemReader reader = builder.build(); + JsonItemReader reader = builder.build(); reader.open(new ExecutionContext()); try { return readAll(reader); @@ -182,10 +183,10 @@ void fileExportXml(TestInfo info) throws Exception { generate(info, generator(73)); Path file = tempFile("redis.xml"); execute(info, filename, r -> executeFileDumpExport(r, info)); - XmlItemReaderBuilder builder = new XmlItemReaderBuilder<>(); + XmlItemReaderBuilder builder = new XmlItemReaderBuilder<>(); builder.name("xml-reader"); builder.resource(new FileSystemResource(file)); - XmlObjectReader xmlObjectReader = new XmlObjectReader<>(KeyValue.class); + XmlObjectReader xmlObjectReader = new XmlObjectReader<>(MemKeyValue.class); xmlObjectReader.setMapper(new XmlMapper()); builder.xmlObjectReader(xmlObjectReader); XmlItemReader> reader = (XmlItemReader) builder.build();