From bdbd2acb0d847891149fc02d521a030dad35faa1 Mon Sep 17 00:00:00 2001 From: Julien Ruaux Date: Wed, 24 May 2023 23:05:58 -0700 Subject: [PATCH] refactor!: Major refactoring to simplify codebase --- .github/workflows/build.yml | 1 + build.gradle | 5 +- .../com/redis/riot/core/FakerItemReader.java | 2 +- .../java/com/redis/riot/core/FileUtils.java | 13 +- .../redis/riot/core/KeyComparisonLogger.java | 214 ++++++++++++ .../redis/riot/core/ThrottledItemReader.java | 56 ++++ .../riot/core/convert/CompositeConverter.java | 27 -- .../core/convert/FieldExtractorFactory.java | 35 +- .../riot/core/convert/IdConverterBuilder.java | 44 +-- .../core/convert/MapFilteringConverter.java | 6 +- .../redis/riot/core/convert/MapFlattener.java | 12 +- .../convert/MapToStringArrayConverter.java | 29 +- .../core/convert/ObjectMapperConverter.java | 7 +- .../core/convert/ObjectToNumberConverter.java | 47 +-- .../core/convert/ObjectToStringConverter.java | 6 +- .../TimeSeriesToStringMapConverter.java | 22 ++ .../processor/DataStructureProcessor.java | 6 +- .../DataStructureToMapProcessor.java | 27 +- .../com/redis/riot/core/ConverterTests.java | 22 +- .../redis/riot/core/XmlItemWriterTests.java | 9 +- gradle.properties | 2 +- jreleaser.yml | 230 ++----------- plugins/{riot => riot-cli}/gradle.properties | 0 .../riot.gradle => riot-cli/riot-cli.gradle} | 19 +- .../java/com/redis/riot/cli/DbExport.java | 47 ++- .../java/com/redis/riot/cli/DbImport.java | 40 ++- .../java/com/redis/riot/cli/DumpImport.java | 36 +-- .../java/com/redis/riot/cli/FakerImport.java | 17 +- .../java/com/redis/riot/cli/FileExport.java | 10 +- .../java/com/redis/riot/cli/FileImport.java | 47 +-- .../java/com/redis/riot/cli/Generate.java | 34 +- .../main/java/com/redis/riot/cli/Main.java} | 6 +- .../main/java/com/redis/riot/cli/Ping.java | 31 +- .../java/com/redis/riot/cli/Replicate.java | 305 ++++++++++++++++++ .../riot/cli/common/AbstractCommand.java | 134 ++++++++ .../cli/common/AbstractExportCommand.java | 34 ++ .../cli/common/AbstractImportCommand.java | 43 ++- .../riot/cli/common/CommandContext.java} | 65 ++-- .../cli/common/DoubleRangeTypeConverter.java | 0 .../redis/riot/cli/common/HelpOptions.java | 0 .../cli/common/IntRangeTypeConverter.java | 0 .../cli/common/KeyComparisonStepListener.java | 48 +++ .../common/KeyComparisonWriteListener.java | 32 ++ .../redis/riot/cli/common/LoggingOptions.java | 63 +--- .../cli/common/ManifestVersionProvider.java | 0 .../riot/cli/common/MapProcessorOptions.java | 6 +- .../redis/riot/cli/common/PingOptions.java | 25 +- .../riot/cli/common/ProgressMonitor.java | 52 ++- .../com/redis/riot/cli/common/ReadFrom.java | 6 + .../redis/riot/cli/common/RedisOptions.java | 10 +- .../riot/cli/common/RedisReaderOptions.java | 74 ++--- .../riot/cli/common/RedisWriterOptions.java | 34 +- .../cli/common/ReplicateCommandContext.java | 72 +++++ .../riot/cli/common/ReplicationOptions.java} | 53 ++- .../cli/common/RiotExecutionStrategy.java | 0 .../riot/cli/common/RuntimeIOException.java | 22 ++ .../redis/riot/cli/common/StepSkipPolicy.java | 0 .../riot/cli/common/TransferOptions.java | 42 +-- .../redis/riot/cli/db/DataSourceOptions.java | 0 .../redis/riot/cli/db/DbExportOptions.java} | 9 +- .../redis/riot/cli/db/DbImportOptions.java} | 12 +- .../cli/db/NullableSqlParameterSource.java | 0 .../redis/riot/cli/file/FileDumpOptions.java | 0 .../riot/cli/file/FileExportOptions.java | 3 +- .../riot/cli/file/FileImportOptions.java | 0 .../com/redis/riot/cli/file/FileOptions.java | 8 +- .../redis/riot/cli/file/FlatFileOptions.java | 5 +- .../com/redis/riot/cli/file/GcsOptions.java | 0 .../com/redis/riot/cli/file/S3Options.java | 0 .../gen/DataStructureGeneratorOptions.java | 8 +- .../riot/cli/gen/FakerGeneratorOptions.java | 7 +- .../redis/riot/cli/gen/GeneratorOptions.java | 14 +- .../operation/AbstractCollectionCommand.java | 5 +- .../cli/operation/AbstractKeyCommand.java | 5 +- .../operation/AbstractOperationCommand.java | 33 +- .../riot/cli/operation/CollectionOptions.java | 9 +- .../redis/riot/cli/operation/DelCommand.java | 17 + .../redis/riot/cli/operation/EvalCommand.java | 6 +- .../redis/riot/cli/operation/EvalOptions.java | 8 +- .../riot/cli/operation/ExpireCommand.java | 4 +- .../riot/cli/operation/ExpireOptions.java | 38 +++ .../riot/cli/operation/FilteringOptions.java | 10 +- .../riot/cli/operation/GeoaddCommand.java | 5 +- .../riot/cli/operation/GeoaddOptions.java | 1 + .../redis/riot/cli/operation/HsetCommand.java | 2 +- .../riot/cli/operation/JsonSetCommand.java | 21 +- .../redis/riot/cli/operation/KeyOptions.java | 9 +- .../riot/cli/operation/LpushCommand.java | 2 +- .../redis/riot/cli/operation/NoopCommand.java | 2 +- .../riot/cli/operation/OperationCommand.java | 9 + .../cli/operation/RedisCommandOptions.java | 12 +- .../riot/cli/operation/RpushCommand.java | 2 +- .../redis/riot/cli/operation/SaddCommand.java | 2 +- .../redis/riot/cli/operation/SetCommand.java | 7 +- .../redis/riot/cli/operation/SetOptions.java | 8 +- .../riot/cli/operation/StringFormat.java | 5 + .../riot/cli/operation/SugaddCommand.java | 14 +- .../riot/cli/operation/SugaddOptions.java | 31 +- .../riot/cli/operation/TsAddCommand.java | 49 +++ .../riot/cli/operation/TsAddOptions.java | 4 +- .../redis/riot/cli/operation/XaddCommand.java | 3 +- .../redis/riot/cli/operation/XaddOptions.java | 4 +- .../redis/riot/cli/operation/ZaddCommand.java | 4 +- .../redis/riot/cli/operation/ZaddOptions.java | 37 +++ .../riot/cli/AbstractDatabaseTestBase.java | 0 .../com/redis/riot/cli/AbstractRiotTests.java | 90 +++--- .../com/redis/riot/cli/AbstractTestBase.java | 110 +++---- .../com/redis/riot/cli/PostgresTests.java | 7 +- .../redis/riot/cli/RedisContainerFactory.java | 0 ...RedisEnterpriseSourceIntegrationTests.java | 0 ...RedisEnterpriseTargetIntegrationTests.java | 0 .../redis/riot/cli/RedisStackRiotTests.java | 0 .../java/com/redis/riot/cli/ScriptRunner.java | 0 .../riot/cli/SynchronizedListItemWriter.java | 22 +- .../redis/riot/cli/TypeConverterTests.java | 0 .../src/test/resources/db-export-postgresql | 0 .../src/test/resources/db-import-postgresql | 0 .../db-import-postgresql-multithreaded | 0 .../test/resources/db-import-postgresql-noop | 0 .../test/resources/db-import-postgresql-set | 0 .../src/test/resources/db/northwind.sql | 0 .../src/test/resources/db/oracle.sql | 0 .../src/test/resources/dump-import | 0 .../src/test/resources/faker-hset | 0 .../src/test/resources/faker-infer | 0 .../src/test/resources/faker-sadd | 0 .../src/test/resources/faker-tsadd | 0 .../src/test/resources/faker-tsadd-options | 0 .../src/test/resources/faker-xadd | 0 .../src/test/resources/faker-zadd | 0 .../src/test/resources/file-export-json | 0 .../src/test/resources/file-export-json-gz | 0 .../src/test/resources/file-export-xml | 0 .../src/test/resources/file-import-bad | 0 .../src/test/resources/file-import-csv | 0 .../src/test/resources/file-import-csv-max | 0 .../test/resources/file-import-csv-skiplines | 0 .../src/test/resources/file-import-exclude | 0 .../src/test/resources/file-import-filter | 0 .../src/test/resources/file-import-fw | 0 .../src/test/resources/file-import-gcs | 0 .../test/resources/file-import-geo-processor | 0 .../src/test/resources/file-import-geoadd | 0 .../src/test/resources/file-import-glob | 0 .../src/test/resources/file-import-include | 0 .../src/test/resources/file-import-json | 0 .../test/resources/file-import-json-elastic | 0 .../file-import-json-elastic-jsonset | 0 .../src/test/resources/file-import-json-gz | 0 .../test/resources/file-import-multi-commands | 0 .../src/test/resources/file-import-process | 0 .../test/resources/file-import-process-elvis | 0 .../src/test/resources/file-import-psv | 0 .../src/test/resources/file-import-regex | 0 .../src/test/resources/file-import-s3 | 0 .../src/test/resources/file-import-sugadd | 0 .../src/test/resources/file-import-tsv | 0 .../src/test/resources/file-import-type | 0 .../src/test/resources/file-import-xml | 0 .../src/test/resources/files/accounts.fw | 0 .../src/test/resources/files/airports.csv | 0 .../src/test/resources/files/bad.psv | 0 .../src/test/resources/files/beers.csv | 0 .../src/test/resources/files/beers.json | 0 .../src/test/resources/files/beers.json.gz | Bin .../src/test/resources/files/beers1.csv | 0 .../src/test/resources/files/beers2.csv | 0 .../test/resources/files/es_test-index.json | 0 .../src/test/resources/files/lacity.csv | 0 .../src/test/resources/files/redis.json | 0 .../src/test/resources/files/sample.dat | 0 .../src/test/resources/files/sample.psv | 0 .../src/test/resources/files/sample.tsv | 0 .../src/test/resources/files/timestamp.json | 0 .../src/test/resources/files/trades.xml | 0 .../src/test/resources/generate | 0 .../src/test/resources/replicate | 0 .../src/test/resources/replicate-dry-run | 0 .../src/test/resources/replicate-ds-live | 1 + .../src/test/resources/replicate-hll | 0 .../test/resources/replicate-key-processor | 0 .../src/test/resources/replicate-live | 0 .../src/test/resources/replicate-live-keyslot | 0 .../src/test/resources/replicate-live-threads | 0 .../java/com/redis/riot/cli/Replicate.java | 231 ------------- .../cli/common/AbstractExportCommand.java | 43 --- .../riot/cli/common/AbstractJobCommand.java | 41 --- .../cli/common/AbstractTargetCommand.java | 168 ---------- .../cli/common/AbstractTransferCommand.java | 70 ---- .../redis/riot/cli/common/CompareOptions.java | 36 --- .../cli/common/FlushingTransferOptions.java | 40 --- .../redis/riot/cli/common/ProgressStyle.java | 5 - .../riot/cli/common/TargetCommandContext.java | 60 ---- .../riot/cli/operation/ExpireOptions.java | 35 -- .../riot/cli/operation/OperationCommand.java | 9 - .../riot/cli/operation/TsAddCommand.java | 52 --- .../redis/riot/cli/operation/ZaddOptions.java | 35 -- .../riot/src/test/resources/replicate-ds-live | 1 - riot | 4 +- settings.gradle | 21 +- 200 files changed, 1875 insertions(+), 1869 deletions(-) create mode 100644 core/riot-core/src/main/java/com/redis/riot/core/KeyComparisonLogger.java create mode 100644 core/riot-core/src/main/java/com/redis/riot/core/ThrottledItemReader.java delete mode 100644 core/riot-core/src/main/java/com/redis/riot/core/convert/CompositeConverter.java create mode 100644 core/riot-core/src/main/java/com/redis/riot/core/convert/TimeSeriesToStringMapConverter.java rename plugins/{riot => riot-cli}/gradle.properties (100%) rename plugins/{riot/riot.gradle => riot-cli/riot-cli.gradle} (96%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/DbExport.java (53%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/DbImport.java (56%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/DumpImport.java (73%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/FakerImport.java (81%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/FileExport.java (92%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/FileImport.java (88%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/Generate.java (67%) rename plugins/{riot/src/main/java/com/redis/riot/cli/Riot.java => riot-cli/src/main/java/com/redis/riot/cli/Main.java} (96%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/Ping.java (78%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/Replicate.java create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractCommand.java create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractExportCommand.java rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/AbstractImportCommand.java (82%) rename plugins/{riot/src/main/java/com/redis/riot/cli/common/JobCommandContext.java => riot-cli/src/main/java/com/redis/riot/cli/common/CommandContext.java} (50%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/DoubleRangeTypeConverter.java (100%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/HelpOptions.java (100%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/IntRangeTypeConverter.java (100%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/common/KeyComparisonStepListener.java create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/common/KeyComparisonWriteListener.java rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/LoggingOptions.java (70%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/ManifestVersionProvider.java (100%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/MapProcessorOptions.java (94%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/PingOptions.java (82%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/ProgressMonitor.java (69%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReadFrom.java rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/RedisOptions.java (97%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/RedisReaderOptions.java (56%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/RedisWriterOptions.java (57%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReplicateCommandContext.java rename plugins/{riot/src/main/java/com/redis/riot/cli/common/ReplicateOptions.java => riot-cli/src/main/java/com/redis/riot/cli/common/ReplicationOptions.java} (61%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/RiotExecutionStrategy.java (100%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RuntimeIOException.java rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/StepSkipPolicy.java (100%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/common/TransferOptions.java (70%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/db/DataSourceOptions.java (100%) rename plugins/{riot/src/main/java/com/redis/riot/cli/db/DatabaseExportOptions.java => riot-cli/src/main/java/com/redis/riot/cli/db/DbExportOptions.java} (75%) rename plugins/{riot/src/main/java/com/redis/riot/cli/db/DatabaseImportOptions.java => riot-cli/src/main/java/com/redis/riot/cli/db/DbImportOptions.java} (83%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/db/NullableSqlParameterSource.java (100%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/file/FileDumpOptions.java (100%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/file/FileExportOptions.java (91%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/file/FileImportOptions.java (100%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/file/FileOptions.java (93%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/file/FlatFileOptions.java (96%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/file/GcsOptions.java (100%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/file/S3Options.java (100%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/gen/DataStructureGeneratorOptions.java (98%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/gen/FakerGeneratorOptions.java (89%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/gen/GeneratorOptions.java (79%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/AbstractCollectionCommand.java (74%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/AbstractKeyCommand.java (79%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/AbstractOperationCommand.java (54%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/CollectionOptions.java (76%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/operation/DelCommand.java rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/EvalCommand.java (82%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/EvalOptions.java (89%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/ExpireCommand.java (72%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/operation/ExpireOptions.java rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/FilteringOptions.java (81%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/GeoaddCommand.java (70%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/GeoaddOptions.java (99%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/HsetCommand.java (87%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/JsonSetCommand.java (60%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/KeyOptions.java (73%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/LpushCommand.java (82%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/NoopCommand.java (88%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/operation/OperationCommand.java rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/RedisCommandOptions.java (71%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/RpushCommand.java (82%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/SaddCommand.java (81%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/SetCommand.java (86%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/SetOptions.java (89%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/operation/StringFormat.java rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/SugaddCommand.java (66%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/SugaddOptions.java (70%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/operation/TsAddCommand.java rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/TsAddOptions.java (92%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/XaddCommand.java (83%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/XaddOptions.java (89%) rename plugins/{riot => riot-cli}/src/main/java/com/redis/riot/cli/operation/ZaddCommand.java (73%) create mode 100644 plugins/riot-cli/src/main/java/com/redis/riot/cli/operation/ZaddOptions.java rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/AbstractDatabaseTestBase.java (100%) rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/AbstractRiotTests.java (92%) rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/AbstractTestBase.java (63%) rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/PostgresTests.java (96%) rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/RedisContainerFactory.java (100%) rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/RedisEnterpriseSourceIntegrationTests.java (100%) rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/RedisEnterpriseTargetIntegrationTests.java (100%) rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/RedisStackRiotTests.java (100%) rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/ScriptRunner.java (100%) rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/SynchronizedListItemWriter.java (52%) rename plugins/{riot => riot-cli}/src/test/java/com/redis/riot/cli/TypeConverterTests.java (100%) rename plugins/{riot => riot-cli}/src/test/resources/db-export-postgresql (100%) rename plugins/{riot => riot-cli}/src/test/resources/db-import-postgresql (100%) rename plugins/{riot => riot-cli}/src/test/resources/db-import-postgresql-multithreaded (100%) rename plugins/{riot => riot-cli}/src/test/resources/db-import-postgresql-noop (100%) rename plugins/{riot => riot-cli}/src/test/resources/db-import-postgresql-set (100%) rename plugins/{riot => riot-cli}/src/test/resources/db/northwind.sql (100%) rename plugins/{riot => riot-cli}/src/test/resources/db/oracle.sql (100%) rename plugins/{riot => riot-cli}/src/test/resources/dump-import (100%) rename plugins/{riot => riot-cli}/src/test/resources/faker-hset (100%) rename plugins/{riot => riot-cli}/src/test/resources/faker-infer (100%) rename plugins/{riot => riot-cli}/src/test/resources/faker-sadd (100%) rename plugins/{riot => riot-cli}/src/test/resources/faker-tsadd (100%) rename plugins/{riot => riot-cli}/src/test/resources/faker-tsadd-options (100%) rename plugins/{riot => riot-cli}/src/test/resources/faker-xadd (100%) rename plugins/{riot => riot-cli}/src/test/resources/faker-zadd (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-export-json (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-export-json-gz (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-export-xml (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-bad (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-csv (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-csv-max (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-csv-skiplines (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-exclude (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-filter (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-fw (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-gcs (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-geo-processor (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-geoadd (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-glob (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-include (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-json (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-json-elastic (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-json-elastic-jsonset (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-json-gz (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-multi-commands (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-process (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-process-elvis (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-psv (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-regex (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-s3 (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-sugadd (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-tsv (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-type (100%) rename plugins/{riot => riot-cli}/src/test/resources/file-import-xml (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/accounts.fw (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/airports.csv (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/bad.psv (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/beers.csv (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/beers.json (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/beers.json.gz (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/beers1.csv (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/beers2.csv (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/es_test-index.json (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/lacity.csv (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/redis.json (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/sample.dat (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/sample.psv (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/sample.tsv (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/timestamp.json (100%) rename plugins/{riot => riot-cli}/src/test/resources/files/trades.xml (100%) rename plugins/{riot => riot-cli}/src/test/resources/generate (100%) rename plugins/{riot => riot-cli}/src/test/resources/replicate (100%) rename plugins/{riot => riot-cli}/src/test/resources/replicate-dry-run (100%) create mode 100644 plugins/riot-cli/src/test/resources/replicate-ds-live rename plugins/{riot => riot-cli}/src/test/resources/replicate-hll (100%) rename plugins/{riot => riot-cli}/src/test/resources/replicate-key-processor (100%) rename plugins/{riot => riot-cli}/src/test/resources/replicate-live (100%) rename plugins/{riot => riot-cli}/src/test/resources/replicate-live-keyslot (100%) rename plugins/{riot => riot-cli}/src/test/resources/replicate-live-threads (100%) delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/Replicate.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/common/AbstractExportCommand.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/common/AbstractJobCommand.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/common/AbstractTargetCommand.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/common/AbstractTransferCommand.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/common/CompareOptions.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/common/FlushingTransferOptions.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/common/ProgressStyle.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/common/TargetCommandContext.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/operation/ExpireOptions.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/operation/OperationCommand.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/operation/TsAddCommand.java delete mode 100644 plugins/riot/src/main/java/com/redis/riot/cli/operation/ZaddOptions.java delete mode 100644 plugins/riot/src/test/resources/replicate-ds-live diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ebcd5077f..4f9cce1aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.os }} + if: startsWith(github.event.head_commit.message, 'Releasing version') != true steps: - uses: actions/checkout@v3 diff --git a/build.gradle b/build.gradle index 1249d667c..22e5d375e 100644 --- a/build.gradle +++ b/build.gradle @@ -96,9 +96,10 @@ subprojects { subproj -> dependencies { compileOnly group: 'com.google.code.findbugs', name: 'jsr305', version: jsr305Version - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' testImplementation group: 'commons-io', name: 'commons-io', version: commonsIoVersion - testImplementation(group: 'com.redis.testcontainers', name: 'testcontainers-redis-junit', version: testcontainersRedisVersion) { + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' + testImplementation group: 'org.testcontainers', name: 'junit-jupiter', version: testcontainersVersion + testImplementation(group: 'com.redis.testcontainers', name: 'testcontainers-redis', version: testcontainersRedisVersion) { exclude group: 'com.redis', module: 'lettucemod' } } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/FakerItemReader.java b/core/riot-core/src/main/java/com/redis/riot/core/FakerItemReader.java index 5daeb26da..9f16af86c 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/FakerItemReader.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/FakerItemReader.java @@ -22,7 +22,7 @@ public class FakerItemReader extends AbstractItemCountingItemStreamItemReader> generator; public FakerItemReader(Generator> generator) { - setName(ClassUtils.getShortName(FakerItemReader.class)); + setName(ClassUtils.getShortName(getClass())); Assert.notNull(generator, "A generator is required"); setMaxItemCount(count); this.generator = generator; diff --git a/core/riot-core/src/main/java/com/redis/riot/core/FileUtils.java b/core/riot-core/src/main/java/com/redis/riot/core/FileUtils.java index a76519040..aca72fe11 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/FileUtils.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/FileUtils.java @@ -7,8 +7,10 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -25,12 +27,18 @@ import com.redis.riot.core.resource.XmlItemReaderBuilder; import com.redis.riot.core.resource.XmlObjectReader; -public interface FileUtils { +public class FileUtils { + + private static Logger log = Logger.getLogger(FileUtils.class.getName()); public static final String GS_URI_PREFIX = "gs://"; public static final String S3_URI_PREFIX = "s3://"; public static final Pattern EXTENSION_PATTERN = Pattern.compile("(?i)\\.(?\\w+)(?:\\.(?gz))?$"); + + private FileUtils() { + + } public static boolean isGzip(String file) { return extensionGroup(file, "gz").isPresent(); @@ -117,7 +125,8 @@ public static List expand(Path path) { stream.iterator().forEachRemaining(paths::add); return paths; } catch (IOException e) { - throw new RuntimeException("Could not expand file " + path, e); + log.severe("Could not expand path " + path); + return Collections.emptyList(); } } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/KeyComparisonLogger.java b/core/riot-core/src/main/java/com/redis/riot/core/KeyComparisonLogger.java new file mode 100644 index 000000000..39f166fac --- /dev/null +++ b/core/riot-core/src/main/java/com/redis/riot/core/KeyComparisonLogger.java @@ -0,0 +1,214 @@ +package com.redis.riot.core; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import com.redis.spring.batch.common.DataStructure; +import com.redis.spring.batch.reader.KeyComparison; + +import io.lettuce.core.ScoredValue; +import io.lettuce.core.StreamMessage; + +@SuppressWarnings("unchecked") +public class KeyComparisonLogger { + + /** + * Represents a failed index search. + * + */ + public static final int INDEX_NOT_FOUND = -1; + + private final Logger log; + + public KeyComparisonLogger() { + this(Logger.getLogger(KeyComparisonLogger.class.getName())); + } + + public KeyComparisonLogger(Logger logger) { + this.log = logger; + } + + public void log(KeyComparison comparison) { + switch (comparison.getStatus()) { + case MISSING: + log.log(Level.WARNING, "Missing key {0}", comparison.getSource().getKey()); + break; + case TTL: + log.log(Level.WARNING, "TTL mismatch for key {0}: {1} <> {2}", + new Object[] { comparison.getSource().getKey(), comparison.getSource().getTtl(), + comparison.getTarget().getTtl() }); + break; + case TYPE: + log.log(Level.WARNING, "Type mismatch for key {0}: {1} <> {2}", + new Object[] { comparison.getSource().getKey(), comparison.getSource().getType(), + comparison.getTarget().getType() }); + break; + case VALUE: + switch (comparison.getSource().getType()) { + case DataStructure.SET: + showSetDiff(comparison); + break; + case DataStructure.LIST: + showListDiff(comparison); + break; + case DataStructure.ZSET: + showSortedSetDiff(comparison); + break; + case DataStructure.STREAM: + showStreamDiff(comparison); + break; + case DataStructure.STRING: + case DataStructure.JSON: + showStringDiff(comparison); + break; + case DataStructure.HASH: + showHashDiff(comparison); + break; + case DataStructure.TIMESERIES: + showListDiff(comparison); + break; + default: + log.log(Level.WARNING, "Value mismatch for key '{}'", comparison.getSource().getKey()); + break; + } + break; + case OK: + break; + } + } + + private void showHashDiff(KeyComparison comparison) { + Map sourceHash = (Map) comparison.getSource().getValue(); + Map targetHash = (Map) comparison.getTarget().getValue(); + Map diff = new HashMap<>(); + diff.putAll(sourceHash); + diff.putAll(targetHash); + diff.entrySet() + .removeAll(sourceHash.size() <= targetHash.size() ? sourceHash.entrySet() : targetHash.entrySet()); + log.log(Level.WARNING, "Value mismatch for hash {0} on fields: {1}", + new Object[] { comparison.getSource().getKey(), diff.keySet() }); + } + + private void showStringDiff(KeyComparison comparison) { + String sourceString = (String) comparison.getSource().getValue(); + String targetString = (String) comparison.getTarget().getValue(); + int diffIndex = indexOfDifference(sourceString, targetString); + log.log(Level.WARNING, "Value mismatch for string {0} at offset {1}", + new Object[] { comparison.getSource().getKey(), diffIndex }); + } + + /** + *

+ * Compares two CharSequences, and returns the index at which the CharSequences + * begin to differ. + *

+ * + *

+ * For example, {@code indexOfDifference("i am a machine", "i am a robot") -> 7} + *

+ * + * 
+	 * StringUtils.indexOfDifference(null, null) = -1
+	 * StringUtils.indexOfDifference("", "") = -1
+	 * StringUtils.indexOfDifference("", "abc") = 0
+	 * StringUtils.indexOfDifference("abc", "") = 0
+	 * StringUtils.indexOfDifference("abc", "abc") = -1
+	 * StringUtils.indexOfDifference("ab", "abxyz") = 2
+	 * StringUtils.indexOfDifference("abcde", "abxyz") = 2
+	 * StringUtils.indexOfDifference("abcde", "xyz") = 0
+	 *
+ * + * @param cs1 the first CharSequence, may be null + * @param cs2 the second CharSequence, may be null + * @return the index where cs1 and cs2 begin to differ; -1 if they are equal + * @since 2.0 + * @since 3.0 Changed signature from indexOfDifference(String, String) to + * indexOfDifference(CharSequence, CharSequence) + */ + private static int indexOfDifference(final CharSequence cs1, final CharSequence cs2) { + if (cs1 == cs2) { + return INDEX_NOT_FOUND; + } + if (cs1 == null || cs2 == null) { + return 0; + } + int i; + for (i = 0; i < cs1.length() && i < cs2.length(); ++i) { + if (cs1.charAt(i) != cs2.charAt(i)) { + break; + } + } + if (i < cs2.length() || i < cs1.length()) { + return i; + } + return INDEX_NOT_FOUND; + } + + private void showListDiff(KeyComparison comparison) { + List sourceList = (List) comparison.getSource().getValue(); + List targetList = (List) comparison.getTarget().getValue(); + if (sourceList.size() != targetList.size()) { + log.log(Level.WARNING, "Size mismatch for {0} {1}: {2} <> {3}", + new Object[] { comparison.getSource().getType(), comparison.getSource().getKey(), sourceList.size(), + targetList.size() }); + return; + } + List diff = new ArrayList<>(); + for (int index = 0; index < sourceList.size(); index++) { + if (!sourceList.get(index).equals(targetList.get(index))) { + diff.add(index); + } + } + log.log(Level.WARNING, "Value mismatch for {0} {1} at indexes {2}", + new Object[] { comparison.getSource().getType(), comparison.getSource().getKey(), diff }); + } + + private void showSetDiff(KeyComparison comparison) { + Set sourceSet = (Set) comparison.getSource().getValue(); + Set targetSet = (Set) comparison.getTarget().getValue(); + Set missing = new HashSet<>(sourceSet); + missing.removeAll(targetSet); + Set extra = new HashSet<>(targetSet); + extra.removeAll(sourceSet); + log.log(Level.WARNING, "Value mismatch for set {0}: {1} <> {2}", + new Object[] { comparison.getSource().getKey(), missing, extra }); + } + + private void showSortedSetDiff(KeyComparison comparison) { + List> sourceList = (List>) comparison.getSource().getValue(); + List> targetList = (List>) comparison.getTarget().getValue(); + List> missing = new ArrayList<>(sourceList); + missing.removeAll(targetList); + List> extra = new ArrayList<>(targetList); + extra.removeAll(sourceList); + log.log(Level.WARNING, "Value mismatch for sorted set {0}: {1} <> {2}", + new Object[] { comparison.getSource().getKey(), print(missing), print(extra) }); + } + + private List print(List> list) { + return list.stream().map(v -> v.getValue() + "@" + v.getScore()).collect(Collectors.toList()); + } + + private void showStreamDiff(KeyComparison comparison) { + List> sourceMessages = (List>) comparison + .getSource().getValue(); + List> targetMessages = (List>) comparison + .getTarget().getValue(); + List> missing = new ArrayList<>(sourceMessages); + missing.removeAll(targetMessages); + List> extra = new ArrayList<>(targetMessages); + extra.removeAll(sourceMessages); + log.log(Level.WARNING, "Value mismatch for stream {0}: {1} <> {2}", + new Object[] { comparison.getSource().getKey(), + missing.stream().map(StreamMessage::getId).collect(Collectors.toList()), + extra.stream().map(StreamMessage::getId).collect(Collectors.toList()) }); + } + +} \ No newline at end of file diff --git a/core/riot-core/src/main/java/com/redis/riot/core/ThrottledItemReader.java b/core/riot-core/src/main/java/com/redis/riot/core/ThrottledItemReader.java new file mode 100644 index 000000000..dc18925e6 --- /dev/null +++ b/core/riot-core/src/main/java/com/redis/riot/core/ThrottledItemReader.java @@ -0,0 +1,56 @@ +package com.redis.riot.core; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemStreamReader; +import org.springframework.batch.item.ItemStreamSupport; +import org.springframework.util.Assert; +import org.springframework.util.ClassUtils; + +import com.redis.spring.batch.common.DelegatingItemStreamSupport; +import com.redis.spring.batch.reader.PollableItemReader; + +public class ThrottledItemReader extends DelegatingItemStreamSupport + implements ItemStreamReader, PollableItemReader { + + private final ItemReader delegate; + private final long sleep; + + public ThrottledItemReader(ItemReader delegate, Duration sleepDuration) { + super(delegate); + setName(ClassUtils.getShortName(getClass())); + Assert.notNull(delegate, "Reader delegate must not be null"); + Assert.notNull(sleepDuration, "Sleep duration must not be null"); + Assert.isTrue(!sleepDuration.isNegative() && !sleepDuration.isZero(), + "Sleep duration must be strictly positive"); + this.delegate = delegate; + this.sleep = sleepDuration.toMillis(); + } + + @Override + public void setName(String name) { + super.setName(name); + if (delegate instanceof ItemStreamSupport) { + ((ItemStreamSupport) delegate).setName(name); + } + } + + @Override + public T read() throws Exception { + sleep(); + return delegate.read(); + } + + @Override + public T poll(long timeout, TimeUnit unit) throws InterruptedException, PollingException { + sleep(); + return ((PollableItemReader) delegate).poll(timeout, unit); + } + + private void sleep() throws InterruptedException { + Thread.sleep(sleep); + } + +} diff --git a/core/riot-core/src/main/java/com/redis/riot/core/convert/CompositeConverter.java b/core/riot-core/src/main/java/com/redis/riot/core/convert/CompositeConverter.java deleted file mode 100644 index 724769238..000000000 --- a/core/riot-core/src/main/java/com/redis/riot/core/convert/CompositeConverter.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.redis.riot.core.convert; - -import org.springframework.core.convert.converter.Converter; - -public class CompositeConverter implements Converter { - - private final Converter sourceConverter; - private final Converter targetConverter; - - public CompositeConverter(Converter sourceConverter, Converter targetConverter) { - this.sourceConverter = sourceConverter; - this.targetConverter = targetConverter; - } - - @Override - public T convert(S source) { - if (source == null) { - return null; - } - P intermediary = sourceConverter.convert(source); - if (intermediary == null) { - return null; - } - return targetConverter.convert(intermediary); - } - -} diff --git a/core/riot-core/src/main/java/com/redis/riot/core/convert/FieldExtractorFactory.java b/core/riot-core/src/main/java/com/redis/riot/core/convert/FieldExtractorFactory.java index 0fc0a4991..20023af4b 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/convert/FieldExtractorFactory.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/convert/FieldExtractorFactory.java @@ -1,8 +1,7 @@ package com.redis.riot.core.convert; import java.util.Map; - -import org.springframework.core.convert.converter.Converter; +import java.util.function.Function; public class FieldExtractorFactory { @@ -17,26 +16,26 @@ public void setNullCheck(boolean nullCheck) { this.nullCheck = nullCheck; } - public Converter, Object> field(String field) { - Converter, Object> extractor = extractor(field); + public Function, Object> field(String field) { + Function, Object> extractor = extractor(field); if (nullCheck) { return new NullCheckExtractor(field, extractor); } return extractor; } - private Converter, T> extractor(String field) { + private Function, T> extractor(String field) { if (remove) { return s -> s.remove(field); } return s -> s.get(field); } - public Converter, String> string(String field) { - return new CompositeConverter<>(field(field), new ObjectToStringConverter()); + public Function, String> string(String field) { + return field(field).andThen(new ObjectToStringConverter()); } - public Converter, T> field(String field, T defaultValue) { + public Function, T> field(String field, T defaultValue) { return new DefaultValueExtractor<>(extractor(field), defaultValue); } @@ -50,19 +49,19 @@ public MissingFieldException(String msg) { } - private static class DefaultValueExtractor implements Converter, T> { + private static class DefaultValueExtractor implements Function, T> { - private final Converter, T> extractor; + private final Function, T> extractor; private final T defaultValue; - public DefaultValueExtractor(Converter, T> extractor, T defaultValue) { + public DefaultValueExtractor(Function, T> extractor, T defaultValue) { this.extractor = extractor; this.defaultValue = defaultValue; } @Override - public T convert(Map source) { - T value = extractor.convert(source); + public T apply(Map source) { + T value = extractor.apply(source); if (value == null) { return defaultValue; } @@ -71,19 +70,19 @@ public T convert(Map source) { } } - private static class NullCheckExtractor implements Converter, Object> { + private static class NullCheckExtractor implements Function, Object> { private final String field; - private final Converter, Object> extractor; + private final Function, Object> extractor; - public NullCheckExtractor(String field, Converter, Object> extractor) { + public NullCheckExtractor(String field, Function, Object> extractor) { this.field = field; this.extractor = extractor; } @Override - public Object convert(Map source) { - Object value = extractor.convert(source); + public Object apply(Map source) { + Object value = extractor.apply(source); if (value == null) { throw new MissingFieldException("Error: Missing required field: '" + field + "'"); } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/convert/IdConverterBuilder.java b/core/riot-core/src/main/java/com/redis/riot/core/convert/IdConverterBuilder.java index 565135bc4..ff5a373b4 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/convert/IdConverterBuilder.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/convert/IdConverterBuilder.java @@ -3,15 +3,15 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; - -import org.springframework.core.convert.converter.Converter; +import java.util.Optional; +import java.util.function.Function; public class IdConverterBuilder { public static final String DEFAULT_SEPARATOR = ":"; private String separator = DEFAULT_SEPARATOR; - private String prefix; + private Optional prefix = Optional.empty(); private final FieldExtractorFactory extractorFactory = FieldExtractorFactory.builder().nullCheck(true).build(); private final List fields = new ArrayList<>(); @@ -30,6 +30,10 @@ public IdConverterBuilder fields(String... fields) { } public IdConverterBuilder prefix(String prefix) { + return prefix(Optional.of(prefix)); + } + + public IdConverterBuilder prefix(Optional prefix) { this.prefix = prefix; return this; } @@ -39,50 +43,48 @@ public IdConverterBuilder separator(String separator) { return this; } - public Converter, String> build() { + public Function, String> build() { if (fields.isEmpty()) { - if (prefix == null) { - throw new IllegalArgumentException("No prefix and no fields specified"); + if (prefix.isPresent()) { + return m -> prefix.get(); } - return s -> prefix; + throw new IllegalArgumentException("No prefix and no fields specified"); } if (fields.size() == 1) { - Converter, String> extractor = extractorFactory.string(fields.get(0)); - if (prefix == null) { - return extractor::convert; + Function, String> extractor = extractorFactory.string(fields.get(0)); + if (prefix.isPresent()) { + return s -> prefix.get() + separator + extractor.apply(s); } - return s -> prefix + separator + extractor.convert(s); - } - List, String>> stringConverters = new ArrayList<>(); - if (prefix != null) { - stringConverters.add(s -> prefix); + return extractor::apply; } + List, String>> stringConverters = new ArrayList<>(); + prefix.ifPresent(p -> stringConverters.add(s -> p)); for (String field : fields) { stringConverters.add(extractorFactory.string(field)); } return new ConcatenatingConverter(separator, stringConverters); } - public static class ConcatenatingConverter implements Converter, String> { + public static class ConcatenatingConverter implements Function, String> { private final String separator; - private final List, String>> converters; + private final List, String>> converters; - public ConcatenatingConverter(String separator, List, String>> stringConverters) { + public ConcatenatingConverter(String separator, List, String>> stringConverters) { this.separator = separator; this.converters = stringConverters; } @Override - public String convert(Map source) { + public String apply(Map source) { if (source == null) { return null; } StringBuilder builder = new StringBuilder(); - builder.append(converters.get(0).convert(source)); + builder.append(converters.get(0).apply(source)); for (int index = 1; index < converters.size(); index++) { builder.append(separator); - builder.append(converters.get(index).convert(source)); + builder.append(converters.get(index).apply(source)); } return builder.toString(); } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/convert/MapFilteringConverter.java b/core/riot-core/src/main/java/com/redis/riot/core/convert/MapFilteringConverter.java index 84ad3bc4f..6f9cf8e89 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/convert/MapFilteringConverter.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/convert/MapFilteringConverter.java @@ -7,12 +7,12 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.UnaryOperator; -import org.springframework.core.convert.converter.Converter; import org.springframework.util.Assert; import org.springframework.util.ObjectUtils; -public class MapFilteringConverter implements Converter, Map> { +public class MapFilteringConverter implements UnaryOperator> { private final Set includes; private final Set excludes; @@ -23,7 +23,7 @@ public MapFilteringConverter(Set includes, Set excludes) { } @Override - public Map convert(Map source) { + public Map apply(Map source) { Map filtered = ObjectUtils.isEmpty(includes) ? source : new LinkedHashMap<>(); includes.forEach(f -> filtered.put(f, source.get(f))); excludes.forEach(filtered::remove); diff --git a/core/riot-core/src/main/java/com/redis/riot/core/convert/MapFlattener.java b/core/riot-core/src/main/java/com/redis/riot/core/convert/MapFlattener.java index c8134a538..297697f36 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/convert/MapFlattener.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/convert/MapFlattener.java @@ -4,8 +4,8 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.function.Function; -import org.springframework.core.convert.converter.Converter; import org.springframework.lang.Nullable; import org.springframework.util.StringUtils; @@ -13,16 +13,16 @@ * Flattens a nested map using . and [] notation for key names * */ -public class MapFlattener implements Converter, Map> { +public class MapFlattener implements Function, Map> { - private final Converter elementConverter; + private final Function elementConverter; - public MapFlattener(Converter elementConverter) { + public MapFlattener(Function elementConverter) { this.elementConverter = elementConverter; } @Override - public Map convert(Map source) { + public Map apply(Map source) { Map resultMap = new LinkedHashMap<>(); flatten("", source.entrySet().iterator(), resultMap); return resultMap; @@ -50,7 +50,7 @@ private void flattenElement(String propertyPrefix, @Nullable Object source, Map< } else if (source instanceof Map) { flatten(propertyPrefix, ((Map) source).entrySet().iterator(), flatMap); } else { - ((Map) flatMap).put(propertyPrefix, elementConverter.convert(source)); + ((Map) flatMap).put(propertyPrefix, elementConverter.apply(source)); } } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/convert/MapToStringArrayConverter.java b/core/riot-core/src/main/java/com/redis/riot/core/convert/MapToStringArrayConverter.java index 159910d29..57a339c2a 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/convert/MapToStringArrayConverter.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/convert/MapToStringArrayConverter.java @@ -1,24 +1,23 @@ package com.redis.riot.core.convert; import java.util.Map; +import java.util.function.Function; -import org.springframework.core.convert.converter.Converter; +public class MapToStringArrayConverter implements Function, String[]> { -public class MapToStringArrayConverter implements Converter, String[]> { + private final Function, String>[] fieldConverters; - private final Converter, String>[] fieldConverters; + public MapToStringArrayConverter(Function, String>[] fieldConverters) { + this.fieldConverters = fieldConverters; + } - public MapToStringArrayConverter(Converter, String>[] fieldConverters) { - this.fieldConverters = fieldConverters; - } - - @Override - public String[] convert(Map source) { - String[] array = new String[fieldConverters.length]; - for (int index = 0; index < fieldConverters.length; index++) { - array[index] = fieldConverters[index].convert(source); - } - return array; - } + @Override + public String[] apply(Map source) { + String[] array = new String[fieldConverters.length]; + for (int index = 0; index < fieldConverters.length; index++) { + array[index] = fieldConverters[index].apply(source); + } + return array; + } } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectMapperConverter.java b/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectMapperConverter.java index a7719c39d..db29a8443 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectMapperConverter.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectMapperConverter.java @@ -1,13 +1,14 @@ package com.redis.riot.core.convert; +import java.util.function.Function; + import org.springframework.core.convert.ConversionFailedException; import org.springframework.core.convert.TypeDescriptor; -import org.springframework.core.convert.converter.Converter; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectWriter; -public class ObjectMapperConverter implements Converter { +public class ObjectMapperConverter implements Function { private final ObjectWriter writer; @@ -16,7 +17,7 @@ public ObjectMapperConverter(ObjectWriter writer) { } @Override - public String convert(T source) { + public String apply(T source) { try { return writer.writeValueAsString(source); } catch (JsonProcessingException e) { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectToNumberConverter.java b/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectToNumberConverter.java index 3f554b8b6..badd48038 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectToNumberConverter.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectToNumberConverter.java @@ -1,32 +1,33 @@ package com.redis.riot.core.convert; -import org.springframework.core.convert.converter.Converter; +import java.util.function.Function; + import org.springframework.util.NumberUtils; -public class ObjectToNumberConverter implements Converter { +public class ObjectToNumberConverter implements Function { - private final Class targetType; + private final Class targetType; - public ObjectToNumberConverter(Class targetType) { - this.targetType = targetType; - } + public ObjectToNumberConverter(Class targetType) { + this.targetType = targetType; + } - @Override - public T convert(Object source) { - if (source == null) { - return null; - } - if (source instanceof Number) { - return NumberUtils.convertNumberToTargetClass((Number) source, targetType); - } - if (source instanceof String) { - String string = (String) source; - if (string.isEmpty()) { - return null; - } - return NumberUtils.parseNumber(string, targetType); - } - return null; - } + @Override + public T apply(Object source) { + if (source == null) { + return null; + } + if (source instanceof Number) { + return NumberUtils.convertNumberToTargetClass((Number) source, targetType); + } + if (source instanceof String) { + String string = (String) source; + if (string.isEmpty()) { + return null; + } + return NumberUtils.parseNumber(string, targetType); + } + return null; + } } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectToStringConverter.java b/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectToStringConverter.java index 8fbfd4b22..45973510a 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectToStringConverter.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/convert/ObjectToStringConverter.java @@ -1,11 +1,11 @@ package com.redis.riot.core.convert; -import org.springframework.core.convert.converter.Converter; +import java.util.function.Function; -public class ObjectToStringConverter implements Converter { +public class ObjectToStringConverter implements Function { @Override - public String convert(Object source) { + public String apply(Object source) { if (source == null) { return null; } diff --git a/core/riot-core/src/main/java/com/redis/riot/core/convert/TimeSeriesToStringMapConverter.java b/core/riot-core/src/main/java/com/redis/riot/core/convert/TimeSeriesToStringMapConverter.java new file mode 100644 index 000000000..fffa19695 --- /dev/null +++ b/core/riot-core/src/main/java/com/redis/riot/core/convert/TimeSeriesToStringMapConverter.java @@ -0,0 +1,22 @@ +package com.redis.riot.core.convert; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.core.convert.converter.Converter; + +import com.redis.lettucemod.timeseries.Sample; + +public class TimeSeriesToStringMapConverter implements Converter, Map> { + + @Override + public Map convert(List source) { + Map result = new HashMap<>(); + for (int index = 0; index < source.size(); index++) { + Sample sample = source.get(index); + result.put(String.valueOf(index), sample.getTimestamp() + ":" + sample.getValue()); + } + return result; + } +} diff --git a/core/riot-core/src/main/java/com/redis/riot/core/processor/DataStructureProcessor.java b/core/riot-core/src/main/java/com/redis/riot/core/processor/DataStructureProcessor.java index 08d3dfad5..d09994f83 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/processor/DataStructureProcessor.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/processor/DataStructureProcessor.java @@ -7,7 +7,6 @@ import org.springframework.batch.item.ItemProcessor; import com.redis.spring.batch.common.DataStructure; -import com.redis.spring.batch.common.DataStructure.Type; import io.lettuce.core.ScoredValue; import io.lettuce.core.StreamMessage; @@ -20,8 +19,7 @@ public DataStructure process(DataStructure item) throws Exceptio if (item.getType() == null) { return item; } - Type type = item.getType(); - if (type == Type.ZSET) { + if (DataStructure.ZSET.equals(item.getType())) { Collection> zset = (Collection>) item.getValue(); Collection> values = new ArrayList<>(zset.size()); for (Map map : zset) { @@ -32,7 +30,7 @@ public DataStructure process(DataStructure item) throws Exceptio item.setValue(values); return item; } - if (type == Type.STREAM) { + if (DataStructure.STREAM.equals(item.getType())) { Collection> stream = (Collection>) item.getValue(); Collection> messages = new ArrayList<>(stream.size()); for (Map message : stream) { diff --git a/core/riot-core/src/main/java/com/redis/riot/core/processor/DataStructureToMapProcessor.java b/core/riot-core/src/main/java/com/redis/riot/core/processor/DataStructureToMapProcessor.java index 8f5ebd030..3be6d43d0 100644 --- a/core/riot-core/src/main/java/com/redis/riot/core/processor/DataStructureToMapProcessor.java +++ b/core/riot-core/src/main/java/com/redis/riot/core/processor/DataStructureToMapProcessor.java @@ -8,13 +8,14 @@ import org.springframework.batch.item.ItemProcessor; import org.springframework.core.convert.converter.Converter; +import com.redis.lettucemod.timeseries.Sample; import com.redis.riot.core.convert.CollectionToStringMapConverter; import com.redis.riot.core.convert.RegexNamedGroupsExtractor; import com.redis.riot.core.convert.StreamToStringMapConverter; import com.redis.riot.core.convert.StringToStringMapConverter; +import com.redis.riot.core.convert.TimeSeriesToStringMapConverter; import com.redis.riot.core.convert.ZsetToStringMapConverter; import com.redis.spring.batch.common.DataStructure; -import com.redis.spring.batch.common.DataStructure.Type; import io.lettuce.core.ScoredValue; import io.lettuce.core.StreamMessage; @@ -23,10 +24,12 @@ public class DataStructureToMapProcessor implements ItemProcessor> keyFieldsExtractor; private Converter, Map> hashConverter = s -> s; + private TimeSeriesToStringMapConverter tsConverter = new TimeSeriesToStringMapConverter(); private StreamToStringMapConverter streamConverter = new StreamToStringMapConverter(); private CollectionToStringMapConverter listConverter = new CollectionToStringMapConverter(); private CollectionToStringMapConverter setConverter = new CollectionToStringMapConverter(); private ZsetToStringMapConverter zsetConverter = new ZsetToStringMapConverter(); + private Converter> jsonConverter = new StringToStringMapConverter(); private Converter> stringConverter = new StringToStringMapConverter(); private Converter> defaultConverter = s -> null; @@ -84,23 +87,23 @@ public Map process(DataStructure item) throws Exception @SuppressWarnings("unchecked") private Map map(DataStructure item) { - Type type = item.getType(); - if (type == null) { - return defaultConverter.convert(item.getValue()); - } - switch (type) { - case HASH: + switch (item.getType()) { + case DataStructure.HASH: return hashConverter.convert((Map) item.getValue()); - case LIST: + case DataStructure.LIST: return listConverter.convert((List) item.getValue()); - case SET: + case DataStructure.SET: return setConverter.convert((Set) item.getValue()); - case ZSET: + case DataStructure.ZSET: return zsetConverter.convert((List>) item.getValue()); - case STREAM: + case DataStructure.STREAM: return streamConverter.convert((List>) item.getValue()); - case STRING: + case DataStructure.JSON: + return jsonConverter.convert((String) item.getValue()); + case DataStructure.STRING: return stringConverter.convert((String) item.getValue()); + case DataStructure.TIMESERIES: + return tsConverter.convert((List) item.getValue()); default: return defaultConverter.convert(item.getValue()); } diff --git a/core/riot-core/src/test/java/com/redis/riot/core/ConverterTests.java b/core/riot-core/src/test/java/com/redis/riot/core/ConverterTests.java index ec3f8cf39..2509d2283 100644 --- a/core/riot-core/src/test/java/com/redis/riot/core/ConverterTests.java +++ b/core/riot-core/src/test/java/com/redis/riot/core/ConverterTests.java @@ -2,10 +2,10 @@ import java.util.HashMap; import java.util.Map; +import java.util.function.Function; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.springframework.core.convert.converter.Converter; import com.redis.riot.core.convert.FieldExtractorFactory.MissingFieldException; import com.redis.riot.core.convert.IdConverterBuilder; @@ -16,12 +16,12 @@ class ConverterTests { void testNoKeyConverter() { String prefix = "beer"; String idField = "id"; - Converter, String> keyMaker = new IdConverterBuilder().prefix(prefix).build(); + Function, String> keyMaker = new IdConverterBuilder().prefix(prefix).build(); Map map = new HashMap<>(); String id = "123"; map.put(idField, id); map.put("name", "La fin du monde"); - String key = keyMaker.convert(map); + String key = keyMaker.apply(map); Assertions.assertEquals(prefix, key); } @@ -29,13 +29,13 @@ void testNoKeyConverter() { void testSingleKeyConverter() { String prefix = "beer"; String idField = "id"; - Converter, String> keyMaker = new IdConverterBuilder().prefix(prefix).fields(idField) + Function, String> keyMaker = new IdConverterBuilder().prefix(prefix).fields(idField) .build(); Map map = new HashMap<>(); String id = "123"; map.put(idField, id); map.put("name", "La fin du monde"); - String key = keyMaker.convert(map); + String key = keyMaker.apply(map); Assertions.assertEquals(prefix + IdConverterBuilder.DEFAULT_SEPARATOR + id, key); } @@ -50,10 +50,10 @@ void testMultiKeyConverter() { map.put("name", "La fin du monde"); Assertions.assertEquals( prefix + IdConverterBuilder.DEFAULT_SEPARATOR + store + IdConverterBuilder.DEFAULT_SEPARATOR + sku, - new IdConverterBuilder().prefix(prefix).fields("store", "sku").build().convert(map)); + new IdConverterBuilder().prefix(prefix).fields("store", "sku").build().apply(map)); String separator = "~][]:''~"; - Assertions.assertEquals(prefix + separator + store + separator + sku, new IdConverterBuilder().prefix(prefix) - .separator(separator).fields("store", "sku").build().convert(map)); + Assertions.assertEquals(prefix + separator + store + separator + sku, + new IdConverterBuilder().prefix(prefix).separator(separator).fields("store", "sku").build().apply(map)); } @Test @@ -64,9 +64,9 @@ void testNullCheck() { map.put("store", store); map.put("sku", null); map.put("name", "La fin du monde"); - Converter, String> converter = new IdConverterBuilder().prefix(prefix) - .fields("store", "sku").build(); - Assertions.assertThrows(MissingFieldException.class, () -> converter.convert(map)); + Function, String> converter = new IdConverterBuilder().prefix(prefix).fields("store", "sku") + .build(); + Assertions.assertThrows(MissingFieldException.class, () -> converter.apply(map)); } } diff --git a/core/riot-core/src/test/java/com/redis/riot/core/XmlItemWriterTests.java b/core/riot-core/src/test/java/com/redis/riot/core/XmlItemWriterTests.java index a45acaa3e..7219cd77f 100644 --- a/core/riot-core/src/test/java/com/redis/riot/core/XmlItemWriterTests.java +++ b/core/riot-core/src/test/java/com/redis/riot/core/XmlItemWriterTests.java @@ -17,7 +17,6 @@ import com.redis.riot.core.resource.XmlResourceItemWriter; import com.redis.riot.core.resource.XmlResourceItemWriterBuilder; import com.redis.spring.batch.common.DataStructure; -import com.redis.spring.batch.common.DataStructure.Type; class XmlItemWriterTests { @@ -37,13 +36,13 @@ void test() throws Exception { DataStructure item1 = new DataStructure<>(); item1.setKey("key1"); item1.setTtl(123l); - item1.setType(Type.HASH); + item1.setType(DataStructure.HASH); Map hash1 = Map.of("field1", "value1", "field2", "value2"); item1.setValue(hash1); DataStructure item2 = new DataStructure<>(); item2.setKey("key2"); item2.setTtl(456l); - item2.setType(Type.STREAM); + item2.setType(DataStructure.STREAM); Map hash2 = Map.of("field1", "value1", "field2", "value2"); item2.setValue(hash2); writer.write(Arrays.asList(item1, item2)); @@ -55,8 +54,8 @@ void test() throws Exception { Assertions.assertEquals(item2.getKey(), keyValues.get(1).getKey()); Assertions.assertEquals(item1.getTtl(), keyValues.get(0).getTtl()); Assertions.assertEquals(item2.getTtl(), keyValues.get(1).getTtl()); - Assertions.assertEquals(item1.getValue(), keyValues.get(0).getValue()); - Assertions.assertEquals(item2.getValue(), keyValues.get(1).getValue()); + Assertions.assertEquals((Object) item1.getValue(), keyValues.get(0).getValue()); + Assertions.assertEquals((Object) item2.getValue(), keyValues.get(1).getValue()); } diff --git a/gradle.properties b/gradle.properties index a6b1be204..57a2eec4c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -27,7 +27,7 @@ ociBuildCacheVersion = 0.10.0 awaitilityVersion = 4.2.0 awsVersion = 2.2.6.RELEASE -batchRedisVersion = 3.2.2-SNAPSHOT +batchRedisVersion = 3.2.4-SNAPSHOT commonsIoVersion = 2.11.0 db2Version = 11.5.8.0 datafakerVersion = 1.9.0 diff --git a/jreleaser.yml b/jreleaser.yml index 299dc9d6f..d7fa25ce8 100644 --- a/jreleaser.yml +++ b/jreleaser.yml @@ -1,29 +1,28 @@ environment: properties: jdkPathPrefix: 'plugins/riot/build/jdks' - jdkFilePrefix: 'zulu17.32.13-ca-jdk17.0.2' - graalFilePrefix: 'graalvm-ce-java17-22.3.0' - nativeImageDir: out/jreleaser/assemble/riot-native/native-image - jpackageDir: out/jreleaser/assemble/riot-installer/jpackage + jdkFilePrefix: 'zulu17.32.13-ca-jdk19.0.2' project: name: riot description: Redis Input/Output Tools - longDescription: Get data in and out of Redis with RIOT links: homepage: https://developer.redis.com/riot authors: - Julien Ruaux license: Apache-2.0 inceptionYear: 2020 + stereotype: cli java: - groupId: com.redis - version: 17 + groupId: com.redis.riot + version: 11 multiProject: true + mainClass: com.redis.riot.cli.Main + mainModule: com.redis.riot.cli tags: - 'riot' - 'redis' - - 'tool' + - 'tool' - 'import' - 'export' - 'replication' @@ -36,11 +35,14 @@ release: github: overwrite: true sign: true + issues: + enabled: true changelog: formatted: ALWAYS preset: conventional-commits contributors: enabled: false + contentTemplate: 'src/jreleaser/changelog.tpl' labelers: - label: 'dependencies' title: 'regex:^(?:deps(?:\(.*\))?!?):\s.*' @@ -64,6 +66,10 @@ release: signing: active: ALWAYS armored: true + files: false + +checksum: + files: false deploy: maven: @@ -86,35 +92,15 @@ assemble: jlink: riot-standalone: active: ALWAYS - java: - version: 11 imageName: '{{distributionName}}-{{projectEffectiveVersion}}' executable: riot - fileSets: - - input: '.' - includes: - - NOTICE - - LICENSE - - input: licenses - output: licenses jdeps: - multiRelease: base + multiRelease: 11 ignoreMissingDeps: true - targets: - - 'plugins/riot/build/libs/riot-{{projectVersion}}.jar' - additionalModuleNames: - - 'java.security.sasl' - - 'java.security.jgss' - - 'jdk.crypto.cryptoki' - - 'jdk.crypto.ec' - - 'jdk.localedata' - - 'jdk.net' - - 'jdk.security.auth' - - 'jdk.security.jgss' targetJdks: - - path: '{{jdkPathPrefix}}/zulu17Osx/{{jdkFilePrefix}}-macosx_x64/zulu-17.jdk/Contents/Home' + - path: '{{jdkPathPrefix}}/zulu17Osx/{{jdkFilePrefix}}-macosx_x64/zulu-19.jdk/Contents/Home' platform: 'osx-x86_64' - - path: '{{jdkPathPrefix}}/zulu17OsxArm/{{jdkFilePrefix}}-macosx_aarch64/zulu-17.jdk/Contents/Home' + - path: '{{jdkPathPrefix}}/zulu17OsxArm/{{jdkFilePrefix}}-macosx_aarch64/zulu-19.jdk/Contents/Home' platform: 'osx-aarch_64' - path: '{{jdkPathPrefix}}/zulu17Linux/{{jdkFilePrefix}}-linux_x64' platform: 'linux-x86_64' @@ -126,156 +112,34 @@ assemble: platform: 'linux_musl-aarch_64' - path: '{{jdkPathPrefix}}/zulu17Windows/{{jdkFilePrefix}}-win_x64' platform: 'windows-x86_64' - - path: '{{jdkPathPrefix}}/zulu17WindowsArm/{{jdkFilePrefix}}-win_aarch64' - platform: 'windows-aarch_64' mainJar: - path: 'plugins/riot/build/libs/riot-{{projectVersion}}.jar' + path: 'plugins/riot-cli/build/libs/riot-cli-{{projectVersion}}.jar' jars: - - pattern: 'plugins/riot/build/dependencies/flat/*.jar' - - jpackage: - riot-installer: - active: ALWAYS - jlink: riot-standalone - attachPlatform: true - exported: false - applicationPackage: - appName: riot - appVersion: '{{projectVersionNumber}}' - vendor: Redis - osx: - types: [pkg] - appName: RIOT - packageName: RIOT - packageIdentifier: com.redis.riot.cli - icon: 'src/media/riot.icns' - resourceDir: 'src/jpackage/osx' - linux: - types: [deb,rpm] - maintainer: jruaux+riot@gmail.com - icon: 'src/media/icon_256x256.png' - windows: - types: [msi] - console: true - dirChooser: true - icon: 'src/media/riot.ico' - resourceDir: 'src/jpackage/windows' - - nativeImage: - riot-native: - active: ALWAYS - java: - version: 17 - imageName: '{{distributionName}}-{{projectEffectiveVersion}}' - executable: riot - fileSets: - - input: '.' - includes: - - LICENSE - - input: licenses - output: licenses - mainJar: - path: 'plugins/riot/build/libs/riot-{{projectVersion}}.jar' - jars: - - pattern: 'plugins/riot/build/dependencies/flat/*.jar' - graalJdks: - - path: '{{jdkPathPrefix}}/graal17Osx/{{graalFilePrefix}}/Contents/Home' - platform: 'osx-x86_64' - - path: '{{jdkPathPrefix}}/graal17Linux/{{graalFilePrefix}}' - platform: 'linux-x86_64' - - path: '{{jdkPathPrefix}}/graal17Windows/{{graalFilePrefix}}' - platform: 'windows-x86_64' - upx: - active: ALWAYS - version: '3.96' - args: - - '-Duser.language=en' - - '-H:IncludeLocales=en,ca,de,es,fr,hi,it,ja,nl,pt_BR,zh_TW,ru,ko' - - '-H:Optimize=2' - - '-H:+RemoveUnusedSymbols' + - pattern: 'plugins/riot-cli/build/dependencies/flat/*.jar' distributions: - riot: - flatpak: - active: ALWAYS - continueOnError: true - componentId: com.redis.riot.cli - developerName: Redis - runtime: FREEDESKTOP - runtimeVersion: 21.08 - finishArgs: - - --share=network - - --filesystem=host - categories: - - Developer Tools - skipReleases: - - '.*-RC.*' - - '.*-M.*' - repository: - active: RELEASE - owner: flathub - name: com.redis.riot.cli - branch: master - chocolatey: - active: ALWAYS - remoteBuild: true - title: RIOT - iconUrl: 'https://raw.githubusercontent.com/redis-developer/riot/master/src/media/icon_128x128.png' - bucket: - active: RELEASE + riot-cli: jbang: active: ALWAYS - macports: - active: ALWAYS - categories: - - devel - - java - maintainers: - - '@jruaux' - repository: - active: RELEASE - name: riot-macports + alias: riot + catalog: + commitMessage: 'riot {{tagName}}' scoop: active: ALWAYS bucket: active: RELEASE + commitMessage: 'riot {{tagName}}' sdkman: active: RELEASE + candidate: riot continueOnError: true - snap: - active: ALWAYS - remoteBuild: true - base: core18 - architectures: - - buildOn: [ amd64, arm64 ] - localPlugs: - - network - spec: - active: ALWAYS - repository: - active: RELEASE - name: riot-copr artifacts: - - path: plugins/{{distributionName}}/build/distributions/{{distributionName}}-{{projectVersion}}.zip - transform: '{{distributionName}}/{{distributionName}}-{{projectEffectiveVersion}}.zip' - extraProperties: - skipSpec: true - - path: plugins/{{distributionName}}/build/distributions/{{distributionName}}-{{projectVersion}}.tar - transform: '{{distributionName}}/{{distributionName}}-{{projectEffectiveVersion}}.tar' - extraProperties: - skipFlatpak: true - + - path: plugins/{{distributionName}}/build/distributions/riot-{{projectVersion}}.zip + transform: '{{distributionName}}/riot-{{projectEffectiveVersion}}.zip' + - path: plugins/{{distributionName}}/build/distributions/riot-{{projectVersion}}.tar + transform: '{{distributionName}}/riot-{{projectEffectiveVersion}}.tar' + riot-standalone: - appImage: - active: ALWAYS - componentId: com.redis.riot.cli - developerName: Redis - categories: - - Development - repository: - active: RELEASE - name: riot-appimage - commitMessage: 'riot {{tagName}}' brew: active: ALWAYS formulaName: riot @@ -315,38 +179,6 @@ distributions: preCommands: - 'RUN apk add unzip binutils fakeroot rpm bash' - riot-installer: - type: NATIVE_PACKAGE - executable: - name: riot - windowsExtension: exe - artifacts: - - path: '{{jpackageDir}}/RIOT-{{projectVersionNumber}}-osx-x86_64.pkg' - transform: '{{distributionName}}/{{distributionName}}-{{projectEffectiveVersion}}-osx-x86_64.pkg' - platform: 'osx-x86_64' - - path: '{{jpackageDir}}/riot{{projectVersionNumber}}-1_amd64.deb' - transform: '{{distributionName}}/{{distributionName}}_{{projectEffectiveVersion}}-1_amd64.deb' - platform: 'linux-x86_64' - - path: '{{jpackageDir}}/riot-{{projectVersionNumber}}-1.x86_64.rpm' - transform: '{{distributionName}}/{{distributionName}}-{{projectEffectiveVersion}}-1.x86_64.rpm' - platform: 'linux-x86_64' - - path: '{{jpackageDir}}/riot-{{projectVersionNumber}}-windows-x86_64.msi' - transform: '{{distributionName}}/{{distributionName}}-{{projectEffectiveVersion}}-windows-x86_64.msi' - platform: 'windows-x86_64' - - riot-native: - artifacts: - - path: '{{nativeImageDir}}/{{distributionName}}-{{projectEffectiveVersion}}-osx-x86_64.zip' - platform: 'osx-x86_64' - - path: '{{nativeImageDir}}/{{distributionName}}-{{projectEffectiveVersion}}-linux-x86_64.zip' - platform: 'linux-x86_64' - - path: '{{nativeImageDir}}/{{distributionName}}-{{projectEffectiveVersion}}-windows-x86_64.zip' - platform: 'windows-x86_64' - files: artifacts: - - path: VERSION - extraProperties: - skipChecksum: true - skipSigning: true - skipSbom: true \ No newline at end of file + - path: VERSION \ No newline at end of file diff --git a/plugins/riot/gradle.properties b/plugins/riot-cli/gradle.properties similarity index 100% rename from plugins/riot/gradle.properties rename to plugins/riot-cli/gradle.properties diff --git a/plugins/riot/riot.gradle b/plugins/riot-cli/riot-cli.gradle similarity index 96% rename from plugins/riot/riot.gradle rename to plugins/riot-cli/riot-cli.gradle index 0ffdcb330..1df0109f5 100644 --- a/plugins/riot/riot.gradle +++ b/plugins/riot-cli/riot-cli.gradle @@ -21,12 +21,13 @@ plugins { } application { - mainClassName = 'com.redis.riot.cli.Riot' + applicationName = 'riot' + mainClassName = 'com.redis.riot.cli.Main' } jar { manifest { - attributes('Automatic-Module-Name': project.findProperty('automatic.module.name')) + attributes('Main-Class': 'com.redis.riot.cli.Main') } } @@ -42,6 +43,7 @@ jar { config { info { + bytecodeVersion = 8 specification { enabled = true } } licensing { @@ -59,13 +61,13 @@ dependencies { runtimeOnly 'org.slf4j:slf4j-jdk14' implementation 'org.springframework.boot:spring-boot-autoconfigure' implementation 'org.springframework:spring-jdbc' + implementation 'com.mysql:mysql-connector-j' + implementation 'org.postgresql:postgresql' + implementation 'org.springframework:spring-oxm' implementation group: 'com.ibm.db2', name: 'jcc', version: db2Version implementation group: 'com.microsoft.sqlserver', name: 'mssql-jdbc', version: mssqlVersion implementation group: 'com.oracle.ojdbc', name: 'ojdbc8', version: oracleVersion - implementation 'com.mysql:mysql-connector-j' - implementation 'org.postgresql:postgresql' implementation group: 'org.xerial', name: 'sqlite-jdbc', version: sqliteVersion - implementation 'org.springframework:spring-oxm' implementation group: 'org.springframework.cloud', name: 'spring-cloud-aws-context', version: awsVersion implementation group: 'org.springframework.cloud', name: 'spring-cloud-aws-autoconfigure', version: awsVersion implementation (group: 'org.springframework.cloud', name: 'spring-cloud-gcp-starter-storage', version: gcpVersion) { @@ -75,6 +77,7 @@ dependencies { testImplementation('org.springframework.boot:spring-boot-starter-test') { exclude group: 'org.junit.vintage', module: 'junit-vintage-engine' } + testImplementation group: 'org.awaitility', name: 'awaitility', version: awaitilityVersion testImplementation group: 'org.testcontainers', name: 'postgresql', version: testcontainersVersion testImplementation group: 'org.testcontainers', name: 'oracle-xe', version: testcontainersVersion } @@ -167,10 +170,4 @@ assemble.dependsOn copyDependencies compileJava { options.release = 8 -} - -eclipse { - project { - name = 'riot-cli' - } } \ No newline at end of file diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/DbExport.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/DbExport.java similarity index 53% rename from plugins/riot/src/main/java/com/redis/riot/cli/DbExport.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/DbExport.java index f6bd54744..ca71215c1 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/DbExport.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/DbExport.java @@ -1,7 +1,5 @@ package com.redis.riot.cli; -import java.sql.Connection; -import java.sql.SQLException; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -9,17 +7,17 @@ import javax.sql.DataSource; import org.springframework.batch.core.Job; -import org.springframework.batch.core.job.builder.JobBuilderException; import org.springframework.batch.item.ItemProcessor; import org.springframework.batch.item.database.JdbcBatchItemWriter; import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; import com.redis.riot.cli.common.AbstractExportCommand; -import com.redis.riot.cli.common.JobCommandContext; +import com.redis.riot.cli.common.CommandContext; import com.redis.riot.cli.db.DataSourceOptions; -import com.redis.riot.cli.db.DatabaseExportOptions; +import com.redis.riot.cli.db.DbExportOptions; import com.redis.riot.cli.db.NullableSqlParameterSource; import com.redis.riot.core.processor.DataStructureToMapProcessor; +import com.redis.spring.batch.RedisItemReader; import com.redis.spring.batch.common.DataStructure; import picocli.CommandLine.Command; @@ -29,7 +27,7 @@ @Command(name = "db-export", description = "Export Redis data to a relational database") public class DbExport extends AbstractExportCommand { - private static final String COMMAND_NAME = "import db"; + private static final String TASK_NAME = "Exporting to database"; private static Logger log = Logger.getLogger(DbExport.class.getName()); @Parameters(arity = "1", description = "SQL INSERT statement.", paramLabel = "SQL") @@ -37,13 +35,13 @@ public class DbExport extends AbstractExportCommand { @Mixin private DataSourceOptions dataSourceOptions = new DataSourceOptions(); @Mixin - private DatabaseExportOptions options = new DatabaseExportOptions(); + private DbExportOptions options = new DbExportOptions(); - public DatabaseExportOptions getOptions() { + public DbExportOptions getOptions() { return options; } - public void setOptions(DatabaseExportOptions exportOptions) { + public void setOptions(DbExportOptions exportOptions) { this.options = exportOptions; } @@ -64,26 +62,21 @@ public void setSql(String sql) { } @Override - protected Job job(JobCommandContext context) { + protected Job job(CommandContext context) { log.log(Level.FINE, "Creating data source with {0}", dataSourceOptions); DataSource dataSource = dataSourceOptions.dataSource(); - try (Connection connection = dataSource.getConnection()) { - String dbName = connection.getMetaData().getDatabaseProductName(); - log.log(Level.FINE, "Creating writer for database {0} with {1}", new Object[] { dbName, options }); - JdbcBatchItemWriterBuilder> builder = new JdbcBatchItemWriterBuilder<>(); - builder.itemSqlParameterSourceProvider(NullableSqlParameterSource::new); - builder.dataSource(dataSource); - builder.sql(sql); - builder.assertUpdates(options.isAssertUpdates()); - JdbcBatchItemWriter> writer = builder.build(); - writer.afterPropertiesSet(); - ItemProcessor, Map> processor = DataStructureToMapProcessor - .of(options.getKeyRegex()); - String task = String.format("Exporting to %s", dbName); - return job(context, COMMAND_NAME, step(context, COMMAND_NAME, reader(context), processor, writer), task); - } catch (SQLException e) { - throw new JobBuilderException(e); - } + log.log(Level.FINE, "Creating writer for database with {0}", options); + JdbcBatchItemWriterBuilder> builder = new JdbcBatchItemWriterBuilder<>(); + builder.itemSqlParameterSourceProvider(NullableSqlParameterSource::new); + builder.dataSource(dataSource); + builder.sql(sql); + builder.assertUpdates(options.isAssertUpdates()); + JdbcBatchItemWriter> writer = builder.build(); + writer.afterPropertiesSet(); + ItemProcessor, Map> processor = DataStructureToMapProcessor + .of(options.getKeyRegex()); + RedisItemReader> reader = reader(context); + return job(context, step(context, reader, processor, writer), TASK_NAME); } } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/DbImport.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/DbImport.java similarity index 56% rename from plugins/riot/src/main/java/com/redis/riot/cli/DbImport.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/DbImport.java index 16b039cfb..6d91c2f34 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/DbImport.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/DbImport.java @@ -1,6 +1,5 @@ package com.redis.riot.cli; -import java.sql.Connection; import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -14,10 +13,10 @@ import org.springframework.jdbc.core.ColumnMapRowMapper; import com.redis.riot.cli.common.AbstractImportCommand; -import com.redis.riot.cli.common.JobCommandContext; +import com.redis.riot.cli.common.CommandContext; import com.redis.riot.cli.common.ProgressMonitor; import com.redis.riot.cli.db.DataSourceOptions; -import com.redis.riot.cli.db.DatabaseImportOptions; +import com.redis.riot.cli.db.DbImportOptions; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; @@ -26,15 +25,15 @@ @Command(name = "db-import", description = "Import from relational databases") public class DbImport extends AbstractImportCommand { - private static final String COMMAND_NAME = "export db"; private static final Logger log = Logger.getLogger(DbImport.class.getName()); + private static final String TASK_NAME = "Importing from database"; @Parameters(arity = "1", description = "SQL SELECT statement", paramLabel = "SQL") private String sql; @Mixin private DataSourceOptions dataSourceOptions = new DataSourceOptions(); @Mixin - private DatabaseImportOptions importOptions = new DatabaseImportOptions(); + private DbImportOptions importOptions = new DbImportOptions(); public DataSourceOptions getDataSourceOptions() { return dataSourceOptions; @@ -44,35 +43,34 @@ public void setDataSourceOptions(DataSourceOptions dataSourceOptions) { this.dataSourceOptions = dataSourceOptions; } - public DatabaseImportOptions getImportOptions() { + public DbImportOptions getImportOptions() { return importOptions; } - public void setImportOptions(DatabaseImportOptions importOptions) { + public void setImportOptions(DbImportOptions importOptions) { this.importOptions = importOptions; } @Override - protected Job job(JobCommandContext context) { + protected Job job(CommandContext context) { log.log(Level.FINE, "Creating data source: {0}", dataSourceOptions); DataSource dataSource = dataSourceOptions.dataSource(); - try (Connection connection = dataSource.getConnection()) { - String productName = connection.getMetaData().getDatabaseProductName(); - log.log(Level.FINE, "Creating {0} database reader: {1}", new Object[] { productName, importOptions }); - JdbcCursorItemReaderBuilder> builder = new JdbcCursorItemReaderBuilder<>(); - builder.saveState(false); - builder.dataSource(dataSource); - builder.name(productName + "-database-reader"); - builder.rowMapper(new ColumnMapRowMapper()); - builder.sql(sql); - importOptions.configure(builder); - JdbcCursorItemReader> reader = builder.build(); + log.log(Level.FINE, "Creating database reader with {0}", importOptions); + JdbcCursorItemReaderBuilder> builder = new JdbcCursorItemReaderBuilder<>(); + builder.saveState(false); + builder.dataSource(dataSource); + builder.name("database-reader"); + builder.rowMapper(new ColumnMapRowMapper()); + builder.sql(sql); + importOptions.configure(builder); + JdbcCursorItemReader> reader = builder.build(); + try { reader.afterPropertiesSet(); - ProgressMonitor monitor = progressMonitor().task("Importing from " + productName).build(); - return context.job(COMMAND_NAME).start(step(step(context, COMMAND_NAME, reader), monitor).build()).build(); } catch (Exception e) { throw new JobBuilderException(e); } + ProgressMonitor monitor = progressMonitor().task(TASK_NAME).build(); + return context.getJobRunner().job(commandName()).start(step(step(context, reader), monitor).build()).build(); } } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/DumpImport.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/DumpImport.java similarity index 73% rename from plugins/riot/src/main/java/com/redis/riot/cli/DumpImport.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/DumpImport.java index b66286aac..57900c192 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/DumpImport.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/DumpImport.java @@ -6,14 +6,12 @@ import org.springframework.batch.core.job.builder.SimpleJobBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemStreamSupport; import org.springframework.batch.item.json.JsonItemReader; import org.springframework.core.io.Resource; -import com.redis.riot.cli.common.AbstractTransferCommand; -import com.redis.riot.cli.common.JobCommandContext; +import com.redis.riot.cli.common.AbstractCommand; +import com.redis.riot.cli.common.CommandContext; import com.redis.riot.cli.common.ProgressMonitor; -import com.redis.riot.cli.common.RedisOptions; import com.redis.riot.cli.common.RedisWriterOptions; import com.redis.riot.cli.file.FileDumpOptions; import com.redis.riot.cli.file.FileImportOptions; @@ -24,18 +22,13 @@ import com.redis.spring.batch.RedisItemWriter; import com.redis.spring.batch.common.DataStructure; +import io.lettuce.core.codec.StringCodec; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; -import picocli.CommandLine.ParentCommand; @Command(name = "dump-import", description = "Import Redis data files into Redis") -public class DumpImport extends AbstractTransferCommand { - - private static final String COMMAND_NAME = "file-dump-import"; - - @ParentCommand - private Riot parent; +public class DumpImport extends AbstractCommand { @Mixin private FileImportOptions options = new FileImportOptions(); @@ -44,11 +37,6 @@ public class DumpImport extends AbstractTransferCommand { @ArgGroup(exclusive = false, heading = "Writer options%n") private RedisWriterOptions writerOptions = new RedisWriterOptions(); - @Override - protected RedisOptions getRedisOptions() { - return parent.getRedisOptions(); - } - public FileDumpOptions getDumpFileOptions() { return dumpFileOptions; } @@ -66,25 +54,23 @@ public void setOptions(FileImportOptions options) { } @Override - protected Job job(JobCommandContext context) { + protected Job job(CommandContext context) { Iterator stepIterator = options.getResources().map(r -> step(context, r)).iterator(); - SimpleJobBuilder job = context.job(COMMAND_NAME).start(stepIterator.next()); + SimpleJobBuilder job = context.getJobRunner().job(commandName()).start(stepIterator.next()); while (stepIterator.hasNext()) { job.next(stepIterator.next()); } return job.build(); } - public TaskletStep step(JobCommandContext context, Resource resource) { - String name = String.join("-", COMMAND_NAME, resource.getDescription()); + public TaskletStep step(CommandContext context, Resource resource) { + String name = String.join("-", commandName(), resource.getDescription()); ItemReader> reader = reader(resource); - if (reader instanceof ItemStreamSupport) { - ((ItemStreamSupport) reader).setName(name); - } DataStructureProcessor processor = new DataStructureProcessor(); ProgressMonitor monitor = progressMonitor().task("Importing " + resource).build(); - RedisItemWriter> writer = context.writer() - .options(writerOptions.writerOptions()).dataStructure(); + RedisItemWriter> writer = context.dataStructureWriter(StringCodec.UTF8) + .options(writerOptions.writerOptions()).dataStructureOptions(writerOptions.dataStructureOptions()) + .build(); return step(step(context, name, reader, processor, writer), monitor).build(); } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/FakerImport.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/FakerImport.java similarity index 81% rename from plugins/riot/src/main/java/com/redis/riot/cli/FakerImport.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/FakerImport.java index f8843a3d4..feb7420c9 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/FakerImport.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/FakerImport.java @@ -15,7 +15,7 @@ import com.redis.lettucemod.search.IndexInfo; import com.redis.lettucemod.util.RedisModulesUtils; import com.redis.riot.cli.common.AbstractImportCommand; -import com.redis.riot.cli.common.JobCommandContext; +import com.redis.riot.cli.common.CommandContext; import com.redis.riot.cli.common.ProgressMonitor; import com.redis.riot.cli.gen.FakerGeneratorOptions; import com.redis.riot.core.FakerItemReader; @@ -29,7 +29,6 @@ @Command(name = "faker", description = "Import from Faker") public class FakerImport extends AbstractImportCommand { - private static final String COMMAND_NAME = "import faker"; private static final Logger log = Logger.getLogger(FakerImport.class.getName()); @Mixin @@ -44,20 +43,20 @@ public void setOptions(FakerGeneratorOptions options) { } @Override - protected Job job(JobCommandContext context) { - JobBuilder job = context.job(COMMAND_NAME); - ProgressMonitor monitor = options.configure(progressMonitor()).task("Generating").build(); - return job.start(step(step(context, COMMAND_NAME, reader(context)), monitor).build()).build(); + protected Job job(CommandContext context) { + JobBuilder job = context.getJobRunner().job(commandName()); + ProgressMonitor monitor = progressMonitor().initialMax(options.getCount()).task("Generating").build(); + return job.start(step(step(context, reader(context)), monitor).build()).build(); } - private AbstractItemCountingItemStreamItemReader> reader(JobCommandContext context) { + private AbstractItemCountingItemStreamItemReader> reader(CommandContext context) { log.log(Level.FINE, "Creating Faker reader with {0}", options); FakerItemReader reader = new FakerItemReader(generator(context)); options.configure(reader); return reader; } - private void addFieldsFromIndex(JobCommandContext context, String index, Map fields) { + private void addFieldsFromIndex(CommandContext context, String index, Map fields) { try (StatefulRedisModulesConnection connection = context.connection()) { RediSearchCommands commands = connection.sync(); IndexInfo info = RedisModulesUtils.indexInfo(commands.ftInfo(index)); @@ -67,7 +66,7 @@ private void addFieldsFromIndex(JobCommandContext context, String index, Map> generator(JobCommandContext context) { + private Generator> generator(CommandContext context) { Map fields = new LinkedHashMap<>(options.getFields()); options.getRedisearchIndex().ifPresent(index -> addFieldsFromIndex(context, index, fields)); MapGenerator generator = MapGenerator.builder().locale(options.getLocale()).fields(fields).build(); diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/FileExport.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/FileExport.java similarity index 92% rename from plugins/riot/src/main/java/com/redis/riot/cli/FileExport.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/FileExport.java index 46dc5ecc8..aaeacfcb0 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/FileExport.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/FileExport.java @@ -11,12 +11,13 @@ import com.fasterxml.jackson.dataformat.xml.XmlMapper; import com.redis.riot.cli.common.AbstractExportCommand; -import com.redis.riot.cli.common.JobCommandContext; +import com.redis.riot.cli.common.CommandContext; import com.redis.riot.cli.file.FileDumpOptions; import com.redis.riot.cli.file.FileExportOptions; import com.redis.riot.core.FileDumpType; import com.redis.riot.core.resource.JsonResourceItemWriterBuilder; import com.redis.riot.core.resource.XmlResourceItemWriterBuilder; +import com.redis.spring.batch.RedisItemReader; import com.redis.spring.batch.common.DataStructure; import picocli.CommandLine.ArgGroup; @@ -27,8 +28,6 @@ @Command(name = "file-export", description = "Export Redis data to JSON or XML files") public class FileExport extends AbstractExportCommand { - private static final String COMMAND_NAME = "export file"; - @Parameters(arity = "1", description = "File path or URL", paramLabel = "FILE") private String file; @@ -55,15 +54,16 @@ public void setOptions(FileExportOptions options) { } @Override - protected Job job(JobCommandContext context) { + protected Job job(CommandContext context) { WritableResource resource; try { resource = options.outputResource(file); } catch (IOException e) { throw new JobBuilderException(e); } + RedisItemReader> reader = reader(context); String task = String.format("Exporting %s", resource.getFilename()); - return job(context, COMMAND_NAME, step(context, COMMAND_NAME, reader(context), null, writer(resource)), task); + return job(context, step(context, reader, writer(resource)), task); } private ItemWriter> writer(WritableResource resource) { diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/FileImport.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/FileImport.java similarity index 88% rename from plugins/riot/src/main/java/com/redis/riot/cli/FileImport.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/FileImport.java index 1a5e6433e..fbcbd05ab 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/FileImport.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/FileImport.java @@ -15,7 +15,6 @@ import org.springframework.batch.core.job.builder.SimpleJobBuilder; import org.springframework.batch.core.step.tasklet.TaskletStep; import org.springframework.batch.item.ItemReader; -import org.springframework.batch.item.ItemStreamSupport; import org.springframework.batch.item.file.FlatFileItemReader; import org.springframework.batch.item.file.FlatFileParseException; import org.springframework.batch.item.file.LineCallbackHandler; @@ -34,7 +33,7 @@ import org.springframework.util.ObjectUtils; import com.redis.riot.cli.common.AbstractImportCommand; -import com.redis.riot.cli.common.JobCommandContext; +import com.redis.riot.cli.common.CommandContext; import com.redis.riot.cli.common.ProgressMonitor; import com.redis.riot.cli.file.FileImportOptions; import com.redis.riot.cli.file.FlatFileOptions; @@ -45,6 +44,7 @@ import com.redis.riot.core.ItemReaderIterator; import com.redis.riot.core.MapFieldSetMapper; import com.redis.riot.core.resource.XmlItemReader; +import com.redis.spring.batch.common.StepOptions; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; @@ -56,10 +56,9 @@ public class FileImport extends AbstractImportCommand { private static final Logger log = Logger.getLogger(FileImport.class.getName()); private static final String DELIMITER_PIPE = "|"; - private static final String COMMAND_NAME = "import file"; @Mixin - private FileImportOptions options = new FileImportOptions(); + private FileImportOptions fileImportOptions = new FileImportOptions(); @ArgGroup(exclusive = false, heading = "Flat file options%n") private FlatFileOptions flatFileOptions = new FlatFileOptions(); @Option(names = { "-t", "--filetype" }, description = "File type: ${COMPLETION-CANDIDATES}.", paramLabel = "") @@ -71,7 +70,7 @@ public FileImport() { private FileImport(Builder builder) { this.flatFileOptions = builder.flatFileOptions; this.fileType = builder.fileType; - this.options = builder.options; + this.fileImportOptions = builder.options; } public Optional getFileType() { @@ -83,11 +82,11 @@ public void setFileType(FileType fileType) { } public FileImportOptions getOptions() { - return options; + return fileImportOptions; } public void setOptions(FileImportOptions options) { - this.options = options; + this.fileImportOptions = options; } public FlatFileOptions getFlatFileOptions() { @@ -99,23 +98,27 @@ public void setFlatFileOptions(FlatFileOptions options) { } @Override - protected Job job(JobCommandContext context) { - Iterator stepIterator = options.getResources().map(r -> step(context, r)).iterator(); - SimpleJobBuilder job = context.job(COMMAND_NAME).start(stepIterator.next()); + protected StepOptions stepOptions() { + StepOptions stepOptions = super.stepOptions(); + stepOptions.getSkip().add(FlatFileParseException.class); + return stepOptions; + } + + @Override + protected Job job(CommandContext context) { + Iterator stepIterator = fileImportOptions.getResources().map(r -> step(context, r)).iterator(); + SimpleJobBuilder job = context.getJobRunner().job(commandName()).start(stepIterator.next()); while (stepIterator.hasNext()) { job.next(stepIterator.next()); } return job.build(); } - private TaskletStep step(JobCommandContext context, Resource resource) { + private TaskletStep step(CommandContext context, Resource resource) { AbstractItemCountingItemStreamItemReader> reader = reader(resource); - String name = String.join("-", COMMAND_NAME, resource.getDescription()); - if (reader instanceof ItemStreamSupport) { - ((ItemStreamSupport) reader).setName(name); - } + String name = String.join("-", commandName(), resource.getDescription()); ProgressMonitor monitor = progressMonitor().task("Importing " + resource.getFilename()).build(); - return step(step(context, name, reader), monitor).skip(FlatFileParseException.class).build(); + return step(step(context, name, reader), monitor).build(); } @SuppressWarnings({ "unchecked", "rawtypes" }) @@ -130,7 +133,8 @@ private AbstractItemCountingItemStreamItemReader> reader(Res if (!ObjectUtils.isEmpty(flatFileOptions.getIncludedFields())) { tokenizer.setIncludedFields(flatFileOptions.getIncludedFields()); } - log.log(Level.FINE, "Creating delimited reader with {0} for {1}", new Object[] { options, resource }); + log.log(Level.FINE, "Creating delimited reader with {0} for {1}", + new Object[] { fileImportOptions, resource }); return flatFileReader(resource, tokenizer); case FIXED: FixedLengthTokenizer fixedLengthTokenizer = new FixedLengthTokenizer(); @@ -142,7 +146,8 @@ private AbstractItemCountingItemStreamItemReader> reader(Res throw new IllegalArgumentException("Invalid ranges specified: " + flatFileOptions.getColumnRanges()); } fixedLengthTokenizer.setColumns(ranges); - log.log(Level.FINE, "Creating fixed-width reader with {0} for {1}", new Object[] { options, resource }); + log.log(Level.FINE, "Creating fixed-width reader with {0} for {1}", + new Object[] { fileImportOptions, resource }); return flatFileReader(resource, fixedLengthTokenizer); case XML: log.log(Level.FINE, "Creating XML reader for {0}", resource); @@ -192,7 +197,7 @@ private FlatFileItemReader> flatFileReader(Resource resource } FlatFileItemReaderBuilder> builder = new FlatFileItemReaderBuilder<>(); builder.resource(resource); - builder.encoding(options.getEncoding().name()); + builder.encoding(fileImportOptions.getEncoding().name()); builder.lineTokenizer(tokenizer); builder.recordSeparatorPolicy(new DefaultRecordSeparatorPolicy(flatFileOptions.getQuoteCharacter().toString(), flatFileOptions.getContinuationString())); @@ -295,7 +300,7 @@ public FileImport build() { * @throws IOException */ public List>> readers(String... files) { - return options.resources(Arrays.asList(files)).map(this::reader).collect(Collectors.toList()); + return fileImportOptions.resources(Arrays.asList(files)).map(this::reader).collect(Collectors.toList()); } /** @@ -306,7 +311,7 @@ public List>> readers(String... files) { * @throws Exception */ public Iterator> read(String file) throws Exception { - return new ItemReaderIterator<>(reader(options.inputResource(file))); + return new ItemReaderIterator<>(reader(fileImportOptions.inputResource(file))); } } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/Generate.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/Generate.java similarity index 67% rename from plugins/riot/src/main/java/com/redis/riot/cli/Generate.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/Generate.java index 054a4291b..81b2a9f1e 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/Generate.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/Generate.java @@ -6,42 +6,31 @@ import org.springframework.batch.core.Job; import org.springframework.batch.core.step.builder.SimpleStepBuilder; -import com.redis.riot.cli.common.AbstractTransferCommand; -import com.redis.riot.cli.common.JobCommandContext; +import com.redis.riot.cli.common.AbstractCommand; +import com.redis.riot.cli.common.CommandContext; import com.redis.riot.cli.common.ProgressMonitor; -import com.redis.riot.cli.common.RedisOptions; import com.redis.riot.cli.common.RedisWriterOptions; import com.redis.riot.cli.gen.DataStructureGeneratorOptions; import com.redis.spring.batch.RedisItemWriter; import com.redis.spring.batch.common.DataStructure; import com.redis.spring.batch.reader.GeneratorItemReader; +import io.lettuce.core.codec.StringCodec; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; -import picocli.CommandLine.ParentCommand; @Command(name = "generate", description = "Generate data structures") -public class Generate extends AbstractTransferCommand { +public class Generate extends AbstractCommand { private static final Logger log = Logger.getLogger(Generate.class.getName()); - private static final String COMMAND_NAME = "generate"; - - @ParentCommand - private Riot parent; - @Mixin private DataStructureGeneratorOptions options = new DataStructureGeneratorOptions(); @ArgGroup(exclusive = false, heading = "Writer options%n") private RedisWriterOptions writerOptions = new RedisWriterOptions(); - @Override - protected RedisOptions getRedisOptions() { - return parent.getRedisOptions(); - } - public DataStructureGeneratorOptions getOptions() { return options; } @@ -59,17 +48,16 @@ public void setWriterOptions(RedisWriterOptions writerOptions) { } @Override - protected Job job(JobCommandContext context) { - RedisItemWriter> writer = context.writer() - .options(writerOptions.writerOptions()).dataStructure(); + protected Job job(CommandContext context) { + RedisItemWriter> writer = context.dataStructureWriter(StringCodec.UTF8) + .options(writerOptions.writerOptions()).dataStructureOptions(writerOptions.dataStructureOptions()) + .build(); log.log(Level.FINE, "Creating random data structure reader with {0}", options); GeneratorItemReader reader = new GeneratorItemReader(options.generatorOptions()); options.configure(reader); - SimpleStepBuilder, DataStructure> step = step(context, COMMAND_NAME, reader, null, - writer); - ProgressMonitor monitor = options.configure(progressMonitor()).task("Generating").build(); - return context.job(COMMAND_NAME).start(step(step, monitor).build()).build(); - + SimpleStepBuilder, DataStructure> step = step(context, reader, null, writer); + ProgressMonitor monitor = progressMonitor().initialMax(options.getCount()).task("Generating").build(); + return context.getJobRunner().job(commandName()).start(step(step, monitor).build()).build(); } } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/Riot.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/Main.java similarity index 96% rename from plugins/riot/src/main/java/com/redis/riot/cli/Riot.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/Main.java index a3f9c5da1..f1e8fa21f 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/Riot.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/Main.java @@ -26,7 +26,7 @@ @Command(name = "riot", usageHelpAutoWidth = true, versionProvider = ManifestVersionProvider.class, subcommands = { DbImport.class, DbExport.class, DumpImport.class, FileImport.class, FileExport.class, FakerImport.class, Generate.class, Replicate.class, Ping.class, GenerateCompletion.class }) -public class Riot { +public class Main { @Mixin private HelpOptions helpOptions = new HelpOptions(); @@ -35,13 +35,13 @@ public class Riot { private boolean versionRequested; @Mixin - LoggingOptions loggingOptions = new LoggingOptions(); + private LoggingOptions loggingOptions = new LoggingOptions(); @ArgGroup(heading = "Redis connection options%n", exclusive = false) private RedisOptions redisOptions = new RedisOptions(); public static void main(String[] args) { - System.exit(new Riot().execute(args)); + System.exit(new Main().execute(args)); } public RedisOptions getRedisOptions() { diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/Ping.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/Ping.java similarity index 78% rename from plugins/riot/src/main/java/com/redis/riot/cli/Ping.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/Ping.java index edfce6084..388b65311 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/Ping.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/Ping.java @@ -17,56 +17,45 @@ import org.threeten.bp.Duration; import com.redis.lettucemod.api.StatefulRedisModulesConnection; -import com.redis.riot.cli.common.AbstractJobCommand; -import com.redis.riot.cli.common.JobCommandContext; +import com.redis.riot.cli.common.AbstractCommand; +import com.redis.riot.cli.common.CommandContext; import com.redis.riot.cli.common.PingOptions; -import com.redis.riot.cli.common.RedisOptions; import io.lettuce.core.metrics.CommandMetrics; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; -import picocli.CommandLine.ParentCommand; @Command(name = "ping", description = "Test connection to a Redis database") -public class Ping extends AbstractJobCommand { +public class Ping extends AbstractCommand { - private static final String COMMAND_NAME = "ping"; private static final PrintStream PRINT_STREAM = System.out; - @ParentCommand - private Riot riot; - - @Override - protected RedisOptions getRedisOptions() { - return riot.getRedisOptions(); - } - @Mixin private PingOptions options = PingOptions.builder().build(); @Override - protected Job job(JobCommandContext context) { + protected Job job(CommandContext context) { CallableTaskletAdapter tasklet = new CallableTaskletAdapter(); tasklet.setCallable(new PingTask(context, options)); - TaskletStep step = context.step(COMMAND_NAME).tasklet(tasklet).build(); - return context.job(COMMAND_NAME).start(step).build(); + TaskletStep step = context.getJobRunner().step(commandName()).tasklet(tasklet).build(); + return context.getJobRunner().job(commandName()).start(step).build(); } private class PingTask implements Callable { - private final JobCommandContext context; + private final CommandContext context; private final PingOptions options; private final AtomicInteger iteration = new AtomicInteger(); - public PingTask(JobCommandContext context, PingOptions options) { + public PingTask(CommandContext context, PingOptions options) { this.context = context; this.options = options; } @Override public RepeatStatus call() throws Exception { - if (iteration.get() > 0) { - Thread.sleep(Duration.ofSeconds(options.getSleep()).toMillis()); + if (iteration.get() > 0 && getTransferOptions().getSleep() > 0) { + Thread.sleep(Duration.ofSeconds(getTransferOptions().getSleep()).toMillis()); } try (StatefulRedisModulesConnection connection = context.connection()) { execute(connection); diff --git a/plugins/riot-cli/src/main/java/com/redis/riot/cli/Replicate.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/Replicate.java new file mode 100644 index 000000000..e7daeb911 --- /dev/null +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/Replicate.java @@ -0,0 +1,305 @@ +package com.redis.riot.cli; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.job.builder.FlowBuilder; +import org.springframework.batch.core.job.builder.JobFlowBuilder; +import org.springframework.batch.core.job.builder.SimpleJobBuilder; +import org.springframework.batch.core.job.flow.support.SimpleFlow; +import org.springframework.batch.core.step.builder.SimpleStepBuilder; +import org.springframework.batch.core.step.tasklet.TaskletStep; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.core.task.SimpleAsyncTaskExecutor; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; + +import com.redis.riot.cli.common.AbstractCommand; +import com.redis.riot.cli.common.CommandContext; +import com.redis.riot.cli.common.KeyComparisonStepListener; +import com.redis.riot.cli.common.KeyComparisonWriteListener; +import com.redis.riot.cli.common.ProgressMonitor; +import com.redis.riot.cli.common.RedisOptions; +import com.redis.riot.cli.common.RedisReaderOptions; +import com.redis.riot.cli.common.RedisWriterOptions; +import com.redis.riot.cli.common.ReplicateCommandContext; +import com.redis.riot.cli.common.ReplicationOptions; +import com.redis.riot.cli.common.ReplicationOptions.ReplicationStrategy; +import com.redis.riot.core.KeyComparisonLogger; +import com.redis.riot.core.processor.CompositeItemStreamItemProcessor; +import com.redis.riot.core.processor.KeyValueProcessor; +import com.redis.spring.batch.RedisItemReader; +import com.redis.spring.batch.RedisItemReader.ComparatorBuilder; +import com.redis.spring.batch.RedisItemReader.LiveBuilder; +import com.redis.spring.batch.RedisItemReader.ScanBuilder; +import com.redis.spring.batch.RedisItemWriter; +import com.redis.spring.batch.RedisItemWriter.AbstractBuilder; +import com.redis.spring.batch.common.IntRange; +import com.redis.spring.batch.common.JobRunner; +import com.redis.spring.batch.common.KeyValue; +import com.redis.spring.batch.common.StepOptions; +import com.redis.spring.batch.reader.KeyComparison; +import com.redis.spring.batch.reader.KeyComparison.Status; +import com.redis.spring.batch.reader.ReaderOptions; +import com.redis.spring.batch.reader.ScanSizeEstimator; +import com.redis.spring.batch.reader.SlotRangeFilter; +import com.redis.spring.batch.writer.KeyComparisonCountItemWriter; +import com.redis.spring.batch.writer.KeyComparisonCountItemWriter.Results; +import com.redis.spring.batch.writer.operation.Noop; + +import io.lettuce.core.codec.ByteArrayCodec; +import me.tongfei.progressbar.ProgressBarStyle; +import picocli.CommandLine.ArgGroup; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; + +@SuppressWarnings({ "rawtypes", "unchecked" }) +@Command(name = "replicate", description = "Replicate a Redis DB into another Redis DB") +public class Replicate extends AbstractCommand { + + private static final Logger log = Logger.getLogger(Replicate.class.getName()); + + private static final String COMPARE_MESSAGE_ASCII = " >%,d T%,d ≠%,d ⧗%,d <%,d"; + private static final String COMPARE_MESSAGE_COLOR = " \u001b[31m>%,d \u001b[33mT%,d \u001b[35m≠%,d \u001b[36m⧗%,d\u001b[0m"; + private static final Noop, Object> NOOP_OPERATION = new Noop<>(); + + @ArgGroup(exclusive = false, heading = "Target Redis connection options%n") + private RedisOptions targetRedisOptions = new RedisOptions(); + + @ArgGroup(exclusive = false, heading = "Reader options%n") + private RedisReaderOptions readerOptions = new RedisReaderOptions(); + + @ArgGroup(exclusive = false, heading = "Writer options%n") + private RedisWriterOptions writerOptions = new RedisWriterOptions(); + + @Mixin + private ReplicationOptions replicateOptions = new ReplicationOptions(); + + public RedisOptions getTargetRedisOptions() { + return targetRedisOptions; + } + + public void setTargetRedisOptions(RedisOptions targetRedisOptions) { + this.targetRedisOptions = targetRedisOptions; + } + + public RedisReaderOptions getReaderOptions() { + return readerOptions; + } + + public void setReaderOptions(RedisReaderOptions readerOptions) { + this.readerOptions = readerOptions; + } + + @Override + protected CommandContext context(JobRunner jobRunner, RedisOptions redisOptions) { + return new ReplicateCommandContext(jobRunner, redisOptions, targetRedisOptions); + } + + protected ScanSizeEstimator estimator(CommandContext context) { + return new ScanSizeEstimator(context.getRedisClient(), readerOptions.scanSizeEstimatorOptions()); + } + + protected Step verificationStep(ReplicateCommandContext context) { + log.log(Level.FINE, "Creating key comparator with TTL tolerance of {0} seconds", + replicateOptions.getTtlTolerance()); + RedisItemReader reader = configure(context.comparator()).build(); + KeyComparisonCountItemWriter writer = new KeyComparisonCountItemWriter(); + SimpleStepBuilder step = step(context, "verification", reader, null, writer); + if (replicateOptions.isShowDiffs()) { + step.listener(new KeyComparisonWriteListener(new KeyComparisonLogger(log))); + } + step.listener(new KeyComparisonStepListener(writer, getTransferOptions().getProgressUpdateInterval())); + ProgressMonitor monitor = progressMonitor().task("Verifying").initialMax(estimator(context)) + .extraMessage(() -> extraMessage(writer.getResults())).build(); + return step(step, monitor).build(); + } + + private ComparatorBuilder configure(ComparatorBuilder builder) { + return builder.rightPoolOptions(readerOptions.poolOptions()).scanOptions(readerOptions.scanOptions()) + .ttlTolerance(replicateOptions.getTtlToleranceDuration()); + } + + private String extraMessage(Results results) { + return String.format(extraMessageFormat(), results.getCount(Status.MISSING), results.getCount(Status.TYPE), + results.getCount(Status.VALUE), results.getCount(Status.TTL)); + } + + private String extraMessageFormat() { + ProgressBarStyle progressStyle = getTransferOptions().getProgressBarStyle(); + switch (progressStyle) { + case COLORFUL_UNICODE_BAR: + case COLORFUL_UNICODE_BLOCK: + return COMPARE_MESSAGE_COLOR; + default: + return COMPARE_MESSAGE_ASCII; + } + } + + public ReplicationOptions getReplicateOptions() { + return replicateOptions; + } + + public void setReplicationOptions(ReplicationOptions replicationOptions) { + this.replicateOptions = replicationOptions; + } + + public RedisWriterOptions getWriterOptions() { + return writerOptions; + } + + public void setWriterOptions(RedisWriterOptions writerOptions) { + this.writerOptions = writerOptions; + } + + @Override + protected Job job(CommandContext jobCommandContext) { + ReplicateCommandContext context = (ReplicateCommandContext) jobCommandContext; + switch (replicateOptions.getMode()) { + case COMPARE: + return compareJob(context); + case LIVE: + return liveJob(context); + case LIVEONLY: + return liveOnlyJob(context); + case SNAPSHOT: + return snapshotJob(context); + default: + throw new IllegalArgumentException("Unknown replication mode: " + replicateOptions.getMode()); + } + } + + private Job liveOnlyJob(ReplicateCommandContext context) { + SimpleJobBuilder job = context.getJobRunner().job("liveonly-replication").start(liveStep(context)); + return job.build(); + } + + private Job compareJob(ReplicateCommandContext context) { + SimpleJobBuilder job = context.getJobRunner().job("compare").start(verificationStep(context)); + return job.build(); + } + + private Job snapshotJob(ReplicateCommandContext context) { + SimpleJobBuilder snapshotJob = context.getJobRunner().job("snapshot-replication").start(scanStep(context)); + optionalVerificationStep(context).ifPresent(snapshotJob::next); + return snapshotJob.build(); + } + + private Job liveJob(ReplicateCommandContext context) { + TaskletStep liveStep = liveStep(context); + SimpleFlow liveFlow = new FlowBuilder("live-flow").start(liveStep).build(); + TaskletStep scanStep = scanStep(context); + SimpleFlow scanFlow = new FlowBuilder("scan-flow").start(scanStep).build(); + SimpleFlow replicationFlow = new FlowBuilder("replication-flow") + .split(new SimpleAsyncTaskExecutor()).add(liveFlow, scanFlow).build(); + JobFlowBuilder liveJob = context.getJobRunner().job("live-replication").start(replicationFlow); + optionalVerificationStep(context).ifPresent(liveJob::next); + return liveJob.build().build(); + } + + protected Optional optionalVerificationStep(ReplicateCommandContext context) { + if (replicateOptions.isNoVerify()) { + return Optional.empty(); + } + if (writerOptions.isDryRun()) { + return Optional.empty(); + } + if (replicateOptions.getKeyProcessor().isPresent()) { + // Verification cannot be done if a processor is set + log.warning("Key processor enabled, verification will be skipped"); + return Optional.empty(); + } + return Optional.of(verificationStep(context)); + } + + private TaskletStep scanStep(ReplicateCommandContext context) { + RedisItemReader reader = reader(context).options(readerOptions.readerOptions()) + .scanOptions(readerOptions.scanOptions()).build(); + RedisItemWriter writer = checkWriter(context).build(); + ScanSizeEstimator estimator = estimator(context); + ProgressMonitor monitor = progressMonitor().task("Scanning").initialMax(estimator).build(); + return step(step(context, "snapshot-replication", reader, processor(context), writer), monitor).build(); + } + + private TaskletStep liveStep(ReplicateCommandContext context) { + RedisItemReader reader = liveReader(context).build(); + RedisItemWriter writer = checkWriter(context).build(); + StepOptions stepOptions = liveStepOptions(stepOptions()); + ItemProcessor processor = processor(context); + SimpleStepBuilder step = step(context, "live-replication", reader, processor, writer, stepOptions); + ProgressMonitor monitor = progressMonitor().task("Listening").build(); + return step(step, monitor).build(); + } + + private LiveBuilder liveReader(ReplicateCommandContext context) { + LiveBuilder builder = reader(context).scanOptions(readerOptions.scanOptions()).live(); + ReaderOptions liveReaderOptions = readerOptions.readerOptions(); + liveStepOptions(liveReaderOptions.getStepOptions()); + builder.options(liveReaderOptions); + builder.eventQueueOptions(replicateOptions.notificationQueueOptions()); + builder.database(context.getRedisURI().getDatabase()); + replicateOptions.getKeySlot().map(this::keySlotFilter).ifPresent(builder::keyFilter); + return builder; + } + + private StepOptions liveStepOptions(StepOptions stepOptions) { + stepOptions.setFlushingInterval(Duration.ofMillis(replicateOptions.getFlushInterval())); + if (replicateOptions.getIdleTimeout() > 0) { + stepOptions.setIdleTimeout(Duration.ofMillis(replicateOptions.getIdleTimeout())); + } + return stepOptions; + } + + private AbstractBuilder checkWriter(ReplicateCommandContext context) { + if (writerOptions.isDryRun()) { + return RedisItemWriter.operation(context.getTargetRedisClient(), ByteArrayCodec.INSTANCE, NOOP_OPERATION); + } + if (isDataStructure()) { + return configure(context.targetDataStructureWriter(ByteArrayCodec.INSTANCE) + .dataStructureOptions(writerOptions.dataStructureOptions())); + } + return configure(context.targetKeyDumpWriter()); + } + + private ScanBuilder reader(ReplicateCommandContext context) { + if (isDataStructure()) { + return context.dataStructureReader(ByteArrayCodec.INSTANCE); + } + return context.keyDumpReader(); + } + + private Predicate keySlotFilter(IntRange range) { + return SlotRangeFilter.of(ByteArrayCodec.INSTANCE, range.getMin(), range.getMax()); + } + + private B configure(B writer) { + return (B) writer.options(writerOptions.writerOptions()); + } + + private boolean isDataStructure() { + return replicateOptions.getStrategy() == ReplicationStrategy.DS; + } + + private ItemProcessor processor(ReplicateCommandContext context) { + SpelExpressionParser parser = new SpelExpressionParser(); + List processors = new ArrayList<>(); + replicateOptions.getKeyProcessor().ifPresent(p -> { + EvaluationContext evaluationContext = new StandardEvaluationContext(); + evaluationContext.setVariable("src", context.getRedisURI()); + evaluationContext.setVariable("dest", context.getTargetRedisURI()); + Expression expression = parser.parseExpression(p); + processors.add(new KeyValueProcessor<>(expression, evaluationContext)); + }); + return CompositeItemStreamItemProcessor.delegates(processors.toArray(new ItemProcessor[0])); + } + +} diff --git a/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractCommand.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractCommand.java new file mode 100644 index 000000000..c270a5c6f --- /dev/null +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractCommand.java @@ -0,0 +1,134 @@ +package com.redis.riot.cli.common; + +import java.time.Duration; +import java.util.concurrent.Callable; + +import org.springframework.batch.core.ItemWriteListener; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobExecution; +import org.springframework.batch.core.StepExecutionListener; +import org.springframework.batch.core.step.builder.SimpleStepBuilder; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemReader; +import org.springframework.batch.item.ItemWriter; + +import com.redis.riot.cli.Main; +import com.redis.riot.core.ThrottledItemReader; +import com.redis.spring.batch.common.JobRunner; +import com.redis.spring.batch.common.StepOptions; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.ParentCommand; +import picocli.CommandLine.Spec; + +@Command(usageHelpAutoWidth = true) +public abstract class AbstractCommand implements Callable { + + @ParentCommand + private Main riot; + + @Spec + private CommandSpec commandSpec; + + @Mixin + private HelpOptions helpOptions; + + @Mixin + private TransferOptions transferOptions = new TransferOptions(); + + protected String commandName() { + return commandSpec.qualifiedName("-"); + } + + protected RedisOptions getRedisOptions() { + return riot.getRedisOptions(); + } + + public void setRiot(Main riot) { + this.riot = riot; + } + + public void setCommandSpec(CommandSpec commandSpec) { + this.commandSpec = commandSpec; + } + + @Override + public Integer call() throws Exception { + JobRunner jobRunner = JobRunner.inMemory(); + try (CommandContext context = context(jobRunner, getRedisOptions())) { + Job job = job(context); + JobExecution execution = jobRunner.run(job); + jobRunner.awaitTermination(execution); + if (execution.getStatus().isUnsuccessful()) { + return 1; + } + return 0; + } + } + + protected CommandContext context(JobRunner jobRunner, RedisOptions redisOptions) { + return new CommandContext(jobRunner, redisOptions); + } + + protected abstract Job job(CommandContext context); + + public TransferOptions getTransferOptions() { + return transferOptions; + } + + public void setTransferOptions(TransferOptions options) { + this.transferOptions = options; + } + + protected SimpleStepBuilder step(CommandContext context, ItemReader reader, ItemWriter writer) { + return step(context, reader, null, writer); + } + + protected SimpleStepBuilder step(CommandContext context, ItemReader reader, + ItemProcessor processor, ItemWriter writer) { + return step(context, commandName(), reader, processor, writer); + } + + protected SimpleStepBuilder step(CommandContext context, String name, ItemReader reader, + ItemWriter writer) { + return step(context, name, reader, null, writer); + } + + protected SimpleStepBuilder step(CommandContext context, String name, ItemReader reader, + ItemProcessor processor, ItemWriter writer) { + return step(context, name, reader, processor, writer, stepOptions()); + } + + protected SimpleStepBuilder step(CommandContext context, String name, ItemReader reader, + ItemProcessor processor, ItemWriter writer, StepOptions stepOptions) { + return context.getJobRunner().step(name, throttle(reader), processor, writer, stepOptions); + } + + protected StepOptions stepOptions() { + return transferOptions.stepOptions(); + } + + private ItemReader throttle(ItemReader reader) { + Duration sleep = Duration.ofMillis(transferOptions.getSleep()); + if (sleep.isNegative() || sleep.isZero()) { + return reader; + } + return new ThrottledItemReader<>(reader, sleep); + } + + protected ProgressMonitor.Builder progressMonitor() { + return ProgressMonitor.style(transferOptions.getProgressBarStyle()) + .updateInterval(Duration.ofMillis(transferOptions.getProgressUpdateInterval())); + } + + protected SimpleStepBuilder step(SimpleStepBuilder step, ProgressMonitor monitor) { + if (transferOptions.isProgressEnabled()) { + step.listener((StepExecutionListener) monitor); + step.listener((ItemWriteListener) monitor); + } + return step; + } + +} diff --git a/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractExportCommand.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractExportCommand.java new file mode 100644 index 000000000..504d0a523 --- /dev/null +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractExportCommand.java @@ -0,0 +1,34 @@ +package com.redis.riot.cli.common; + +import org.springframework.batch.core.Job; +import org.springframework.batch.core.step.builder.SimpleStepBuilder; + +import com.redis.spring.batch.RedisItemReader; +import com.redis.spring.batch.common.DataStructure; +import com.redis.spring.batch.reader.ScanSizeEstimator; + +import io.lettuce.core.codec.StringCodec; +import picocli.CommandLine.ArgGroup; + +public abstract class AbstractExportCommand extends AbstractCommand { + + @ArgGroup(exclusive = false, heading = "Reader options%n") + private RedisReaderOptions readerOptions = new RedisReaderOptions(); + + public void setReaderOptions(RedisReaderOptions readerOptions) { + this.readerOptions = readerOptions; + } + + protected RedisItemReader> reader(CommandContext context) { + return context.dataStructureReader(StringCodec.UTF8).options(readerOptions.readerOptions()) + .scanOptions(readerOptions.scanOptions()).build(); + } + + protected Job job(CommandContext context, SimpleStepBuilder step, String task) { + ScanSizeEstimator estimator = new ScanSizeEstimator(context.getRedisClient(), + readerOptions.scanSizeEstimatorOptions()); + ProgressMonitor monitor = progressMonitor().task(task).initialMax(estimator).build(); + return context.getJobRunner().job(commandName()).start(step(step, monitor).build()).build(); + } + +} diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/AbstractImportCommand.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractImportCommand.java similarity index 82% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/AbstractImportCommand.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractImportCommand.java index fe91fd93a..bf0a9a0a7 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/AbstractImportCommand.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/AbstractImportCommand.java @@ -19,7 +19,7 @@ import org.springframework.util.Assert; import com.redis.lettucemod.util.GeoLocation; -import com.redis.riot.cli.Riot; +import com.redis.riot.cli.operation.DelCommand; import com.redis.riot.cli.operation.EvalCommand; import com.redis.riot.cli.operation.ExpireCommand; import com.redis.riot.cli.operation.GeoaddCommand; @@ -41,31 +41,24 @@ import com.redis.riot.core.processor.MapAccessor; import com.redis.riot.core.processor.MapProcessor; import com.redis.riot.core.processor.SpelProcessor; -import com.redis.spring.batch.writer.Operation; +import com.redis.spring.batch.RedisItemWriter; +import com.redis.spring.batch.common.Operation; +import io.lettuce.core.codec.StringCodec; import picocli.CommandLine.ArgGroup; import picocli.CommandLine.Command; -import picocli.CommandLine.ParentCommand; -@Command(subcommands = { EvalCommand.class, ExpireCommand.class, GeoaddCommand.class, HsetCommand.class, - LpushCommand.class, NoopCommand.class, RpushCommand.class, SaddCommand.class, SetCommand.class, - XaddCommand.class, ZaddCommand.class, SugaddCommand.class, JsonSetCommand.class, +@Command(subcommands = { EvalCommand.class, ExpireCommand.class, DelCommand.class, GeoaddCommand.class, + HsetCommand.class, LpushCommand.class, NoopCommand.class, RpushCommand.class, SaddCommand.class, + SetCommand.class, XaddCommand.class, ZaddCommand.class, SugaddCommand.class, JsonSetCommand.class, TsAddCommand.class }, subcommandsRepeatable = true, synopsisSubcommandLabel = "[REDIS COMMAND...]", commandListHeading = "Redis commands:%n") -public abstract class AbstractImportCommand extends AbstractTransferCommand { - - @ParentCommand - private Riot riot; +public abstract class AbstractImportCommand extends AbstractCommand { @ArgGroup(exclusive = false, heading = "Processor options%n") private MapProcessorOptions processorOptions = new MapProcessorOptions(); @ArgGroup(exclusive = false, heading = "Writer options%n") private RedisWriterOptions writerOptions = new RedisWriterOptions(); - @Override - protected RedisOptions getRedisOptions() { - return riot.getRedisOptions(); - } - /** * Initialized manually during command parsing */ @@ -95,7 +88,7 @@ public void setWriterOptions(RedisWriterOptions writerOptions) { this.writerOptions = writerOptions; } - protected ItemProcessor, Map> processor(JobCommandContext context) { + protected ItemProcessor, Map> processor(CommandContext context) { List, Map>> processors = new ArrayList<>(); if (processorOptions.hasSpelFields()) { StandardEvaluationContext evaluationContext = new StandardEvaluationContext(); @@ -126,7 +119,7 @@ protected ItemProcessor, Map> processor(JobC } @SuppressWarnings({ "unchecked", "rawtypes" }) - protected ItemWriter> writer(JobCommandContext context) { + protected ItemWriter> writer(CommandContext context) { Assert.notNull(redisCommands, "RedisCommands not set"); Assert.isTrue(!redisCommands.isEmpty(), "No Redis command specified"); List>> writers = redisCommands.stream().map(OperationCommand::operation) @@ -139,18 +132,20 @@ protected ItemWriter> writer(JobCommandContext context) { return writer; } - private ItemWriter> writer(JobCommandContext context, - Operation> operation) { - return context.writer().options(writerOptions.writerOptions()).operation(operation); + private ItemWriter> writer(CommandContext context, + Operation, ?> operation) { + return RedisItemWriter.operation(context.getRedisClient(), StringCodec.UTF8, operation) + .options(writerOptions.writerOptions()).build(); } - protected SimpleStepBuilder, Map> step(JobCommandContext context, String name, + protected SimpleStepBuilder, Map> step(CommandContext context, AbstractItemCountingItemStreamItemReader> reader) { - return step(context, name, reader, processor(context), writer(context)); + return step(context, commandName(), reader); } - public void setRiot(Riot riot) { - this.riot = riot; + protected SimpleStepBuilder, Map> step(CommandContext context, String name, + AbstractItemCountingItemStreamItemReader> reader) { + return step(context, name, reader, processor(context), writer(context)); } } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/JobCommandContext.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/CommandContext.java similarity index 50% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/JobCommandContext.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/CommandContext.java index cf970c2e0..bf9823c51 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/JobCommandContext.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/CommandContext.java @@ -1,35 +1,41 @@ package com.redis.riot.cli.common; -import org.springframework.batch.core.job.builder.JobBuilder; -import org.springframework.batch.core.step.builder.StepBuilder; - import com.redis.lettucemod.RedisModulesClient; import com.redis.lettucemod.api.StatefulRedisModulesConnection; import com.redis.lettucemod.cluster.RedisModulesClusterClient; -import com.redis.spring.batch.RedisItemReader; -import com.redis.spring.batch.RedisItemWriter; +import com.redis.spring.batch.RedisItemReader.ScanBuilder; +import com.redis.spring.batch.RedisItemWriter.DataStructureWriterBuilder; +import com.redis.spring.batch.common.DataStructure; import com.redis.spring.batch.common.JobRunner; +import com.redis.spring.batch.common.KeyDump; +import com.redis.spring.batch.common.Operation; +import com.redis.spring.batch.reader.DataStructureCodecReadOperation; +import com.redis.spring.batch.reader.KeyDumpReadOperation; import io.lettuce.core.AbstractRedisClient; import io.lettuce.core.RedisURI; +import io.lettuce.core.codec.ByteArrayCodec; import io.lettuce.core.codec.RedisCodec; -import io.lettuce.core.codec.StringCodec; import io.lettuce.core.pubsub.StatefulRedisPubSubConnection; -public class JobCommandContext implements AutoCloseable { +public class CommandContext implements AutoCloseable { - private final JobRunner jobRunner; + protected final JobRunner jobRunner; private final RedisOptions redisOptions; private final AbstractRedisClient redisClient; private final RedisURI redisURI; - public JobCommandContext(JobRunner jobRunner, RedisOptions redisOptions) { + public CommandContext(JobRunner jobRunner, RedisOptions redisOptions) { this.jobRunner = jobRunner; this.redisOptions = redisOptions; this.redisURI = redisOptions.uri(); this.redisClient = redisOptions.client(); } + public JobRunner getJobRunner() { + return jobRunner; + } + public RedisOptions getRedisOptions() { return redisOptions; } @@ -71,42 +77,35 @@ public StatefulRedisPubSubConnection pubSubConnection(AbstractRedis return ((RedisModulesClient) client).connectPubSub(codec); } - public RedisItemReader.Builder reader(RedisCodec codec) { - return reader(redisClient, codec); + public ScanBuilder> dataStructureReader(RedisCodec codec) { + return dataStructureReader(redisClient, codec); } - public RedisItemReader.Builder reader() { - return reader(StringCodec.UTF8); + public ScanBuilder> keyDumpReader() { + return keyDumpReader(redisClient); } - protected RedisItemReader.Builder reader(AbstractRedisClient client, RedisCodec codec) { - if (client instanceof RedisModulesClusterClient) { - return RedisItemReader.client((RedisModulesClusterClient) client, codec).jobRunner(jobRunner); - } - return RedisItemReader.client((RedisModulesClient) client, codec).jobRunner(jobRunner); + protected ScanBuilder> keyDumpReader(AbstractRedisClient client) { + return scanBuilder(client, ByteArrayCodec.INSTANCE, new KeyDumpReadOperation(client)); } - public RedisItemWriter.Builder writer() { - return writer(StringCodec.UTF8); - } - - public RedisItemWriter.Builder writer(RedisCodec codec) { - return writer(redisClient, codec); + protected ScanBuilder> dataStructureReader(AbstractRedisClient client, + RedisCodec codec) { + return scanBuilder(client, codec, new DataStructureCodecReadOperation<>(client, codec)); } - protected static RedisItemWriter.Builder writer(AbstractRedisClient client, RedisCodec codec) { - if (client instanceof RedisModulesClusterClient) { - return RedisItemWriter.client((RedisModulesClusterClient) client, codec); - } - return RedisItemWriter.client((RedisModulesClient) client, codec); + private ScanBuilder scanBuilder(AbstractRedisClient client, RedisCodec codec, + Operation operation) { + return new ScanBuilder<>(client, codec, operation).jobRunner(jobRunner); } - public StepBuilder step(String name) { - return jobRunner.step(name); + public DataStructureWriterBuilder dataStructureWriter(RedisCodec codec) { + return dataStructureWriter(redisClient, codec); } - public JobBuilder job(String name) { - return jobRunner.job(name); + protected static DataStructureWriterBuilder dataStructureWriter(AbstractRedisClient client, + RedisCodec codec) { + return new DataStructureWriterBuilder<>(client, codec); } } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/DoubleRangeTypeConverter.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/DoubleRangeTypeConverter.java similarity index 100% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/DoubleRangeTypeConverter.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/DoubleRangeTypeConverter.java diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/HelpOptions.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/HelpOptions.java similarity index 100% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/HelpOptions.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/HelpOptions.java diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/IntRangeTypeConverter.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/IntRangeTypeConverter.java similarity index 100% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/IntRangeTypeConverter.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/IntRangeTypeConverter.java diff --git a/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/KeyComparisonStepListener.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/KeyComparisonStepListener.java new file mode 100644 index 000000000..3716f4c67 --- /dev/null +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/KeyComparisonStepListener.java @@ -0,0 +1,48 @@ +package com.redis.riot.cli.common; + +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.springframework.batch.core.ExitStatus; +import org.springframework.batch.core.StepExecution; +import org.springframework.batch.core.listener.StepExecutionListenerSupport; + +import com.redis.spring.batch.reader.KeyComparison.Status; +import com.redis.spring.batch.writer.KeyComparisonCountItemWriter; +import com.redis.spring.batch.writer.KeyComparisonCountItemWriter.Results; + +public class KeyComparisonStepListener extends StepExecutionListenerSupport { + + private static final Logger log = Logger.getLogger(KeyComparisonStepListener.class.getName()); + + private final KeyComparisonCountItemWriter writer; + private final long sleep; + + public KeyComparisonStepListener(KeyComparisonCountItemWriter writer, long sleep) { + this.writer = writer; + this.sleep = sleep; + } + + @Override + public ExitStatus afterStep(StepExecution stepExecution) { + if (stepExecution.getStatus().isUnsuccessful()) { + return null; + } + Results results = writer.getResults(); + if (results.getTotalCount() == results.getCount(Status.OK)) { + log.info("Verification completed - all OK"); + return ExitStatus.COMPLETED; + } + try { + Thread.sleep(sleep); + } catch (InterruptedException e) { + log.fine("Verification interrupted"); + Thread.currentThread().interrupt(); + return ExitStatus.STOPPED; + } + log.log(Level.WARNING, "Verification failed: OK={0} Missing={1} Values={2} TTLs={3} Types={4}", + new Object[] { results.getCount(Status.OK), results.getCount(Status.MISSING), + results.getCount(Status.VALUE), results.getCount(Status.TTL), results.getCount(Status.TYPE) }); + return new ExitStatus(ExitStatus.FAILED.getExitCode(), "Verification failed"); + } +} \ No newline at end of file diff --git a/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/KeyComparisonWriteListener.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/KeyComparisonWriteListener.java new file mode 100644 index 000000000..5356b9557 --- /dev/null +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/KeyComparisonWriteListener.java @@ -0,0 +1,32 @@ +package com.redis.riot.cli.common; + +import java.util.List; + +import org.springframework.batch.core.ItemWriteListener; + +import com.redis.riot.core.KeyComparisonLogger; +import com.redis.spring.batch.reader.KeyComparison; + +public class KeyComparisonWriteListener implements ItemWriteListener { + + private final KeyComparisonLogger logger; + + public KeyComparisonWriteListener(KeyComparisonLogger logger) { + this.logger = logger; + } + + @Override + public void onWriteError(Exception exception, List items) { + // do nothing + } + + @Override + public void beforeWrite(List items) { + // do nothing + } + + @Override + public void afterWrite(List items) { + items.forEach(logger::log); + } +} \ No newline at end of file diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/LoggingOptions.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/LoggingOptions.java similarity index 70% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/LoggingOptions.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/LoggingOptions.java index 8d085f2ed..587db31eb 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/LoggingOptions.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/LoggingOptions.java @@ -20,7 +20,7 @@ import org.springframework.core.NestedExceptionUtils; -import com.redis.riot.cli.Riot; +import com.redis.riot.cli.Main; import io.netty.util.internal.logging.InternalLoggerFactory; import io.netty.util.internal.logging.JdkLoggerFactory; @@ -31,32 +31,17 @@ public class LoggingOptions { + public static final Level DEFAULT_LEVEL = Level.SEVERE; + public static final boolean DEFAULT_STACKTRACE = false; private static final String ROOT_LOGGER = ""; - /** - * This mixin is able to climb the command hierarchy because the - * {@code @Spec(Target.MIXEE)}-annotated field gets a reference to the command - * where it is used. - */ - private @Spec(MIXEE) CommandSpec mixee; // spec of the command where the @Mixin is used - - private Level level = Level.SEVERE; - private boolean stacktrace; - - // Each subcommand that mixes in the LoggingMixin has its own instance of this - // class, - // so there may be many LoggingMixin instances. - // We want to store the verbosity value in a single, central place, so - // we find the top-level command, - // and store the verbosity level on our top-level command's LoggingMixin. - // - // In the main method, `LoggingMixin::executionStrategy` should be set as the - // execution strategy: - // that will take the verbosity level that we stored in the top-level command's - // LoggingMixin - // to configure Log4j2 before executing the command that the user specified. + private @Spec(MIXEE) CommandSpec mixee; + + private Level level = DEFAULT_LEVEL; + private boolean stacktrace = DEFAULT_STACKTRACE; + private static LoggingOptions getTopLevelCommandLoggingMixin(CommandSpec commandSpec) { - return ((Riot) commandSpec.root().userObject()).getLoggingOptions(); + return ((Main) commandSpec.root().userObject()).getLoggingOptions(); } @Option(names = { "-d", "--debug" }, description = "Log in debug mode (includes normal stacktrace).") @@ -101,39 +86,11 @@ public Level getLevel() { return getTopLevelCommandLoggingMixin(mixee).level; } - /** - * Configures Log4j2 based on the verbosity level of the top-level command's - * LoggingMixin, before invoking the default execution strategy - * ({@link picocli.CommandLine.RunLast RunLast}) and returning the result. - *

- * Example usage: - *

- * - * 
-	 * public void main(String... args) {
-	 *     new CommandLine(new MyApp())
-	 *             .setExecutionStrategy(LoggingMixin::executionStrategy))
-	 *             .execute(args);
-	 * }
-	 *
- * - * @param parseResult represents the result of parsing the command line - * @return the exit code of executing the most specific subcommand - */ public static int executionStrategy(ParseResult parseResult) { getTopLevelCommandLoggingMixin(parseResult.commandSpec()).configureLoggers(); return 0; } - /** - * Configures the Log4j2 console appender(s), using the specified verbosity: - *
    - *
  • {@code -vvv} : enable TRACE level
    • - *
  • {@code -vv} : enable DEBUG level
    • - *
  • {@code -v} : enable INFO level
    • - *
  • (not specified) : enable WARN level
    • - *
- */ public void configureLoggers() { Level logLevel = getTopLevelCommandLoggingMixin(mixee).level; boolean printStacktrace = getTopLevelCommandLoggingMixin(mixee).stacktrace; @@ -143,7 +100,7 @@ public void configureLoggers() { ConsoleHandler handler = new ConsoleHandler(); handler.setLevel(Level.ALL); handler.setFormatter( - printStacktrace || logLevel.intValue() <= Level.FINE.intValue() ? new StackTraceOneLineLogFormat() + printStacktrace || logLevel.intValue() <= Level.INFO.intValue() ? new StackTraceOneLineLogFormat() : new OneLineLogFormat()); activeLogger.addHandler(handler); Logger.getLogger(ROOT_LOGGER).setLevel(logLevel); diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/ManifestVersionProvider.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ManifestVersionProvider.java similarity index 100% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/ManifestVersionProvider.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ManifestVersionProvider.java diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/MapProcessorOptions.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/MapProcessorOptions.java similarity index 94% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/MapProcessorOptions.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/MapProcessorOptions.java index 8c16befc2..464d78d75 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/MapProcessorOptions.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/MapProcessorOptions.java @@ -16,7 +16,7 @@ public class MapProcessorOptions { @Option(arity = "1..*", names = "--var", description = "Register a variable in the SpEL processor context.", paramLabel = "") private Map variables; @Option(names = "--date", description = "Processor date format (default: ${DEFAULT-VALUE}).", paramLabel = "") - private String dateFormat = new SimpleDateFormat().toPattern(); + private String dateFormat = defaultDateFormat(); @Option(arity = "1..*", names = "--filter", description = "Discard records using SpEL boolean expressions.", paramLabel = "") private String[] filters; @Option(arity = "1..*", names = "--regex", description = "Extract named values from source field using regex.", paramLabel = "") @@ -26,6 +26,10 @@ public Map getSpelFields() { return spelFields; } + public static String defaultDateFormat() { + return new SimpleDateFormat().toPattern(); + } + public void setSpelFields(Map spelFields) { this.spelFields = spelFields; } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/PingOptions.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/PingOptions.java similarity index 82% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/PingOptions.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/PingOptions.java index 8746c91d7..c2093a230 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/PingOptions.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/PingOptions.java @@ -1,6 +1,5 @@ package com.redis.riot.cli.common; -import java.time.Duration; import java.util.Arrays; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -11,18 +10,15 @@ public class PingOptions { - private static final int DEFAULT_ITERATIONS = 1; - private static final int DEFAULT_COUNT = 10; - private static final Duration DEFAULT_SLEEP_DURATION = Duration.ofSeconds(1); - private static final TimeUnit DEFAULT_TIME_UNIT = TimeUnit.MILLISECONDS; + public static final int DEFAULT_ITERATIONS = 1; + public static final int DEFAULT_COUNT = 10; + public static final TimeUnit DEFAULT_TIME_UNIT = TimeUnit.MILLISECONDS; private static final double[] DEFAULT_PERCENTILES = DefaultCommandLatencyCollectorOptions.DEFAULT_TARGET_PERCENTILES; @Option(names = "--iterations", description = "Number of test iterations. Use a negative value to test endlessly. (default: ${DEFAULT-VALUE}).", paramLabel = "") private int iterations = DEFAULT_ITERATIONS; @Option(names = "--count", description = "Number of pings to perform per iteration (default: ${DEFAULT-VALUE}).", paramLabel = "") private int count = DEFAULT_COUNT; - @Option(names = "--sleep", description = "Sleep duration in seconds between iterations (default: ${DEFAULT-VALUE}).", paramLabel = "") - private long sleep = DEFAULT_SLEEP_DURATION.getSeconds(); @Option(names = "--unit", description = "Time unit used to display latencies (default: ${DEFAULT-VALUE}).", paramLabel = "") private TimeUnit timeUnit = DEFAULT_TIME_UNIT; @Option(names = "--distribution", description = "Show latency distribution.") @@ -33,7 +29,6 @@ public class PingOptions { private PingOptions(Builder builder) { this.iterations = builder.iterations; this.count = builder.count; - this.sleep = builder.sleep.getSeconds(); this.timeUnit = builder.unit; this.latencyDistribution = builder.latencyDistribution; this.percentiles = builder.percentiles; @@ -63,14 +58,6 @@ public void setCount(int count) { this.count = count; } - public long getSleep() { - return sleep; - } - - public void setSleep(long sleep) { - this.sleep = sleep; - } - public TimeUnit getTimeUnit() { return timeUnit; } @@ -107,7 +94,6 @@ public static final class Builder { private int iterations = DEFAULT_ITERATIONS; private int count = DEFAULT_COUNT; - private Duration sleep = DEFAULT_SLEEP_DURATION; private TimeUnit unit = DEFAULT_TIME_UNIT; private boolean latencyDistribution; private Set percentiles = defaultPercentiles(); @@ -125,11 +111,6 @@ public Builder count(int count) { return this; } - public Builder sleep(Duration sleep) { - this.sleep = sleep; - return this; - } - public Builder unit(TimeUnit unit) { this.unit = unit; return this; diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/ProgressMonitor.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ProgressMonitor.java similarity index 69% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/ProgressMonitor.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ProgressMonitor.java index 4a07b8d51..22ea15517 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/ProgressMonitor.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ProgressMonitor.java @@ -3,6 +3,7 @@ import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.function.LongSupplier; import java.util.function.Supplier; import org.springframework.batch.core.BatchStatus; @@ -10,9 +11,6 @@ import org.springframework.batch.core.ItemWriteListener; import org.springframework.batch.core.StepExecution; import org.springframework.batch.core.StepExecutionListener; -import org.springframework.util.Assert; - -import com.redis.spring.batch.common.Utils; import me.tongfei.progressbar.ProgressBar; import me.tongfei.progressbar.ProgressBarBuilder; @@ -20,10 +18,12 @@ public class ProgressMonitor implements StepExecutionListener, ItemWriteListener { - private final ProgressStyle style; + public static final long DEFAULT_INITIAL_MAX = -1; + + private final ProgressBarStyle style; private final String task; private final Duration updateInterval; - private final Optional> initialMax; + private final LongSupplier initialMax; protected ProgressBar progressBar; private ProgressMonitor(Builder builder) { @@ -37,12 +37,7 @@ private ProgressMonitor(Builder builder) { public void beforeStep(StepExecution stepExecution) { ProgressBarBuilder builder = new ProgressBarBuilder(); builder.setStyle(progressBarStyle()); - initialMax.ifPresent(m -> { - Long initialMaxValue = m.get(); - if (initialMaxValue != null) { - builder.setInitialMax(initialMaxValue); - } - }); + builder.setInitialMax(initialMax.getAsLong()); builder.setUpdateIntervalMillis(Math.toIntExact(updateInterval.toMillis())); builder.setTaskName(task); builder.showSpeed(); @@ -50,16 +45,7 @@ public void beforeStep(StepExecution stepExecution) { } private ProgressBarStyle progressBarStyle() { - switch (style) { - case ASCII: - return ProgressBarStyle.ASCII; - case UNICODE: - return ProgressBarStyle.UNICODE_BLOCK; - case BLOCK: - return ProgressBarStyle.COLORFUL_UNICODE_BLOCK; - default: - return ProgressBarStyle.COLORFUL_UNICODE_BAR; - } + return style; } @Override @@ -101,40 +87,42 @@ public void afterWrite(List items) { } } - public static Builder style(ProgressStyle style) { + public static Builder style(ProgressBarStyle style) { return new Builder(style); } public static class Builder { - private final ProgressStyle style; + private final ProgressBarStyle style; private String task; private Duration updateInterval = Duration.ofMillis(300); - private Optional> initialMax = Optional.empty(); + private LongSupplier initialMax = () -> DEFAULT_INITIAL_MAX; private Optional> extraMessage = Optional.empty(); - public Builder(ProgressStyle style) { + public Builder(ProgressBarStyle style) { this.style = style; } - public ProgressMonitor.Builder task(String task) { + public Builder task(String task) { this.task = task; return this; } - public ProgressMonitor.Builder updateInterval(Duration updateInterval) { - Utils.assertPositive(updateInterval, "Update interval"); + public Builder updateInterval(Duration updateInterval) { this.updateInterval = updateInterval; return this; } - public ProgressMonitor.Builder initialMax(Supplier initialMax) { - Assert.notNull(initialMax, "InitialMax supplier must not be null"); - this.initialMax = Optional.of(initialMax); + public Builder initialMax(LongSupplier initialMax) { + this.initialMax = initialMax; return this; } - public ProgressMonitor.Builder extraMessage(Supplier extraMessage) { + public Builder initialMax(long initialMax) { + return initialMax(() -> initialMax); + } + + public Builder extraMessage(Supplier extraMessage) { this.extraMessage = Optional.of(extraMessage); return this; } diff --git a/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReadFrom.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReadFrom.java new file mode 100644 index 000000000..2992bdcdc --- /dev/null +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReadFrom.java @@ -0,0 +1,6 @@ +package com.redis.riot.cli.common; + +public enum ReadFrom { + + MASTER, MASTER_PREFERRED, UPSTREAM, UPSTREAM_PREFERRED, REPLICA_PREFERRED, REPLICA, LOWEST_LATENCY, ANY, ANY_REPLICA +} \ No newline at end of file diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/RedisOptions.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RedisOptions.java similarity index 97% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/RedisOptions.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RedisOptions.java index 4fd977525..a4cd10d88 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/RedisOptions.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RedisOptions.java @@ -8,6 +8,7 @@ import com.redis.lettucemod.util.RedisURIBuilder; import io.lettuce.core.AbstractRedisClient; +import io.lettuce.core.ReadFrom; import io.lettuce.core.RedisURI; import io.lettuce.core.SslVerifyMode; import io.lettuce.core.event.DefaultEventPublisherOptions; @@ -17,13 +18,8 @@ public class RedisOptions { - public enum ReadFrom { - - MASTER, MASTER_PREFERRED, UPSTREAM, UPSTREAM_PREFERRED, REPLICA_PREFERRED, REPLICA, LOWEST_LATENCY, ANY, - ANY_REPLICA - } - public static final Duration DEFAULT_METRICS_STEP = Duration.ofSeconds(5); + public static final SslVerifyMode DEFAULT_SSL_VERIFY_MODE = SslVerifyMode.FULL; @Option(names = { "-h", "--hostname" }, description = "Server hostname.", paramLabel = "") private Optional host = Optional.empty(); @@ -58,7 +54,7 @@ public enum ReadFrom { private boolean tls; @Option(names = "--tls-verify", description = "TLS peer-verify mode: FULL (default), NONE, CA.", paramLabel = "") - private SslVerifyMode tlsVerifyMode = SslVerifyMode.FULL; + private SslVerifyMode tlsVerifyMode = DEFAULT_SSL_VERIFY_MODE; @Option(names = "--ks", description = "Path to keystore.", paramLabel = "", hidden = true) private Optional keystore = Optional.empty(); diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/RedisReaderOptions.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RedisReaderOptions.java similarity index 56% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/RedisReaderOptions.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RedisReaderOptions.java index 5e076fa87..cb31b27ff 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/RedisReaderOptions.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RedisReaderOptions.java @@ -4,11 +4,10 @@ import org.springframework.batch.core.step.skip.SkipPolicy; -import com.redis.spring.batch.common.FaultToleranceOptions; import com.redis.spring.batch.common.PoolOptions; +import com.redis.spring.batch.common.StepOptions; import com.redis.spring.batch.reader.QueueOptions; import com.redis.spring.batch.reader.ReaderOptions; -import com.redis.spring.batch.reader.ReaderOptions.Builder; import com.redis.spring.batch.reader.ScanOptions; import com.redis.spring.batch.reader.ScanSizeEstimatorOptions; @@ -16,47 +15,56 @@ public class RedisReaderOptions { + public static final int DEFAULT_QUEUE_CAPACITY = QueueOptions.DEFAULT_CAPACITY; + public static final int DEFAULT_THREADS = StepOptions.DEFAULT_THREADS; + public static final int DEFAULT_CHUNK_SIZE = StepOptions.DEFAULT_CHUNK_SIZE; + public static final StepSkipPolicy DEFAULT_SKIP_POLICY = StepSkipPolicy.NEVER; + public static final int DEFAULT_SKIP_LIMIT = StepOptions.DEFAULT_SKIP_LIMIT; + public static final String DEFAULT_SCAN_MATCH = ScanOptions.DEFAULT_MATCH; + public static final long DEFAULT_SCAN_COUNT = ScanOptions.DEFAULT_COUNT; + public static final int DEFAULT_POOL_MAX_TOTAL = PoolOptions.DEFAULT_MAX_TOTAL; + @Option(names = "--read-queue", description = "Capacity of the reader queue (default: ${DEFAULT-VALUE}).", paramLabel = "") - private int queueCapacity = QueueOptions.DEFAULT_CAPACITY; + private int queueCapacity = DEFAULT_QUEUE_CAPACITY; @Option(names = "--read-threads", description = "Number of reader threads (default: ${DEFAULT-VALUE}).", paramLabel = "") - private int threads = ReaderOptions.DEFAULT_THREADS; + private int threads = DEFAULT_THREADS; @Option(names = "--read-batch", description = "Number of reader values to process at once (default: ${DEFAULT-VALUE}).", paramLabel = "") - private int batchSize = ReaderOptions.DEFAULT_CHUNK_SIZE; + private int chunkSize = DEFAULT_CHUNK_SIZE; @Option(names = "--read-skip-policy", description = "Policy to determine if some reading should be skipped: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).", paramLabel = "

") - private StepSkipPolicy skipPolicy = StepSkipPolicy.NEVER; + private StepSkipPolicy skipPolicy = DEFAULT_SKIP_POLICY; @Option(names = "--read-skip-limit", description = "LIMIT skip policy: max number of failed items before considering reader has failed (default: ${DEFAULT-VALUE}).", paramLabel = "") - private int skipLimit = FaultToleranceOptions.DEFAULT_SKIP_LIMIT; + private int skipLimit = DEFAULT_SKIP_LIMIT; @Option(names = "--scan-match", description = "SCAN MATCH pattern (default: ${DEFAULT-VALUE}).", paramLabel = "") - private String match = ScanOptions.DEFAULT_MATCH; + private String scanMatch = DEFAULT_SCAN_MATCH; @Option(names = "--scan-count", description = "SCAN COUNT option (default: ${DEFAULT-VALUE}).", paramLabel = "") - private long count = ScanOptions.DEFAULT_COUNT; + private long scanCount = DEFAULT_SCAN_COUNT; @Option(names = "--scan-type", description = "SCAN TYPE option.", paramLabel = "") private Optional type = Optional.empty(); @Option(names = "--read-pool", description = "Max connections for reader pool (default: ${DEFAULT-VALUE}).", paramLabel = "") - private int poolMaxTotal = PoolOptions.DEFAULT_MAX_TOTAL; + private int poolMaxTotal = DEFAULT_POOL_MAX_TOTAL; - public String getMatch() { - return match; + public String getScanMatch() { + return scanMatch; } - public void setMatch(String match) { - this.match = match; + public void setScanMatch(String match) { + this.scanMatch = match; } - public long getCount() { - return count; + public long getScanCount() { + return scanCount; } - public void setCount(long count) { - this.count = count; + public void setScanCount(long count) { + this.scanCount = count; } public Optional getType() { @@ -67,42 +75,34 @@ public void setType(Optional type) { this.type = type; } - public ReaderOptions readerOptions() { - Builder builder = ReaderOptions.builder(); - builder.chunkSize(batchSize); - builder.threads(threads); - builder.faultToleranceOptions(faultToleranceOptions()); - builder.queueOptions(queueOptions()); - builder.poolOptions(poolOptions()); - return builder.build(); + private StepOptions stepOptions() { + return StepOptions.builder().chunkSize(chunkSize).threads(threads).skipPolicy(skipPolicy()).skipLimit(skipLimit) + .build(); } private QueueOptions queueOptions() { return QueueOptions.builder().capacity(queueCapacity).build(); } - private PoolOptions poolOptions() { + public PoolOptions poolOptions() { return PoolOptions.builder().maxTotal(poolMaxTotal).build(); } - private FaultToleranceOptions faultToleranceOptions() { - return FaultToleranceOptions.builder().skipPolicy(skipPolicy()).skipLimit(skipLimit).build(); - } - private SkipPolicy skipPolicy() { return TransferOptions.skipPolicy(skipPolicy, skipLimit); } public ScanOptions scanOptions() { - return ScanOptions.builder().match(match).count(count).type(type).build(); + return ScanOptions.builder().match(scanMatch).count(scanCount).type(type).build(); } public ScanSizeEstimatorOptions scanSizeEstimatorOptions() { - ScanSizeEstimatorOptions.Builder builder = ScanSizeEstimatorOptions.builder(); - builder.match(match); - builder.sampleSize(count); - builder.type(type); - return builder.build(); + return ScanSizeEstimatorOptions.builder().match(scanMatch).sampleSize(scanCount).type(type).build(); + } + + public ReaderOptions readerOptions() { + return ReaderOptions.builder().poolOptions(poolOptions()).queueOptions(queueOptions()) + .stepOptions(stepOptions()).build(); } } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/RedisWriterOptions.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RedisWriterOptions.java similarity index 57% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/RedisWriterOptions.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RedisWriterOptions.java index f8e95268e..28720dc8d 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/RedisWriterOptions.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RedisWriterOptions.java @@ -4,9 +4,11 @@ import java.util.Optional; import com.redis.spring.batch.common.PoolOptions; -import com.redis.spring.batch.writer.WaitForReplication; +import com.redis.spring.batch.writer.DataStructureWriteOptions; +import com.redis.spring.batch.writer.DataStructureWriteOptions.MergePolicy; +import com.redis.spring.batch.writer.DataStructureWriteOptions.StreamIdPolicy; +import com.redis.spring.batch.writer.ReplicaOptions; import com.redis.spring.batch.writer.WriterOptions; -import com.redis.spring.batch.writer.WriterOptions.Builder; import picocli.CommandLine.Option; @@ -14,15 +16,25 @@ public class RedisWriterOptions { @Option(names = "--dry-run", description = "Enable dummy writes.") private boolean dryRun; + @Option(names = "--multi-exec", description = "Enable MULTI/EXEC writes.") private boolean multiExec; + @Option(names = "--wait-replicas", description = "Number of replicas for WAIT command (default: ${DEFAULT-VALUE}).", paramLabel = "") private int waitReplicas; + @Option(names = "--wait-timeout", description = "Timeout in millis for WAIT command (default: ${DEFAULT-VALUE}).", paramLabel = "") - private long waitTimeout = WaitForReplication.DEFAULT_TIMEOUT.toMillis(); + private long waitTimeout = ReplicaOptions.DEFAULT_TIMEOUT.toMillis(); + @Option(names = "--write-pool", description = "Max connections for writer pool (default: ${DEFAULT-VALUE}).", paramLabel = "") private int poolMaxTotal = PoolOptions.DEFAULT_MAX_TOTAL; + @Option(names = "--merge-policy", description = "Policy to merge collection data structures: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).", paramLabel = "

") + private MergePolicy mergePolicy = MergePolicy.OVERWRITE; + + @Option(names = "--stream-id", description = "Policy for stream IDs: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).", paramLabel = "

") + private StreamIdPolicy streamIdPolicy = StreamIdPolicy.PROPAGATE; + public boolean isDryRun() { return dryRun; } @@ -56,16 +68,18 @@ public void setWaitTimeout(long waitTimeout) { } public WriterOptions writerOptions() { - Builder builder = WriterOptions.builder(); - builder.waitForReplication(waitForReplication()); - builder.multiExec(multiExec); - builder.poolOptions(poolOptions()); - return builder.build(); + return WriterOptions.builder().replicaOptions(replicaOptions()).multiExec(multiExec).poolOptions(poolOptions()) + .build(); + } + + public DataStructureWriteOptions dataStructureOptions() { + return DataStructureWriteOptions.builder().mergePolicy(mergePolicy).streamIdPolicy(streamIdPolicy).build(); } - private Optional waitForReplication() { + private Optional replicaOptions() { if (waitReplicas > 0) { - return Optional.of(WaitForReplication.of(waitReplicas, Duration.ofMillis(waitTimeout))); + return Optional.of( + ReplicaOptions.builder().replicas(waitReplicas).timeout(Duration.ofMillis(waitTimeout)).build()); } return Optional.empty(); } diff --git a/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReplicateCommandContext.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReplicateCommandContext.java new file mode 100644 index 000000000..92b028747 --- /dev/null +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReplicateCommandContext.java @@ -0,0 +1,72 @@ +package com.redis.riot.cli.common; + +import com.redis.spring.batch.RedisItemReader.ComparatorBuilder; +import com.redis.spring.batch.RedisItemReader.ScanBuilder; +import com.redis.spring.batch.RedisItemWriter.DataStructureWriterBuilder; +import com.redis.spring.batch.RedisItemWriter.KeyDumpWriterBuilder; +import com.redis.spring.batch.common.DataStructure; +import com.redis.spring.batch.common.JobRunner; + +import io.lettuce.core.AbstractRedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.codec.ByteArrayCodec; +import io.lettuce.core.codec.RedisCodec; +import io.lettuce.core.codec.StringCodec; + +public class ReplicateCommandContext extends CommandContext { + + private final RedisOptions targetRedisOptions; + private final AbstractRedisClient targetRedisClient; + private final RedisURI targetRedisURI; + + public ReplicateCommandContext(JobRunner jobRunner, RedisOptions redisOptions, RedisOptions targetRedisOptions) { + super(jobRunner, redisOptions); + this.targetRedisOptions = targetRedisOptions; + this.targetRedisURI = targetRedisOptions.uri(); + this.targetRedisClient = targetRedisOptions.client(); + } + + @Override + public void close() throws Exception { + targetRedisClient.shutdown(); + targetRedisClient.getResources().shutdown(); + super.close(); + } + + public RedisOptions getTargetRedisOptions() { + return targetRedisOptions; + } + + public AbstractRedisClient getTargetRedisClient() { + return targetRedisClient; + } + + public RedisURI getTargetRedisURI() { + return targetRedisURI; + } + + public ComparatorBuilder comparator() { + return new ComparatorBuilder(getRedisClient(), targetRedisClient).jobRunner(jobRunner); + } + + public ScanBuilder> targetDataStructureReader(RedisCodec codec) { + return dataStructureReader(targetRedisClient, codec); + } + + public ScanBuilder> targetDataStructureReader() { + return targetDataStructureReader(StringCodec.UTF8); + } + + public DataStructureWriterBuilder targetDataStructureWriter() { + return targetDataStructureWriter(StringCodec.UTF8); + } + + public DataStructureWriterBuilder targetDataStructureWriter(RedisCodec codec) { + return dataStructureWriter(targetRedisClient, codec); + } + + public KeyDumpWriterBuilder targetKeyDumpWriter() { + return new KeyDumpWriterBuilder<>(targetRedisClient, ByteArrayCodec.INSTANCE); + } + +} diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/ReplicateOptions.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReplicationOptions.java similarity index 61% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/ReplicateOptions.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReplicationOptions.java index 48b9e0dd5..1dba19f65 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/ReplicateOptions.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/ReplicationOptions.java @@ -1,13 +1,16 @@ package com.redis.riot.cli.common; +import java.time.Duration; import java.util.Optional; +import com.redis.spring.batch.RedisItemReader.ComparatorBuilder; import com.redis.spring.batch.common.IntRange; import com.redis.spring.batch.reader.QueueOptions; +import com.redis.spring.batch.step.FlushingChunkProvider; import picocli.CommandLine.Option; -public class ReplicateOptions { +public class ReplicationOptions { public enum ReplicationMode { SNAPSHOT, LIVE, LIVEONLY, COMPARE @@ -35,6 +38,34 @@ public enum ReplicationStrategy { @Option(names = "--key-slot", description = "Key slot range filter for keyspace notifications.", paramLabel = "") private Optional keySlot = Optional.empty(); + @Option(names = "--ttl-tolerance", description = "Max TTL difference to use for dataset verification (default: ${DEFAULT-VALUE}).", paramLabel = "") + private long ttlTolerance = ComparatorBuilder.DEFAULT_TTL_TOLERANCE.toMillis(); + + @Option(names = "--show-diffs", description = "Print details of key mismatches during dataset verification.") + private boolean showDiffs; + + @Option(names = "--flush-interval", description = "Max duration between flushes (default: ${DEFAULT-VALUE}).", paramLabel = "") + private long flushInterval = FlushingChunkProvider.DEFAULT_FLUSHING_INTERVAL.toMillis(); + + @Option(names = "--idle-timeout", description = "Min duration of inactivity to consider transfer complete.", paramLabel = "") + private long idleTimeout; + + public long getFlushInterval() { + return flushInterval; + } + + public void setFlushInterval(long millis) { + this.flushInterval = millis; + } + + public long getIdleTimeout() { + return idleTimeout; + } + + public void setIdleTimeout(long millis) { + this.idleTimeout = millis; + } + public ReplicationMode getMode() { return mode; } @@ -87,4 +118,24 @@ public QueueOptions notificationQueueOptions() { return QueueOptions.builder().capacity(notificationQueueCapacity).build(); } + public long getTtlTolerance() { + return ttlTolerance; + } + + public void setTtlTolerance(long ttlTolerance) { + this.ttlTolerance = ttlTolerance; + } + + public Duration getTtlToleranceDuration() { + return Duration.ofMillis(ttlTolerance); + } + + public boolean isShowDiffs() { + return showDiffs; + } + + public void setShowDiffs(boolean showDiffs) { + this.showDiffs = showDiffs; + } + } diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/RiotExecutionStrategy.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RiotExecutionStrategy.java similarity index 100% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/RiotExecutionStrategy.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RiotExecutionStrategy.java diff --git a/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RuntimeIOException.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RuntimeIOException.java new file mode 100644 index 000000000..7f6fbc9a2 --- /dev/null +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/RuntimeIOException.java @@ -0,0 +1,22 @@ +package com.redis.riot.cli.common; + +public class RuntimeIOException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public RuntimeIOException() { + super(); + } + + public RuntimeIOException(String message) { + super(message); + } + + public RuntimeIOException(Throwable cause) { + super(cause); + } + + public RuntimeIOException(String message, Throwable cause) { + super(message, cause); + } +} \ No newline at end of file diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/StepSkipPolicy.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/StepSkipPolicy.java similarity index 100% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/StepSkipPolicy.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/StepSkipPolicy.java diff --git a/plugins/riot/src/main/java/com/redis/riot/cli/common/TransferOptions.java b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/TransferOptions.java similarity index 70% rename from plugins/riot/src/main/java/com/redis/riot/cli/common/TransferOptions.java rename to plugins/riot-cli/src/main/java/com/redis/riot/cli/common/TransferOptions.java index 5b7915fd0..bd2422779 100644 --- a/plugins/riot/src/main/java/com/redis/riot/cli/common/TransferOptions.java +++ b/plugins/riot-cli/src/main/java/com/redis/riot/cli/common/TransferOptions.java @@ -12,60 +12,62 @@ import org.springframework.batch.core.step.skip.NeverSkipItemSkipPolicy; import org.springframework.batch.core.step.skip.SkipPolicy; -import com.redis.spring.batch.common.FaultToleranceOptions; +import com.redis.spring.batch.common.StepOptions; import io.lettuce.core.RedisCommandExecutionException; import io.lettuce.core.RedisCommandTimeoutException; +import me.tongfei.progressbar.ProgressBarStyle; import picocli.CommandLine.Option; public class TransferOptions { - public static final int DEFAULT_CHUNK_SIZE = 0; + public static final ProgressBarStyle DEFAULT_PROGRESS_BAR_STYLE = ProgressBarStyle.COLORFUL_UNICODE_BLOCK; + public static final StepSkipPolicy DEFAULT_SKIP_POLICY = StepSkipPolicy.LIMIT; + public static final int DEFAULT_CHUNK_SIZE = StepOptions.DEFAULT_CHUNK_SIZE; + public static final int DEFAULT_THREADS = 1; + public static final int DEFAULT_SKIP_LIMIT = 3; + public static final Duration DEFAULT_PROGRESS_UPDATE_INTERVAL = Duration.ofMillis(1000); @Option(names = "--sleep", description = "Duration in ms to sleep before each item read (default: ${DEFAULT-VALUE}).", paramLabel = "") private long sleep; @Option(names = "--threads", description = "Thread count (default: ${DEFAULT-VALUE}).", paramLabel = "") - private int threads = 1; + private int threads = DEFAULT_THREADS; @Option(names = { "-b", "--batch" }, description = "Number of items in each batch (default: ${DEFAULT-VALUE}).", paramLabel = "") private int chunkSize = DEFAULT_CHUNK_SIZE; @Option(names = "--skip-policy", description = "Policy to determine if some processing should be skipped: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}).", paramLabel = "") - private StepSkipPolicy skipPolicy = StepSkipPolicy.LIMIT; + private StepSkipPolicy skipPolicy = DEFAULT_SKIP_POLICY; @Option(names = "--skip-limit", description = "LIMIT skip policy: max number of failed items before considering the transfer has failed (default: ${DEFAULT-VALUE}).", paramLabel = "") - private int skipLimit = 3; + private int skipLimit = DEFAULT_SKIP_LIMIT; @Option(names = "--progress", description = "Style of progress bar: ${COMPLETION-CANDIDATES} (default: ${DEFAULT-VALUE}),", paramLabel = "