From 9afd22bcbd80a0b1682fa745be82976e67821107 Mon Sep 17 00:00:00 2001 From: Annie Liang <64233642+xinlian12@users.noreply.github.com> Date: Mon, 25 Mar 2024 17:38:37 -0700 Subject: [PATCH] revertKafkaRelatedChange (#39392) * Revert "KafkaV2SinkConnector (#38973)" This reverts commit 6c983ab4f67008d8203fb12638f02f944ee358c5. * Revert "UsingTestContainerForKafkaIntegrationTests (#38884)" This reverts commit 12bec49fdf4b629b7a20c89b7fd22e18bff7b3d0. * Revert "KafkaV2SourceConnector (#38748)" This reverts commit 30835d90059b9991894a1a5da6f7cdfb3efe8f57. * revert one more change * revert change --------- Co-authored-by: annie-mac <xinlian@microsoft.com> --- .vscode/cspell.json | 3 +- .../checkstyle/checkstyle-suppressions.xml | 4 - eng/versioning/external_dependencies.txt | 5 - .../azure-cosmos-kafka-connect/CHANGELOG.md | 2 - .../doc/configuration-reference.md | 38 -- sdk/cosmos/azure-cosmos-kafka-connect/pom.xml | 64 +- .../src/docker/.gitignore | 8 - .../src/docker/Dockerfile | 7 - .../src/docker/docker-compose.yml | 191 ------ .../src/docker/startup.ps1 | 29 - .../src/docker/startup.sh | 27 - .../cosmos/kafka/connect/CosmosConfig.java | 26 + .../kafka/connect/CosmosDBSinkConnector.java | 63 -- .../connect/CosmosDBSourceConnector.java | 344 ---------- .../implementation/CosmosAccountConfig.java | 55 -- .../implementation/CosmosClientStore.java | 44 -- .../implementation/KafkaCosmosConfig.java | 224 ------- .../implementation/KafkaCosmosConstants.java | 13 - .../KafkaCosmosExceptionsHelper.java | 84 --- .../implementation/KafkaCosmosSchedulers.java | 19 - .../sink/CosmosDBWriteException.java | 20 - .../implementation/sink/CosmosSinkConfig.java | 325 ---------- .../sink/CosmosSinkContainersConfig.java | 24 - .../implementation/sink/CosmosSinkTask.java | 95 --- .../sink/CosmosSinkTaskConfig.java | 13 - .../sink/CosmosSinkWriteConfig.java | 54 -- .../connect/implementation/sink/IWriter.java | 13 - .../implementation/sink/IdStrategies.java | 31 - .../sink/ItemWriteStrategy.java | 32 - .../sink/KafkaCosmosBulkWriter.java | 335 ---------- .../sink/KafkaCosmosPointWriter.java | 179 ----- .../sink/KafkaCosmosWriterBase.java | 121 ---- .../implementation/sink/SinkOperation.java | 64 -- .../sink/SinkRecordTransformer.java | 110 ---- .../implementation/sink/StructToJsonMap.java | 118 ---- .../sink/ToleranceOnErrorLevel.java | 28 - .../sink/idstrategy/AbstractIdStrategy.java | 27 - .../idstrategy/AbstractIdStrategyConfig.java | 19 - .../sink/idstrategy/FullKeyStrategy.java | 16 - .../sink/idstrategy/IdStrategy.java | 11 - .../idstrategy/KafkaMetadataStrategy.java | 22 - .../KafkaMetadataStrategyConfig.java | 53 -- .../sink/idstrategy/ProvidedInConfig.java | 53 -- .../idstrategy/ProvidedInKeyStrategy.java | 10 - .../sink/idstrategy/ProvidedInStrategy.java | 46 -- .../idstrategy/ProvidedInValueStrategy.java | 10 - .../sink/idstrategy/TemplateStrategy.java | 71 -- .../idstrategy/TemplateStrategyConfig.java | 57 -- .../source/ContainersMetadataTopicOffset.java | 61 -- .../ContainersMetadataTopicPartition.java | 33 - .../source/CosmosChangeFeedModes.java | 27 - .../CosmosChangeFeedStartFromModes.java | 28 - .../source/CosmosMetadataConfig.java | 29 - .../source/CosmosSourceChangeFeedConfig.java | 40 -- .../source/CosmosSourceConfig.java | 444 ------------- .../source/CosmosSourceContainersConfig.java | 52 -- .../source/CosmosSourceMessageKeyConfig.java | 22 - .../CosmosSourceOffsetStorageReader.java | 51 -- .../source/CosmosSourceTask.java | 341 ---------- .../source/CosmosSourceTaskConfig.java | 139 ---- .../FeedRangeContinuationTopicOffset.java | 55 -- .../FeedRangeContinuationTopicPartition.java | 57 -- .../source/FeedRangeTaskUnit.java | 168 ----- .../source/FeedRangesMetadataTopicOffset.java | 71 -- .../FeedRangesMetadataTopicPartition.java | 41 -- .../implementation/source/ITaskUnit.java | 7 - .../implementation/source/JsonToStruct.java | 207 ------ .../source/MetadataMonitorThread.java | 338 ---------- .../source/MetadataTaskUnit.java | 173 ----- .../src/main/java/module-info.java | 3 +- .../src/test/connectorPlugins/build.ps1 | 20 - .../src/test/connectorPlugins/build.sh | 18 - .../connect/CosmosDBSinkConnectorTest.java | 154 ----- .../connect/CosmosDBSourceConnectorTest.java | 611 ------------------ .../connect/CosmosDbSinkConnectorITest.java | 99 --- .../connect/CosmosDbSourceConnectorITest.java | 113 ---- .../kafka/connect/InMemoryStorageReader.java | 42 -- .../kafka/connect/KafkaCosmosConfigEntry.java | 29 - .../connect/KafkaCosmosConnectContainer.java | 196 ------ .../KafkaCosmosIntegrationTestSuiteBase.java | 154 ----- .../connect/KafkaCosmosReflectionUtils.java | 72 --- .../KafkaCosmosTestConfigurations.java | 208 ------ .../connect/KafkaCosmosTestNGLogListener.java | 58 -- .../connect/KafkaCosmosTestSuiteBase.java | 262 -------- .../azure/cosmos/kafka/connect/TestItem.java | 49 -- .../sink/CosmosSinkTaskTest.java | 571 ---------------- .../idStrategy/ProvidedInStrategyTest.java | 223 ------- .../source/CosmosSourceTaskTest.java | 258 -------- .../source/MetadataMonitorThreadTest.java | 255 -------- .../src/test/resources/kafka-testng.xml | 35 - .../src/test/resources/log4j2.properties | 24 - .../azure/cosmos/CosmosAsyncContainer.java | 14 - .../ImplementationBridgeHelpers.java | 8 - .../src/main/java/module-info.java | 21 +- sdk/cosmos/kafka-integration-matrix.json | 14 - sdk/cosmos/tests.yml | 30 +- 96 files changed, 43 insertions(+), 9089 deletions(-) delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/doc/configuration-reference.md delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/docker/.gitignore delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/docker/Dockerfile delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/docker/docker-compose.yml delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/docker/startup.ps1 delete mode 100755 sdk/cosmos/azure-cosmos-kafka-connect/src/docker/startup.sh create mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnector.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnector.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosAccountConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosClientStore.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConstants.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosExceptionsHelper.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosSchedulers.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosDBWriteException.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkContainersConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTask.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkWriteConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IWriter.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IdStrategies.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ItemWriteStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosBulkWriter.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosPointWriter.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosWriterBase.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkOperation.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkRecordTransformer.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/StructToJsonMap.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ToleranceOnErrorLevel.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategyConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/FullKeyStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/IdStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategyConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInKeyStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInValueStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategy.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategyConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ContainersMetadataTopicOffset.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ContainersMetadataTopicPartition.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedModes.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedStartFromModes.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosMetadataConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceChangeFeedConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceContainersConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceMessageKeyConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceOffsetStorageReader.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTask.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTaskConfig.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeContinuationTopicOffset.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeContinuationTopicPartition.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeTaskUnit.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangesMetadataTopicOffset.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangesMetadataTopicPartition.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ITaskUnit.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/JsonToStruct.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThread.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataTaskUnit.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/connectorPlugins/build.ps1 delete mode 100755 sdk/cosmos/azure-cosmos-kafka-connect/src/test/connectorPlugins/build.sh delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnectorTest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnectorTest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSinkConnectorITest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSourceConnectorITest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/InMemoryStorageReader.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConfigEntry.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConnectContainer.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosIntegrationTestSuiteBase.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosReflectionUtils.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestConfigurations.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestNGLogListener.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/TestItem.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskTest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/idStrategy/ProvidedInStrategyTest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTaskTest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThreadTest.java delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/resources/kafka-testng.xml delete mode 100644 sdk/cosmos/azure-cosmos-kafka-connect/src/test/resources/log4j2.properties delete mode 100644 sdk/cosmos/kafka-integration-matrix.json diff --git a/.vscode/cspell.json b/.vscode/cspell.json index 32f0621d9435..d1ce21af5739 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -140,7 +140,6 @@ "sdk/cosmos/azure-cosmos-encryption/**", "sdk/cosmos/azure-cosmos-spark_3_2-12/**", "sdk/spring/azure-spring-data-cosmos/**", - "sdk/cosmos/azure-cosmos-kafka-connect/**", "sdk/deviceupdate/azure-iot-deviceupdate/**", "sdk/e2e/src/**", "sdk/eventgrid/azure-messaging-eventgrid-cloudnative-cloudevents/**", @@ -724,7 +723,7 @@ "words": [ "Pfast", "Pdirect", - "Pmulti", + "Pmulti", "Psplit", "Pquery", "Pcfp", diff --git a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml index b11579484fd0..f8ac3f277664 100755 --- a/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml +++ b/eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml @@ -316,10 +316,6 @@ the main ServiceBusClientBuilder. --> files="com.azure.cosmos.ClientSideRequestStatistics"/> <!-- Need OperatingSystemMXBean from sun to obtain cpu info --> <suppress checks="EnforceFinalFields" files="com.azure.spring.cloud.config.AppConfigurationPropertySourceLocator"/> <suppress checks="ConstantName" files="com.azure.spring.cloud.config.AppConfigurationPropertySourceLocator"/> - <suppress checks="com.azure.tools.checkstyle.checks.GoodLoggingCheck" - files="[/\\]azure-cosmos-kafka-connect[/\\]"/> - <suppress checks="com.azure.tools.checkstyle.checks.ExternalDependencyExposedCheck" files="com.azure.cosmos.kafka.connect.CosmosDBSourceConnector"/> - <suppress checks="com.azure.tools.checkstyle.checks.ExternalDependencyExposedCheck" files="com.azure.cosmos.kafka.connect.CosmosDBSinkConnector"/> <!-- Checkstyle suppressions for resource manager package --> <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.resourcemanager.*"/> diff --git a/eng/versioning/external_dependencies.txt b/eng/versioning/external_dependencies.txt index e879fca40aaf..a75828c48d2c 100644 --- a/eng/versioning/external_dependencies.txt +++ b/eng/versioning/external_dependencies.txt @@ -394,12 +394,7 @@ cosmos_org.scalastyle:scalastyle-maven-plugin;1.0.0 ## Cosmos Kafka connector under sdk\cosmos\azure-cosmos-kafka-connect\pom.xml # Cosmos Kafka connector runtime dependencies cosmos_org.apache.kafka:connect-api;3.6.0 -cosmos_com.jayway.jsonpath:json-path;2.9.0 # Cosmos Kafka connector tests only -cosmos_org.apache.kafka:connect-runtime;3.6.0 -cosmos_org.testcontainers:testcontainers;1.19.5 -cosmos_org.testcontainers:kafka;1.19.5 -cosmos_org.sourcelab:kafka-connect-client;4.0.4 # Maven Tools for Cosmos Kafka connector only cosmos_io.confluent:kafka-connect-maven-plugin;0.12.0 diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/CHANGELOG.md b/sdk/cosmos/azure-cosmos-kafka-connect/CHANGELOG.md index 840ffe29ca47..8d7cb3d876bc 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/CHANGELOG.md +++ b/sdk/cosmos/azure-cosmos-kafka-connect/CHANGELOG.md @@ -3,8 +3,6 @@ ### 1.0.0-beta.1 (Unreleased) #### Features Added -* Added Source connector. See [PR 38748](https://github.com/Azure/azure-sdk-for-java/pull/38748) -* Added Sink connector. See [PR 38973](https://github.com/Azure/azure-sdk-for-java/pull/38973) #### Breaking Changes diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/doc/configuration-reference.md b/sdk/cosmos/azure-cosmos-kafka-connect/doc/configuration-reference.md deleted file mode 100644 index 064aaab81a76..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/doc/configuration-reference.md +++ /dev/null @@ -1,38 +0,0 @@ -## Configuration Reference: - -## Generic Configuration -| Config Property Name | Default | Description | -|:---------------------------------------------|:--------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `kafka.connect.cosmos.accountEndpoint` | None | Cosmos DB Account Endpoint Uri | -| `kafka.connect.cosmos.accountEndpoint` | None | Cosmos DB Account Key | -| `kafka.connect.cosmos.useGatewayMode` | `false` | Flag to indicate whether to use gateway mode. By default it is false. | -| `kafka.connect.cosmos.preferredRegionsList` | `[]` | Preferred regions list to be used for a multi region Cosmos DB account. This is a comma separated value (e.g., `[East US, West US]` or `East US, West US`) provided preferred regions will be used as hint. You should use a collocated kafka cluster with your Cosmos DB account and pass the kafka cluster region as preferred region. See list of azure regions [here](https://docs.microsoft.com/dotnet/api/microsoft.azure.documents.locationnames?view=azure-dotnet&preserve-view=true). | -| `kafka.connect.cosmos.applicationName` | `""` | Application name. Will be added as the userAgent suffix. | - -## Source Connector Configuration -| Config Property Name | Default | Description | -|:----------------------------------------------------------|:-------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `kafka.connect.cosmos.source.database.name` | None | Cosmos DB database name. | -| `kafka.connect.cosmos.source.containers.includeAll` | `false` | Flag to indicate whether reading from all containers. | -| `kafka.connect.cosmos.source.containers.includedList` | `[]` | Containers included. This config will be ignored if kafka.connect.cosmos.source.includeAllContainers is true. | -| `kafka.connect.cosmos.source.containers.topicMap` | `[]` | A comma delimited list of Kafka topics mapped to Cosmos containers. For example: topic1#con1,topic2#con2. By default, container name is used as the name of the kafka topic to publish data to, can use this property to override the default config | -| `kafka.connect.cosmos.source.changeFeed.startFrom` | `Beginning` | ChangeFeed Start from settings (Now, Beginning or a certain point in time (UTC) for example 2020-02-10T14:15:03) - the default value is 'Beginning'. | -| `kafka.connect.cosmos.source.changeFeed.mode` | `LatestVersion` | ChangeFeed mode (LatestVersion or AllVersionsAndDeletes). | -| `kafka.connect.cosmos.source.changeFeed.maxItemCountHint` | `1000` | The maximum number of documents returned in a single change feed request. But the number of items received might be higher than the specified value if multiple items are changed by the same transaction. | -| `kafka.connect.cosmos.source.metadata.poll.delay.ms` | `300000` | Indicates how often to check the metadata changes (including container split/merge, adding/removing/recreated containers). When changes are detected, it will reconfigure the tasks. Default is 5 minutes. | -| `kafka.connect.cosmos.source.metadata.storage.topic` | `_cosmos.metadata.topic` | The name of the topic where the metadata are stored. The metadata topic will be created if it does not already exist, else it will use the pre-created topic. | -| `kafka.connect.cosmos.source.messageKey.enabled` | `true` | Whether to set the kafka record message key. | -| `kafka.connect.cosmos.source.messageKey.field` | `id` | The field to use as the message key. | - -## Sink Connector Configuration -| Config Property Name | Default | Description | -|:---------------------------------------------------------------|:--------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `kafka.connect.cosmos.sink.database.name` | None | Cosmos DB database name. | -| `kafka.connect.cosmos.sink.containers.topicMap` | None | A comma delimited list of Kafka topics mapped to Cosmos containers. For example: topic1#con1,topic2#con2. | -| `kafka.connect.cosmos.sink.errors.tolerance` | `None` | Error tolerance level after exhausting all retries. `None` for fail on error. `All` for log and continue | -| `kafka.connect.cosmos.sink.bulk.enabled` | `true` | Flag to indicate whether Cosmos DB bulk mode is enabled for Sink connector. By default it is true. | -| `kafka.connect.cosmos.sink.bulk.maxConcurrentCosmosPartitions` | `-1` | Cosmos DB Item Write Max Concurrent Cosmos Partitions. If not specified it will be determined based on the number of the container's physical partitions which would indicate every batch is expected to have data from all Cosmos physical partitions. If specified it indicates from at most how many Cosmos Physical Partitions each batch contains data. So this config can be used to make bulk processing more efficient when input data in each batch has been repartitioned to balance to how many Cosmos partitions each batch needs to write. This is mainly useful for very large containers (with hundreds of physical partitions. | -| `kafka.connect.cosmos.sink.bulk.initialBatchSize` | `1` | Cosmos DB initial bulk micro batch size - a micro batch will be flushed to the backend when the number of documents enqueued exceeds this size - or the target payload size is met. The micro batch size is getting automatically tuned based on the throttling rate. By default the initial micro batch size is 1. Reduce this when you want to avoid that the first few requests consume too many RUs. | -| `kafka.connect.cosmos.sink.write.strategy` | `ItemOverwrite` | Cosmos DB Item write Strategy: `ItemOverwrite` (using upsert), `ItemAppend` (using create, ignore pre-existing items i.e., Conflicts), `ItemDelete` (deletes based on id/pk of data frame), `ItemDeleteIfNotModified` (deletes based on id/pk of data frame if etag hasn't changed since collecting id/pk), `ItemOverwriteIfNotModified` (using create if etag is empty, update/replace with etag pre-condition otherwise, if document was updated the pre-condition failure is ignored) | -| `kafka.connect.cosmos.sink.maxRetryCount` | `10` | Cosmos DB max retry attempts on write failures for Sink connector. By default, the connector will retry on transient write errors for up to 10 times. | -| `kafka.connect.cosmos.sink.id.strategy` | `ProvidedInValueStrategy` | A strategy used to populate the document with an ``id``. Valid strategies are: ``TemplateStrategy``, ``FullKeyStrategy``, ``KafkaMetadataStrategy``, ``ProvidedInKeyStrategy``, ``ProvidedInValueStrategy``. Configuration properties prefixed with``id.strategy`` are passed through to the strategy. For example, when using ``id.strategy=TemplateStrategy`` , the property ``id.strategy.template`` is passed through to the template strategy and used to specify the template string to be used in constructing the ``id``. | diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml index e97f4ae4ebde..5049f848068c 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml +++ b/sdk/cosmos/azure-cosmos-kafka-connect/pom.xml @@ -37,6 +37,8 @@ Licensed under the MIT License. <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <jacoco.min.linecoverage>0.01</jacoco.min.linecoverage> <jacoco.min.branchcoverage>0.02</jacoco.min.branchcoverage> + <maven.compiler.source>11</maven.compiler.source> + <maven.compiler.target>11</maven.compiler.target> <shadingPrefix>azure_cosmos_kafka_connect</shadingPrefix> <!-- CosmosSkip - This is not a module we want/expect external customers to consume. Skip breaking API checks. --> @@ -46,15 +48,7 @@ Licensed under the MIT License. <javaModulesSurefireArgLine> --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect=ALL-UNNAMED --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.implementation=ALL-UNNAMED - --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.implementation.source=com.fasterxml.jackson.databind,ALL-UNNAMED - --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.implementation.sink=ALL-UNNAMED - --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.implementation.sink.idStrategy=ALL-UNNAMED - --add-opens com.azure.cosmos/com.azure.cosmos.implementation=ALL-UNNAMED - --add-opens com.azure.cosmos/com.azure.cosmos.implementation.routing=ALL-UNNAMED - --add-opens com.azure.cosmos/com.azure.cosmos.implementation.caches=ALL-UNNAMED - --add-opens com.azure.cosmos/com.azure.cosmos.implementation.faultinjection=ALL-UNNAMED - --add-opens com.azure.cosmos/com.azure.cosmos.implementation.apachecommons.lang=ALL-UNNAMED - --add-opens com.azure.cosmos/com.azure.cosmos.implementation.guava25.base=ALL-UNNAMED + --add-opens com.azure.cosmos.kafka.connect/com.azure.cosmos.kafka.connect.models=ALL-UNNAMED </javaModulesSurefireArgLine> </properties> @@ -86,13 +80,6 @@ Licensed under the MIT License. <scope>provided</scope> </dependency> - <dependency> - <groupId>com.azure</groupId> - <artifactId>azure-cosmos-test</artifactId> - <version>1.0.0-beta.7</version> <!-- {x-version-update;com.azure:azure-cosmos-test;current} --> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-collections4</artifactId> @@ -106,24 +93,6 @@ Licensed under the MIT License. <scope>test</scope> <version>1.10.0</version> <!-- {x-version-update;org.apache.commons:commons-text;external_dependency} --> </dependency> - <dependency> - <groupId>com.jayway.jsonpath</groupId> - <artifactId>json-path</artifactId> - <version>2.9.0</version> <!-- {x-version-update;cosmos_com.jayway.jsonpath:json-path;external_dependency} --> - </dependency> - - <dependency> - <groupId>org.apache.kafka</groupId> - <artifactId>connect-runtime</artifactId> - <version>3.6.0</version> <!-- {x-version-update;cosmos_org.apache.kafka:connect-runtime;external_dependency} --> - <scope>test</scope> - <exclusions> - <exclusion> - <artifactId>jackson-jaxrs-json-provider</artifactId> - <groupId>com.fasterxml.jackson.jaxrs</groupId> - </exclusion> - </exclusions> - </dependency> <dependency> <groupId>org.testng</groupId> @@ -191,24 +160,6 @@ Licensed under the MIT License. <version>1.14.12</version> <!-- {x-version-update;testdep_net.bytebuddy:byte-buddy-agent;external_dependency} --> <scope>test</scope> </dependency> - <dependency> - <groupId>org.testcontainers</groupId> - <artifactId>testcontainers</artifactId> - <version>1.19.5</version> <!-- {x-version-update;cosmos_org.testcontainers:testcontainers;external_dependency} --> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.testcontainers</groupId> - <artifactId>kafka</artifactId> - <version>1.19.5</version> <!-- {x-version-update;cosmos_org.testcontainers:kafka;external_dependency} --> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.sourcelab</groupId> - <artifactId>kafka-connect-client</artifactId> - <version>4.0.4</version> <!-- {x-version-update;cosmos_org.sourcelab:kafka-connect-client;external_dependency} --> - <scope>test</scope> - </dependency> </dependencies> <build> @@ -253,8 +204,6 @@ Licensed under the MIT License. <include>com.azure:*</include> <include>org.apache.kafka:connect-api:[3.6.0]</include> <!-- {x-include-update;cosmos_org.apache.kafka:connect-api;external_dependency} --> <include>io.confluent:kafka-connect-maven-plugin:[0.12.0]</include> <!-- {x-include-update;cosmos_io.confluent:kafka-connect-maven-plugin;external_dependency} --> - <include>com.jayway.jsonpath:json-path:[2.9.0]</include> <!-- {x-include-update;cosmos_com.jayway.jsonpath:json-path;external_dependency} --> - <include>org.sourcelab:kafka-connect-client:[4.0.4]</include> <!-- {x-include-update;cosmos_org.sourcelab:kafka-connect-client;external_dependency} --> </includes> </bannedDependencies> </rules> @@ -272,7 +221,6 @@ Licensed under the MIT License. <goal>shade</goal> </goals> <configuration> - <finalName>${project.artifactId}-${project.version}-jar-with-dependencies</finalName> <filters> <filter> <artifact>*:*:*:*</artifact> @@ -338,10 +286,6 @@ Licensed under the MIT License. <pattern>reactor</pattern> <shadedPattern>${shadingPrefix}.reactor</shadedPattern> </relocation> - <relocation> - <pattern>com.jayway.jsonpath</pattern> - <shadedPattern>${shadingPrefix}.com.jayway.jsonpath</shadedPattern> - </relocation> </relocations> <artifactSet> <excludes> @@ -479,7 +423,7 @@ Licensed under the MIT License. </profile> <profile> <!-- integration tests, requires Cosmos DB Emulator Endpoint --> - <id>kafka</id> + <id>kafka-integration</id> <properties> <test.groups>kafka</test.groups> </properties> diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/.gitignore b/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/.gitignore deleted file mode 100644 index b170e557735b..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -connectors/ -log.txt - -# Exclude all temporary files in resources -!resources/*example -resources/sink.properties -resources/source.properties -resources/standalone.properties diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/Dockerfile b/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/Dockerfile deleted file mode 100644 index 37da2123ab86..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -# Build the Cosmos DB Connectors on top of the Kafka Connect image -FROM confluentinc/cp-kafka-connect:7.5.0 - -# Install datagen connector -RUN confluent-hub install --no-prompt confluentinc/kafka-connect-datagen:latest - -COPY connectors/ /etc/kafka-connect/jars diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/docker-compose.yml b/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/docker-compose.yml deleted file mode 100644 index 6f733fee3ab7..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/docker-compose.yml +++ /dev/null @@ -1,191 +0,0 @@ -# Adapted from https://github.com/confluentinc/cp-all-in-one and https://github.com/simplesteph/kafka-stack-docker-compose -version: '2.1' - -services: - zookeeper: - image: confluentinc/cp-zookeeper:7.5.0 - restart: unless-stopped - hostname: zookeeper - container_name: zookeeper - ports: - - "2181:2181" - environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ZOOKEEPER_TICK_TIME: 2000 - - broker: - image: confluentinc/cp-server:7.5.0 - hostname: broker - container_name: broker - ports: - - "9092:9092" - - "9101:9101" - environment: - KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 - KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_JMX_PORT: 9101 - KAFKA_JMX_HOSTNAME: localhost - KAFKA_CONFLUENT_SCHEMA_REGISTRY_URL: http://schema-registry:8081 - CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092 - CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1 - CONFLUENT_METRICS_ENABLE: 'true' - CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous' - depends_on: - - zookeeper - - schema-registry: - image: confluentinc/cp-schema-registry:7.5.0 - hostname: schema-registry - container_name: schema-registry - ports: - - "8081:8081" - environment: - SCHEMA_REGISTRY_HOST_NAME: schema-registry - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092' - SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081 - depends_on: - - broker - - schema-registry-ui: - image: landoop/schema-registry-ui:0.9.4 - hostname: schema-registry-ui - container_name: schema-registry-ui - ports: - - "9001:8000" - environment: - SCHEMAREGISTRY_URL: http://schema-registry:8081/ - PROXY: "true" - depends_on: - - schema-registry - - rest-proxy: - image: confluentinc/cp-kafka-rest:7.5.0 - ports: - - "8082:8082" - hostname: rest-proxy - container_name: rest-proxy - environment: - KAFKA_REST_HOST_NAME: rest-proxy - KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092' - KAFKA_REST_LISTENERS: "http://0.0.0.0:8082" - KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' - depends_on: - - broker - - schema-registry - - kafka-topics-ui: - image: landoop/kafka-topics-ui:0.9.4 - hostname: kafka-topics-ui - container_name: kafka-topics-ui - ports: - - "9000:8000" - environment: - KAFKA_REST_PROXY_URL: "http://rest-proxy:8082/" - PROXY: "true" - depends_on: - - zookeeper - - broker - - schema-registry - - rest-proxy - - connect: - # Using modified version of confluentinc/cp-kafka-connect:6.0.0 to avoid dealing with volume mounts - image: cosmosdb-kafka-connect:latest - hostname: connect - container_name: connect - ports: - - "8083:8083" - environment: - CONNECT_BOOTSTRAP_SERVERS: "broker:29092" - CONNECT_REST_ADVERTISED_HOST_NAME: "connect" - CONNECT_REST_PORT: 8083 - CONNECT_GROUP_ID: compose-connect-group - CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1" - CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000 - CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1" - CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1" - CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" - CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' - CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081' - CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.storage.StringConverter" - CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter" - # CLASSPATH required due to CC-2422 - CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-6.0.0.jar - CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" - CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" - CONNECT_LOG4J_ROOT_LOGLEVEL: "WARN" - CONNECT_LOG4J_LOGGERS: "org.apache.kafka=INFO,org.reflections=ERROR,com.azure.cosmos.kafka=DEBUG" - CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components,/etc/kafka-connect/jars' - depends_on: - - zookeeper - - broker - - schema-registry - - rest-proxy - - control-center: - image: confluentinc/cp-enterprise-control-center:7.5.0 - hostname: control-center - container_name: control-center - ports: - - "9021:9021" - environment: - CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092' - CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'http://connect:8083' - CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088" - CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8088" - CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" - CONTROL_CENTER_REPLICATION_FACTOR: 1 - CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1 - CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1 - CONTROL_CENTER_CONNECT_HEALTHCHECK_ENDPOINT: '/connectors' - CONFLUENT_METRICS_TOPIC_REPLICATION: 1 - PORT: 9021 - depends_on: - - broker - - schema-registry - - connect - - ksqldb-server: - image: confluentinc/cp-ksqldb-server:7.5.0 - hostname: ksqldb-server - container_name: ksqldb-server - ports: - - "8088:8088" - environment: - KSQL_CONFIG_DIR: "/etc/ksql" - KSQL_BOOTSTRAP_SERVERS: "broker:29092" - KSQL_HOST_NAME: ksqldb-server - KSQL_LISTENERS: "http://0.0.0.0:8088" - KSQL_CACHE_MAX_BYTES_BUFFERING: 0 - KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081" - KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor" - KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor" - KSQL_KSQL_CONNECT_URL: "http://connect:8083" - KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1 - KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: 'true' - KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: 'true' - depends_on: - - broker - - connect - - zoonavigator: - image: elkozmon/zoonavigator:0.8.0 - container_name: zoonavigator - ports: - - "9004:8000" - environment: - HTTP_PORT: 8000 - AUTO_CONNECT_CONNECTION_STRING: zookeeper:2181 \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/startup.ps1 b/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/startup.ps1 deleted file mode 100644 index 9cb5c13150cd..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/startup.ps1 +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env pwsh -$ErrorActionPreference='Stop' -cd $PSScriptRoot -Write-Host "Shutting down Docker Compose orchestration..." -docker-compose down - -Write-Host "Deleting prior Cosmos DB connectors..." -rm -rf "$PSScriptRoot/connectors" -New-Item -Path "$PSScriptRoot" -ItemType "directory" -Name "connectors" -Force | Out-Null -cd $PSScriptRoot/../.. - -Write-Host "Rebuilding Cosmos DB connectors..." -mvn clean package -DskipTests -Dmaven.javadoc.skip -copy target\*-jar-with-dependencies.jar $PSScriptRoot/connectors -cd $PSScriptRoot - -Write-Host "Adding custom Insert UUID SMT" -cd $PSScriptRoot/connectors -git clone https://github.com/confluentinc/kafka-connect-insert-uuid.git insertuuid -q && cd insertuuid -mvn clean package -DskipTests=true -copy target\*.jar $PSScriptRoot/connectors -rm -rf "$PSScriptRoot/connectors/insertuuid" -cd $PSScriptRoot - -Write-Host "Building Cosmos DB Kafka Connect Docker image" -docker build . -t cosmosdb-kafka-connect:latest - -Write-Host "Starting Docker Compose..." -docker-compose up -d \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/startup.sh b/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/startup.sh deleted file mode 100755 index 1f5dbd056648..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/docker/startup.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -echo "Shutting down Docker Compose orchestration..." -docker-compose down - -echo "Deleting prior Cosmos DB connectors..." -rm -rf connectors -mkdir connectors -cd ../../ - -echo "Rebuilding Cosmos DB connectors..." -mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true -cp target/*-jar-with-dependencies.jar src/docker/connectors -cd src/docker - -echo "Adding custom Insert UUID SMT" -cd connectors -git clone https://github.com/confluentinc/kafka-connect-insert-uuid.git insertuuid -q && cd insertuuid -mvn clean package -DskipTests=true -cp target/*.jar ../ -cd .. && rm -rf insertuuid -cd ../ - -echo "Building Cosmos DB Kafka Connect Docker image" -docker build . -t cosmosdb-kafka-connect:latest - -echo "Starting Docker Compose..." -docker-compose up -d \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosConfig.java new file mode 100644 index 000000000000..41a7703fde72 --- /dev/null +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosConfig.java @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.cosmos.kafka.connect; + +import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.config.ConfigDef; + +import java.util.Map; + +/** + * Configuration for Cosmos DB Kafka connector + */ +public class CosmosConfig extends AbstractConfig { + + /** + * Initializes a new instance of the Cosmos DB Kafka Connector configuration + * @param definition The configuration definition + * @param originals The original config values + * @param configProviderProps The configuration overrides for this provider + * @param doLog Flag indicating whether the configuration should be logged + */ + public CosmosConfig(ConfigDef definition, Map<?, ?> originals, Map<String, ?> configProviderProps, boolean doLog) { + super(definition, originals, configProviderProps, doLog); + } +} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnector.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnector.java deleted file mode 100644 index 91319959ba69..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnector.java +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConstants; -import com.azure.cosmos.kafka.connect.implementation.sink.CosmosSinkConfig; -import com.azure.cosmos.kafka.connect.implementation.sink.CosmosSinkTask; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.connect.connector.Task; -import org.apache.kafka.connect.sink.SinkConnector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -/** - * A Sink connector that publishes topic messages to CosmosDB. - */ -public class CosmosDBSinkConnector extends SinkConnector { - private static final Logger LOGGER = LoggerFactory.getLogger(CosmosDBSinkConnector.class); - - private CosmosSinkConfig sinkConfig; - - @Override - public void start(Map<String, String> props) { - LOGGER.info("Starting the kafka cosmos sink connector"); - this.sinkConfig = new CosmosSinkConfig(props); - } - - @Override - public Class<? extends Task> taskClass() { - return CosmosSinkTask.class; - } - - @Override - public List<Map<String, String>> taskConfigs(int maxTasks) { - LOGGER.info("Setting task configurations with maxTasks {}", maxTasks); - List<Map<String, String>> configs = new ArrayList<>(); - for (int i = 0; i < maxTasks; i++) { - configs.add(this.sinkConfig.originalsStrings()); - } - - return configs; - } - - @Override - public void stop() { - LOGGER.debug("Kafka Cosmos sink connector {} is stopped."); - } - - @Override - public ConfigDef config() { - return CosmosSinkConfig.getConfigDef(); - } - - @Override - public String version() { - return KafkaCosmosConstants.CURRENT_VERSION; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnector.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnector.java deleted file mode 100644 index 4bbb794a5906..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnector.java +++ /dev/null @@ -1,344 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.implementation.Strings; -import com.azure.cosmos.implementation.apachecommons.lang.tuple.Pair; -import com.azure.cosmos.implementation.changefeed.common.ChangeFeedState; -import com.azure.cosmos.implementation.changefeed.common.ChangeFeedStateV1; -import com.azure.cosmos.implementation.feedranges.FeedRangeContinuation; -import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; -import com.azure.cosmos.implementation.feedranges.FeedRangeInternal; -import com.azure.cosmos.implementation.query.CompositeContinuationToken; -import com.azure.cosmos.implementation.routing.Range; -import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConstants; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceConfig; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceOffsetStorageReader; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceTask; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceTaskConfig; -import com.azure.cosmos.kafka.connect.implementation.source.FeedRangeContinuationTopicOffset; -import com.azure.cosmos.kafka.connect.implementation.source.FeedRangeTaskUnit; -import com.azure.cosmos.kafka.connect.implementation.source.FeedRangesMetadataTopicOffset; -import com.azure.cosmos.kafka.connect.implementation.source.MetadataMonitorThread; -import com.azure.cosmos.kafka.connect.implementation.source.MetadataTaskUnit; -import com.azure.cosmos.models.CosmosContainerProperties; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.connect.connector.Task; -import org.apache.kafka.connect.source.SourceConnector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -/*** - * The CosmosDb source connector. - */ -public class CosmosDBSourceConnector extends SourceConnector { - private static final Logger LOGGER = LoggerFactory.getLogger(CosmosDBSourceConnector.class); - private CosmosSourceConfig config; - private CosmosAsyncClient cosmosClient; - private MetadataMonitorThread monitorThread; - private CosmosSourceOffsetStorageReader offsetStorageReader; - - @Override - public void start(Map<String, String> props) { - LOGGER.info("Starting the kafka cosmos source connector"); - this.config = new CosmosSourceConfig(props); - this.cosmosClient = CosmosClientStore.getCosmosClient(this.config.getAccountConfig()); - this.offsetStorageReader = new CosmosSourceOffsetStorageReader(this.context().offsetStorageReader()); - this.monitorThread = new MetadataMonitorThread( - this.config.getContainersConfig(), - this.config.getMetadataConfig(), - this.context(), - this.offsetStorageReader, - this.cosmosClient - ); - - this.monitorThread.start(); - } - - @Override - public Class<? extends Task> taskClass() { - return CosmosSourceTask.class; - } - - @Override - public List<Map<String, String>> taskConfigs(int maxTasks) { - // For now, we start with copying data by feed range - // but in the future, we can have more optimization based on the data size etc. - return this.getTaskConfigs(maxTasks); - } - - @Override - public void stop() { - LOGGER.info("Stopping Kafka CosmosDB source connector"); - if (this.cosmosClient != null) { - LOGGER.debug("Closing cosmos client"); - this.cosmosClient.close(); - } - - if (this.monitorThread != null) { - LOGGER.debug("Closing monitoring thread"); - this.monitorThread.close(); - } - } - - @Override - public ConfigDef config() { - return CosmosSourceConfig.getConfigDef(); - } - - @Override - public String version() { - return KafkaCosmosConstants.CURRENT_VERSION; - } - - private List<Map<String, String>> getTaskConfigs(int maxTasks) { - Pair<MetadataTaskUnit, List<FeedRangeTaskUnit>> taskUnits = this.getAllTaskUnits(); - - // The metadataTaskUnit is a one time only task when the connector starts/restarts, - // so there is no need to assign a dedicated task thread for it - // we are just going to assign it to one of the tasks which processing feedRanges tasks - List<List<FeedRangeTaskUnit>> partitionedTaskUnits = new ArrayList<>(); - if (taskUnits.getRight().size() <= maxTasks) { - partitionedTaskUnits.addAll( - taskUnits.getRight().stream().map(taskUnit -> Arrays.asList(taskUnit)).collect(Collectors.toList())); - } else { - // using round-robin fashion to assign tasks to each buckets - for (int i = 0; i < maxTasks; i++) { - partitionedTaskUnits.add(new ArrayList<>()); - } - - for (int i = 0; i < taskUnits.getRight().size(); i++) { - partitionedTaskUnits.get(i % maxTasks).add(taskUnits.getRight().get(i)); - } - } - - List<Map<String, String>> allSourceTaskConfigs = new ArrayList<>(); - partitionedTaskUnits.forEach(feedRangeTaskUnits -> { - Map<String, String> taskConfigs = this.config.originalsStrings(); - taskConfigs.putAll( - CosmosSourceTaskConfig.getFeedRangeTaskUnitsConfigMap(feedRangeTaskUnits)); - allSourceTaskConfigs.add(taskConfigs); - }); - - // assign the metadata task to the last of the task config as it has least number of feedRange task units - allSourceTaskConfigs - .get(allSourceTaskConfigs.size() - 1) - .putAll(CosmosSourceTaskConfig.getMetadataTaskUnitConfigMap(taskUnits.getLeft())); - - return allSourceTaskConfigs; - } - - private Pair<MetadataTaskUnit, List<FeedRangeTaskUnit>> getAllTaskUnits() { - List<CosmosContainerProperties> allContainers = this.monitorThread.getAllContainers().block(); - Map<String, String> containerTopicMap = this.getContainersTopicMap(allContainers); - List<FeedRangeTaskUnit> allFeedRangeTaskUnits = new ArrayList<>(); - Map<String, List<Range<String>>> updatedContainerToFeedRangesMap = new ConcurrentHashMap<>(); - - for (CosmosContainerProperties containerProperties : allContainers) { - Map<Range<String>, String> effectiveFeedRangesContinuationMap = - this.getEffectiveFeedRangesContinuationMap( - this.config.getContainersConfig().getDatabaseName(), - containerProperties); - - updatedContainerToFeedRangesMap.put( - containerProperties.getResourceId(), - effectiveFeedRangesContinuationMap.keySet().stream().collect(Collectors.toList()) - ); - - // add feedRange task unit - for (Range<String> effectiveFeedRange : effectiveFeedRangesContinuationMap.keySet()) { - allFeedRangeTaskUnits.add( - new FeedRangeTaskUnit( - this.config.getContainersConfig().getDatabaseName(), - containerProperties.getId(), - containerProperties.getResourceId(), - effectiveFeedRange, - effectiveFeedRangesContinuationMap.get(effectiveFeedRange), - containerTopicMap.get(containerProperties.getId()) - ) - ); - } - } - - MetadataTaskUnit metadataTaskUnit = - new MetadataTaskUnit( - this.config.getContainersConfig().getDatabaseName(), - allContainers.stream().map(CosmosContainerProperties::getResourceId).collect(Collectors.toList()), - updatedContainerToFeedRangesMap, - this.config.getMetadataConfig().getMetadataTopicName()); - - return Pair.of(metadataTaskUnit, allFeedRangeTaskUnits); - } - - private Map<Range<String>, String> getEffectiveFeedRangesContinuationMap( - String databaseName, - CosmosContainerProperties containerProperties) { - // Return effective feed ranges to be used for copying data from container - // - If there is no existing offset, then use the result from container.getFeedRanges - // - If there is existing offset, then deciding the final range sets based on: - // -----If we can find offset by matching the feedRange, then use the feedRange - // -----If we can not find offset by matching the exact feedRange, - // then it means the feedRanges of the containers have changed either due to split or merge. - // If a merge is detected, we will use the matched feedRanges from the offsets, - // otherwise use the current feedRange, but constructing the continuationState based on the previous feedRange - - List<Range<String>> containerFeedRanges = this.getFeedRanges(containerProperties); - containerFeedRanges.sort(new Comparator<Range<String>>() { - @Override - public int compare(Range<String> o1, Range<String> o2) { - return o1.getMin().compareTo(o2.getMin()); - } - }); - - FeedRangesMetadataTopicOffset feedRangesMetadataTopicOffset = - this.offsetStorageReader.getFeedRangesMetadataOffset(databaseName, containerProperties.getResourceId()); - - Map<Range<String>, String> effectiveFeedRangesContinuationMap = new LinkedHashMap<>(); - - for (Range<String> containerFeedRange : containerFeedRanges) { - if (feedRangesMetadataTopicOffset == null) { - // there is no existing offset, return the current container feedRanges with continuationState null (start from refresh) - effectiveFeedRangesContinuationMap.put(containerFeedRange, Strings.Emtpy); - } else { - // there is existing offsets, need to find out effective feedRanges based on the offset - effectiveFeedRangesContinuationMap.putAll( - this.getEffectiveContinuationMapForSingleFeedRange( - databaseName, - containerProperties.getResourceId(), - containerFeedRange, - feedRangesMetadataTopicOffset.getFeedRanges()) - ); - } - } - - return effectiveFeedRangesContinuationMap; - } - - private Map<Range<String>, String> getEffectiveContinuationMapForSingleFeedRange( - String databaseName, - String containerRid, - Range<String> containerFeedRange, - List<Range<String>> rangesFromMetadataTopicOffset) { - - //first try to find out whether there is exact feedRange matching - FeedRangeContinuationTopicOffset feedRangeContinuationTopicOffset = - this.offsetStorageReader.getFeedRangeContinuationOffset(databaseName, containerRid, containerFeedRange); - - Map<Range<String>, String> effectiveContinuationMap = new LinkedHashMap<>(); - if (feedRangeContinuationTopicOffset != null) { - // we can find the continuation offset based on exact feedRange matching - effectiveContinuationMap.put( - containerFeedRange, - this.getContinuationStateFromOffset( - containerRid, - feedRangeContinuationTopicOffset, - containerFeedRange)); - - return effectiveContinuationMap; - } - - // we can not find the continuation offset based on the exact feed range matching - // it means the previous Partition key range could have gone due to container split/merge - // need to find out overlapped feedRanges from offset - List<Range<String>> overlappedFeedRangesFromOffset = - rangesFromMetadataTopicOffset - .stream() - .filter(rangeFromOffset -> Range.checkOverlapping(rangeFromOffset, containerFeedRange)) - .collect(Collectors.toList()); - - if (overlappedFeedRangesFromOffset.size() == 1) { - // split - use the current containerFeedRange, but construct the continuationState based on the feedRange from offset - effectiveContinuationMap.put( - containerFeedRange, - this.getContinuationStateFromOffset( - containerRid, - this.offsetStorageReader.getFeedRangeContinuationOffset(databaseName, containerRid, overlappedFeedRangesFromOffset.get(0)), - containerFeedRange)); - return effectiveContinuationMap; - } - - if (overlappedFeedRangesFromOffset.size() > 1) { - // merge - use the feed ranges from the offset - for (Range<String> overlappedRangeFromOffset : overlappedFeedRangesFromOffset) { - effectiveContinuationMap.put( - overlappedRangeFromOffset, - this.getContinuationStateFromOffset( - containerRid, - this.offsetStorageReader.getFeedRangeContinuationOffset(databaseName, containerRid, overlappedRangeFromOffset), - overlappedRangeFromOffset)); - } - - return effectiveContinuationMap; - } - - // Can not find overlapped ranges from offset, this should never happen, fail - LOGGER.error("Can not find overlapped ranges for feedRange {}", containerFeedRange); - throw new IllegalStateException("Can not find overlapped ranges for feedRange " + containerFeedRange); - } - - private String getContinuationStateFromOffset( - String containerRid, - FeedRangeContinuationTopicOffset feedRangeContinuationTopicOffset, - Range<String> range) { - - ChangeFeedState stateFromOffset = ChangeFeedStateV1.fromString(feedRangeContinuationTopicOffset.getContinuationState()); - String itemLsn = feedRangeContinuationTopicOffset.getItemLsn(); - return new ChangeFeedStateV1( - containerRid, - new FeedRangeEpkImpl(range), - stateFromOffset.getMode(), - stateFromOffset.getStartFromSettings(), - FeedRangeContinuation.create( - containerRid, - new FeedRangeEpkImpl(range), - Arrays.asList(new CompositeContinuationToken(itemLsn, range)))).toString(); - } - - private List<Range<String>> getFeedRanges(CosmosContainerProperties containerProperties) { - return this.cosmosClient - .getDatabase(this.config.getContainersConfig().getDatabaseName()) - .getContainer(containerProperties.getId()) - .getFeedRanges() - .onErrorMap(throwable -> - KafkaCosmosExceptionsHelper.convertToConnectException( - throwable, - "GetFeedRanges failed for container " + containerProperties.getId())) - .block() - .stream() - .map(feedRange -> FeedRangeInternal.normalizeRange(((FeedRangeEpkImpl) feedRange).getRange())) - .collect(Collectors.toList()); - } - - private Map<String, String> getContainersTopicMap(List<CosmosContainerProperties> allContainers) { - Map<String, String> topicMapFromConfig = this.config.getContainersConfig().getContainerToTopicMap(); - Map<String, String> effectiveContainersTopicMap = new HashMap<>(); - allContainers.forEach(containerProperties -> { - // by default, we are using container id as the topic name as well unless customer override through containers.topicMap - if (topicMapFromConfig.containsKey(containerProperties.getId())) { - effectiveContainersTopicMap.put( - containerProperties.getId(), - topicMapFromConfig.get(containerProperties.getId())); - } else { - effectiveContainersTopicMap.put( - containerProperties.getId(), - containerProperties.getId()); - } - }); - - return effectiveContainersTopicMap; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosAccountConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosAccountConfig.java deleted file mode 100644 index 49e2f731a05a..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosAccountConfig.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; - -import java.util.List; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; - -public class CosmosAccountConfig { - private final String endpoint; - private final String accountKey; - private final String applicationName; - private final boolean useGatewayMode; - private final List<String> preferredRegionsList; - - public CosmosAccountConfig( - String endpoint, - String accountKey, - String applicationName, - boolean useGatewayMode, - List<String> preferredRegionsList) { - - checkArgument(StringUtils.isNotEmpty(endpoint), "Argument 'endpoint' should not be null"); - checkArgument(StringUtils.isNotEmpty(accountKey), "Argument 'accountKey' should not be null"); - - this.endpoint = endpoint; - this.accountKey = accountKey; - this.applicationName = applicationName; - this.useGatewayMode = useGatewayMode; - this.preferredRegionsList = preferredRegionsList; - } - - public String getEndpoint() { - return endpoint; - } - - public String getAccountKey() { - return accountKey; - } - - public String getApplicationName() { - return applicationName; - } - - public boolean isUseGatewayMode() { - return useGatewayMode; - } - - public List<String> getPreferredRegionsList() { - return preferredRegionsList; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosClientStore.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosClientStore.java deleted file mode 100644 index 10589369d0e8..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/CosmosClientStore.java +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosClientBuilder; -import com.azure.cosmos.GatewayConnectionConfig; -import com.azure.cosmos.ThrottlingRetryOptions; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; - -import java.time.Duration; - -public class CosmosClientStore { - public static CosmosAsyncClient getCosmosClient(CosmosAccountConfig accountConfig) { - if (accountConfig == null) { - return null; - } - - CosmosClientBuilder cosmosClientBuilder = new CosmosClientBuilder() - .endpoint(accountConfig.getEndpoint()) - .key(accountConfig.getAccountKey()) - .preferredRegions(accountConfig.getPreferredRegionsList()) - .throttlingRetryOptions( - new ThrottlingRetryOptions() - .setMaxRetryAttemptsOnThrottledRequests(Integer.MAX_VALUE) - .setMaxRetryWaitTime(Duration.ofSeconds((Integer.MAX_VALUE / 1000) - 1))) - .userAgentSuffix(getUserAgentSuffix(accountConfig)); - - if (accountConfig.isUseGatewayMode()) { - cosmosClientBuilder.gatewayMode(new GatewayConnectionConfig().setMaxConnectionPoolSize(10000)); - } - - return cosmosClientBuilder.buildAsyncClient(); - } - - private static String getUserAgentSuffix(CosmosAccountConfig accountConfig) { - if (StringUtils.isNotEmpty(accountConfig.getApplicationName())) { - return KafkaCosmosConstants.USER_AGENT_SUFFIX + "|" + accountConfig.getApplicationName(); - } - - return KafkaCosmosConstants.USER_AGENT_SUFFIX; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConfig.java deleted file mode 100644 index 5c75056b6f7b..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConfig.java +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation; - -import com.azure.cosmos.implementation.Strings; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceContainersConfig; -import org.apache.kafka.common.config.AbstractConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigException; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Common Configuration for Cosmos DB Kafka source connector and sink connector. - */ -public class KafkaCosmosConfig extends AbstractConfig { - protected static final ConfigDef.Validator NON_EMPTY_STRING = new ConfigDef.NonEmptyString(); - private static final String CONFIG_PREFIX = "kafka.connect.cosmos."; - - // Account config - private static final String ACCOUNT_ENDPOINT_CONFIG = CONFIG_PREFIX + "accountEndpoint"; - private static final String ACCOUNT_ENDPOINT_CONFIG_DOC = "Cosmos DB Account Endpoint Uri."; - private static final String ACCOUNT_ENDPOINT_CONFIG_DISPLAY = "Cosmos DB Account Endpoint Uri."; - - private static final String ACCOUNT_KEY_CONFIG = CONFIG_PREFIX + "accountKey"; - private static final String ACCOUNT_KEY_CONFIG_DOC = "Cosmos DB Account Key."; - private static final String ACCOUNT_KEY_CONFIG_DISPLAY = "Cosmos DB Account Key."; - - private static final String USE_GATEWAY_MODE = CONFIG_PREFIX + "useGatewayMode"; - private static final String USE_GATEWAY_MODE_DOC = "Flag to indicate whether to use gateway mode. By default it is false."; - private static final String USE_GATEWAY_MODE_DISPLAY = "Use gateway mode."; - private static final boolean DEFAULT_USE_GATEWAY_MODE = false; - - private static final String PREFERRED_REGIONS_LIST = CONFIG_PREFIX + "preferredRegionsList"; - private static final String PREFERRED_REGIONS_LIST_DOC = "Preferred regions list to be used for a multi region Cosmos DB account. " - + "This is a comma separated value (e.g., `[East US, West US]` or `East US, West US`) provided preferred regions will be used as hint. " - + "You should use a collocated kafka cluster with your Cosmos DB account and pass the kafka cluster region as preferred region. " - + "See list of azure regions [here](https://docs.microsoft.com/dotnet/api/microsoft.azure.documents.locationnames?view=azure-dotnet&preserve-view=true)."; - private static final String PREFERRED_REGIONS_LIST_DISPLAY = "Preferred regions list."; - - private static final String APPLICATION_NAME = CONFIG_PREFIX + "applicationName"; - private static final String APPLICATION_NAME_DOC = "Application name. Will be added as the userAgent suffix."; - private static final String APPLICATION_NAME_DISPLAY = "Application name."; - - private final CosmosAccountConfig accountConfig; - - public KafkaCosmosConfig(ConfigDef config, Map<String, ?> parsedConfig) { - super(config, parsedConfig); - this.accountConfig = this.parseAccountConfig(); - } - - private CosmosAccountConfig parseAccountConfig() { - String endpoint = this.getString(ACCOUNT_ENDPOINT_CONFIG); - String accountKey = this.getPassword(ACCOUNT_KEY_CONFIG).value(); - String applicationName = this.getString(APPLICATION_NAME); - boolean useGatewayMode = this.getBoolean(USE_GATEWAY_MODE); - List<String> preferredRegionList = this.getPreferredRegionList(); - - return new CosmosAccountConfig( - endpoint, - accountKey, - applicationName, - useGatewayMode, - preferredRegionList); - } - - private List<String> getPreferredRegionList() { - return convertToList(this.getString(PREFERRED_REGIONS_LIST)); - } - - public static ConfigDef getConfigDef() { - ConfigDef configDef = new ConfigDef(); - - defineAccountConfig(configDef); - - return configDef; - } - - private static void defineAccountConfig(ConfigDef result) { - final String accountGroupName = "account"; - int accountGroupOrder = 0; - - // For optional config, need to provide a default value - result - .define( - ACCOUNT_ENDPOINT_CONFIG, - ConfigDef.Type.STRING, - ConfigDef.NO_DEFAULT_VALUE, - new AccountEndpointValidator(), - ConfigDef.Importance.HIGH, - ACCOUNT_ENDPOINT_CONFIG_DOC, - accountGroupName, - accountGroupOrder++, - ConfigDef.Width.LONG, - ACCOUNT_ENDPOINT_CONFIG_DISPLAY - ) - .define( - ACCOUNT_KEY_CONFIG, - ConfigDef.Type.PASSWORD, - ConfigDef.NO_DEFAULT_VALUE, - ConfigDef.Importance.HIGH, - ACCOUNT_KEY_CONFIG_DOC, - accountGroupName, - accountGroupOrder++, - ConfigDef.Width.LONG, - ACCOUNT_KEY_CONFIG_DISPLAY - ) - .define( - APPLICATION_NAME, - ConfigDef.Type.STRING, - Strings.Emtpy, - ConfigDef.Importance.MEDIUM, - APPLICATION_NAME_DOC, - accountGroupName, - accountGroupOrder++, - ConfigDef.Width.LONG, - APPLICATION_NAME_DISPLAY - ) - .define( - USE_GATEWAY_MODE, - ConfigDef.Type.BOOLEAN, - DEFAULT_USE_GATEWAY_MODE, - ConfigDef.Importance.LOW, - USE_GATEWAY_MODE_DOC, - accountGroupName, - accountGroupOrder++, - ConfigDef.Width.MEDIUM, - USE_GATEWAY_MODE_DISPLAY - ) - .define( - PREFERRED_REGIONS_LIST, - ConfigDef.Type.STRING, - Strings.Emtpy, - ConfigDef.Importance.HIGH, - PREFERRED_REGIONS_LIST_DOC, - accountGroupName, - accountGroupOrder++, - ConfigDef.Width.LONG, - PREFERRED_REGIONS_LIST_DISPLAY - ); - } - - public CosmosAccountConfig getAccountConfig() { - return accountConfig; - } - - protected static List<String> convertToList(String configValue) { - if (StringUtils.isNotEmpty(configValue)) { - if (configValue.startsWith("[") && configValue.endsWith("]")) { - configValue = configValue.substring(1, configValue.length() - 1); - } - - return Arrays.stream(configValue.split(",")).map(String::trim).collect(Collectors.toList()); - } - - return new ArrayList<>(); - } - - public static class AccountEndpointValidator implements ConfigDef.Validator { - @Override - @SuppressWarnings("unchecked") - public void ensureValid(String name, Object o) { - String accountEndpointUriString = (String) o; - if (StringUtils.isEmpty(accountEndpointUriString)) { - throw new ConfigException(name, o, "Account endpoint can not be empty"); - } - - try { - new URL(accountEndpointUriString); - } catch (MalformedURLException e) { - throw new ConfigException(name, o, "Invalid account endpoint."); - } - } - - @Override - public String toString() { - return "Account endpoint"; - } - } - - public static class ContainersTopicMapValidator implements ConfigDef.Validator { - private static final String INVALID_TOPIC_MAP_FORMAT = - "Invalid entry for topic-container map. The topic-container map should be a comma-delimited " - + "list of Kafka topic to Cosmos containers. Each mapping should be a pair of Kafka " - + "topic and Cosmos container separated by '#'. For example: topic1#con1,topic2#con2."; - - @Override - @SuppressWarnings("unchecked") - public void ensureValid(String name, Object o) { - String configValue = (String) o; - if (StringUtils.isEmpty(configValue)) { - return; - } - - List<String> containerTopicMapList = convertToList(configValue); - - // validate each item should be in topic#container format - boolean invalidFormatExists = - containerTopicMapList - .stream() - .anyMatch(containerTopicMap -> - containerTopicMap - .split(CosmosSourceContainersConfig.CONTAINER_TOPIC_MAP_SEPARATOR) - .length != 2); - - if (invalidFormatExists) { - throw new ConfigException(name, o, INVALID_TOPIC_MAP_FORMAT); - } - } - - @Override - public String toString() { - return "Containers topic map"; - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConstants.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConstants.java deleted file mode 100644 index 01a89ef20f53..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosConstants.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation; - -import com.azure.core.util.CoreUtils; - -public class KafkaCosmosConstants { - public static final String PROPERTIES_FILE_NAME = "azure-cosmos-kafka-connect.properties"; - public static final String CURRENT_VERSION = CoreUtils.getProperties(PROPERTIES_FILE_NAME).get("version"); - public static final String CURRENT_NAME = CoreUtils.getProperties(PROPERTIES_FILE_NAME).get("name"); - public static final String USER_AGENT_SUFFIX = String.format("KafkaConnect/%s/%s", CURRENT_NAME, CURRENT_VERSION); -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosExceptionsHelper.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosExceptionsHelper.java deleted file mode 100644 index 0d5a8bb8759e..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosExceptionsHelper.java +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation; - -import com.azure.cosmos.CosmosException; -import com.azure.cosmos.implementation.HttpConstants; -import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.errors.RetriableException; - -public class KafkaCosmosExceptionsHelper { - public static boolean isTransientFailure(int statusCode, int substatusCode) { - return statusCode == HttpConstants.StatusCodes.GONE - || statusCode == HttpConstants.StatusCodes.SERVICE_UNAVAILABLE - || statusCode == HttpConstants.StatusCodes.INTERNAL_SERVER_ERROR - || statusCode == HttpConstants.StatusCodes.REQUEST_TIMEOUT - || (statusCode == HttpConstants.StatusCodes.NOTFOUND && substatusCode == HttpConstants.SubStatusCodes.READ_SESSION_NOT_AVAILABLE); - - } - - public static boolean isTransientFailure(Throwable e) { - if (e instanceof CosmosException) { - return isTransientFailure(((CosmosException) e).getStatusCode(), ((CosmosException) e).getSubStatusCode()); - } - - return false; - } - - public static boolean isFeedRangeGoneException(Throwable throwable) { - if (throwable instanceof CosmosException) { - return isFeedRangeGoneException( - ((CosmosException) throwable).getStatusCode(), - ((CosmosException) throwable).getSubStatusCode()); - } - - return false; - } - - public static boolean isFeedRangeGoneException(int statusCode, int substatusCode) { - return statusCode == HttpConstants.StatusCodes.GONE - && (substatusCode == HttpConstants.SubStatusCodes.PARTITION_KEY_RANGE_GONE - || substatusCode == HttpConstants.SubStatusCodes.COMPLETING_SPLIT_OR_MERGE); - } - - public static ConnectException convertToConnectException(Throwable throwable, String message) { - if (KafkaCosmosExceptionsHelper.isTransientFailure(throwable)) { - return new RetriableException(message, throwable); - } - - return new ConnectException(message, throwable); - } - - public static boolean isResourceExistsException(Throwable throwable) { - if (throwable instanceof CosmosException) { - return ((CosmosException) throwable).getStatusCode() == HttpConstants.StatusCodes.CONFLICT; - } - - return false; - } - - public static boolean isNotFoundException(Throwable throwable) { - if (throwable instanceof CosmosException) { - return ((CosmosException) throwable).getStatusCode() == HttpConstants.StatusCodes.NOTFOUND; - } - - return false; - } - - public static boolean isPreconditionFailedException(Throwable throwable) { - if (throwable instanceof CosmosException) { - return ((CosmosException) throwable).getStatusCode() == HttpConstants.StatusCodes.PRECONDITION_FAILED; - } - - return false; - } - - public static boolean isTimeoutException(Throwable throwable) { - if (throwable instanceof CosmosException) { - return ((CosmosException) throwable).getStatusCode() == HttpConstants.StatusCodes.REQUEST_TIMEOUT; - } - - return false; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosSchedulers.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosSchedulers.java deleted file mode 100644 index 081d0baa30f2..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/KafkaCosmosSchedulers.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation; - -import reactor.core.scheduler.Scheduler; -import reactor.core.scheduler.Schedulers; - -public class KafkaCosmosSchedulers { - private static final String SINK_BOUNDED_ELASTIC_THREAD_NAME = "sink-bounded-elastic"; - private static final int TTL_FOR_SCHEDULER_WORKER_IN_SECONDS = 60; // same as BoundedElasticScheduler.DEFAULT_TTL_SECONDS - public static final Scheduler SINK_BOUNDED_ELASTIC = Schedulers.newBoundedElastic( - Schedulers.DEFAULT_BOUNDED_ELASTIC_SIZE, - Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, - SINK_BOUNDED_ELASTIC_THREAD_NAME, - TTL_FOR_SCHEDULER_WORKER_IN_SECONDS, - true - ); -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosDBWriteException.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosDBWriteException.java deleted file mode 100644 index 69f99f35f8ba..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosDBWriteException.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import org.apache.kafka.connect.errors.ConnectException; - -/** - * Generic CosmosDb sink write exceptions. - */ -public class CosmosDBWriteException extends ConnectException { - /** - * - */ - private static final long serialVersionUID = 1L; - - public CosmosDBWriteException(String message) { - super(message); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkConfig.java deleted file mode 100644 index d029105846a5..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkConfig.java +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigException; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Common Configuration for Cosmos DB Kafka sink connector. - */ -public class CosmosSinkConfig extends KafkaCosmosConfig { - private static final String SINK_CONFIG_PREFIX = "kafka.connect.cosmos.sink."; - - // error tolerance - public static final String TOLERANCE_ON_ERROR_CONFIG = SINK_CONFIG_PREFIX + "errors.tolerance"; - public static final String TOLERANCE_ON_ERROR_DOC = - "Error tolerance level after exhausting all retries. 'None' for fail on error. 'All' for log and continue"; - public static final String TOLERANCE_ON_ERROR_DISPLAY = "Error tolerance level."; - public static final String DEFAULT_TOLERANCE_ON_ERROR = ToleranceOnErrorLevel.NONE.getName(); - - // sink bulk config - public static final String BULK_ENABLED_CONF = SINK_CONFIG_PREFIX + "bulk.enabled"; - private static final String BULK_ENABLED_DOC = - "Flag to indicate whether Cosmos DB bulk mode is enabled for Sink connector. By default it is true."; - private static final String BULK_ENABLED_DISPLAY = "enable bulk mode."; - private static final boolean DEFAULT_BULK_ENABLED = true; - - // TODO[Public Preview]: Add other write config, for example patch, bulkUpdate - public static final String BULK_MAX_CONCURRENT_PARTITIONS_CONF = SINK_CONFIG_PREFIX + "bulk.maxConcurrentCosmosPartitions"; - private static final String BULK_MAX_CONCURRENT_PARTITIONS_DOC = - "Cosmos DB Item Write Max Concurrent Cosmos Partitions." - + " If not specified it will be determined based on the number of the container's physical partitions -" - + " which would indicate every batch is expected to have data from all Cosmos physical partitions." - + " If specified it indicates from at most how many Cosmos Physical Partitions each batch contains data." - + " So this config can be used to make bulk processing more efficient when input data in each batch has been" - + " repartitioned to balance to how many Cosmos partitions each batch needs to write. This is mainly" - + " useful for very large containers (with hundreds of physical partitions)."; - private static final String BULK_MAX_CONCURRENT_PARTITIONS_DISPLAY = "Cosmos DB Item Write Max Concurrent Cosmos Partitions."; - private static final int DEFAULT_BULK_MAX_CONCURRENT_PARTITIONS = -1; - - public static final String BULK_INITIAL_BATCH_SIZE_CONF = SINK_CONFIG_PREFIX + "bulk.initialBatchSize"; - private static final String BULK_INITIAL_BATCH_SIZE_DOC = - "Cosmos DB initial bulk micro batch size - a micro batch will be flushed to the backend " - + "when the number of documents enqueued exceeds this size - or the target payload size is met. The micro batch " - + "size is getting automatically tuned based on the throttling rate. By default the " - + "initial micro batch size is 1. Reduce this when you want to avoid that the first few requests consume " - + "too many RUs."; - private static final String BULK_INITIAL_BATCH_SIZE_DISPLAY = "Cosmos DB initial bulk micro batch size."; - private static final int DEFAULT_BULK_INITIAL_BATCH_SIZE = 1; // start with small value to avoid initial RU spike - - // write strategy - public static final String WRITE_STRATEGY_CONF = SINK_CONFIG_PREFIX + "write.strategy"; - private static final String WRITE_STRATEGY_DOC = "Cosmos DB Item write Strategy: `ItemOverwrite` (using upsert), `ItemAppend` (using create, " - + "ignore pre-existing items i.e., Conflicts), `ItemDelete` (deletes based on id/pk of data frame), " - + "`ItemDeleteIfNotModified` (deletes based on id/pk of data frame if etag hasn't changed since collecting " - + "id/pk), `ItemOverwriteIfNotModified` (using create if etag is empty, update/replace with etag pre-condition " - + "otherwise, if document was updated the pre-condition failure is ignored)"; - private static final String WRITE_STRATEGY_DISPLAY = "Cosmos DB Item write Strategy."; - private static final String DEFAULT_WRITE_STRATEGY = ItemWriteStrategy.ITEM_OVERWRITE.getName(); - - // max retry - public static final String MAX_RETRY_COUNT_CONF = SINK_CONFIG_PREFIX + "maxRetryCount"; - private static final String MAX_RETRY_COUNT_DOC = - "Cosmos DB max retry attempts on write failures for Sink connector. By default, the connector will retry on transient write errors for up to 10 times."; - private static final String MAX_RETRY_COUNT_DISPLAY = "Cosmos DB max retry attempts on write failures for Sink connector."; - private static final int DEFAULT_MAX_RETRY_COUNT = 10; - - // database name - private static final String DATABASE_NAME_CONF = SINK_CONFIG_PREFIX + "database.name"; - private static final String DATABASE_NAME_CONF_DOC = "Cosmos DB database name."; - private static final String DATABASE_NAME_CONF_DISPLAY = "Cosmos DB database name."; - - // container topic map - public static final String CONTAINERS_TOPIC_MAP_CONF = SINK_CONFIG_PREFIX + "containers.topicMap"; - private static final String CONTAINERS_TOPIC_MAP_DOC = - "A comma delimited list of Kafka topics mapped to Cosmos containers. For example: topic1#con1,topic2#con2."; - private static final String CONTAINERS_TOPIC_MAP_DISPLAY = "Topic-Container map"; - - // TODO[Public preview]: re-examine idStrategy implementation - // id.strategy - public static final String ID_STRATEGY_CONF = SINK_CONFIG_PREFIX + "id.strategy"; - public static final String ID_STRATEGY_DOC = - "A strategy used to populate the document with an ``id``. Valid strategies are: " - + "``TemplateStrategy``, ``FullKeyStrategy``, ``KafkaMetadataStrategy``, " - + "``ProvidedInKeyStrategy``, ``ProvidedInValueStrategy``. Configuration " - + "properties prefixed with``id.strategy`` are passed through to the strategy. For " - + "example, when using ``id.strategy=TemplateStrategy`` , " - + "the property ``id.strategy.template`` is passed through to the template strategy " - + "and used to specify the template string to be used in constructing the ``id``."; - public static final String ID_STRATEGY_DISPLAY = "ID Strategy"; - public static final String DEFAULT_ID_STRATEGY = IdStrategies.PROVIDED_IN_VALUE_STRATEGY.getName(); - - // TODO[Public Preview] Verify whether compression need to happen in connector - - private final CosmosSinkWriteConfig writeConfig; - private final CosmosSinkContainersConfig containersConfig; - private final IdStrategies idStrategy; - - public CosmosSinkConfig(Map<String, ?> parsedConfig) { - this(getConfigDef(), parsedConfig); - } - - public CosmosSinkConfig(ConfigDef config, Map<String, ?> parsedConfig) { - super(config, parsedConfig); - this.writeConfig = this.parseWriteConfig(); - this.containersConfig = this.parseContainersConfig(); - this.idStrategy = this.parseIdStrategy(); - } - - public static ConfigDef getConfigDef() { - ConfigDef configDef = KafkaCosmosConfig.getConfigDef(); - - defineWriteConfig(configDef); - defineContainersConfig(configDef); - defineIdStrategyConfig(configDef); - return configDef; - } - - private static void defineWriteConfig(ConfigDef configDef) { - final String writeConfigGroupName = "Write config"; - int writeConfigGroupOrder = 0; - configDef - .define( - BULK_ENABLED_CONF, - ConfigDef.Type.BOOLEAN, - DEFAULT_BULK_ENABLED, - ConfigDef.Importance.MEDIUM, - BULK_ENABLED_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - BULK_ENABLED_DISPLAY - ) - .define( - BULK_MAX_CONCURRENT_PARTITIONS_CONF, - ConfigDef.Type.INT, - DEFAULT_BULK_MAX_CONCURRENT_PARTITIONS, - ConfigDef.Importance.LOW, - BULK_MAX_CONCURRENT_PARTITIONS_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - BULK_MAX_CONCURRENT_PARTITIONS_DISPLAY - ) - .define( - BULK_INITIAL_BATCH_SIZE_CONF, - ConfigDef.Type.INT, - DEFAULT_BULK_INITIAL_BATCH_SIZE, - ConfigDef.Importance.MEDIUM, - BULK_INITIAL_BATCH_SIZE_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - BULK_INITIAL_BATCH_SIZE_DISPLAY - ) - .define( - WRITE_STRATEGY_CONF, - ConfigDef.Type.STRING, - DEFAULT_WRITE_STRATEGY, - new ItemWriteStrategyValidator(), - ConfigDef.Importance.HIGH, - WRITE_STRATEGY_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.LONG, - WRITE_STRATEGY_DISPLAY - ) - .define( - MAX_RETRY_COUNT_CONF, - ConfigDef.Type.INT, - DEFAULT_MAX_RETRY_COUNT, - ConfigDef.Importance.MEDIUM, - MAX_RETRY_COUNT_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - MAX_RETRY_COUNT_DISPLAY - ) - .define( - TOLERANCE_ON_ERROR_CONFIG, - ConfigDef.Type.STRING, - DEFAULT_TOLERANCE_ON_ERROR, - ConfigDef.Importance.HIGH, - TOLERANCE_ON_ERROR_DOC, - writeConfigGroupName, - writeConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - TOLERANCE_ON_ERROR_DISPLAY - ); - } - - private static void defineContainersConfig(ConfigDef configDef) { - final String containersGroupName = "Containers"; - int containersGroupOrder = 0; - - configDef - .define( - DATABASE_NAME_CONF, - ConfigDef.Type.STRING, - ConfigDef.NO_DEFAULT_VALUE, - NON_EMPTY_STRING, - ConfigDef.Importance.HIGH, - DATABASE_NAME_CONF_DOC, - containersGroupName, - containersGroupOrder++, - ConfigDef.Width.LONG, - DATABASE_NAME_CONF_DISPLAY - ) - .define( - CONTAINERS_TOPIC_MAP_CONF, - ConfigDef.Type.STRING, - ConfigDef.NO_DEFAULT_VALUE, - new ContainersTopicMapValidator(), - ConfigDef.Importance.MEDIUM, - CONTAINERS_TOPIC_MAP_DOC, - containersGroupName, - containersGroupOrder++, - ConfigDef.Width.LONG, - CONTAINERS_TOPIC_MAP_DISPLAY - ); - } - - private static void defineIdStrategyConfig(ConfigDef configDef) { - final String idStrategyConfigGroupName = "ID Strategy"; - int idStrategyConfigGroupOrder = 0; - configDef - .define( - ID_STRATEGY_CONF, - ConfigDef.Type.STRING, - DEFAULT_ID_STRATEGY, - ConfigDef.Importance.HIGH, - ID_STRATEGY_DOC, - idStrategyConfigGroupName, - idStrategyConfigGroupOrder++, - ConfigDef.Width.MEDIUM, - ID_STRATEGY_DISPLAY); - } - - private CosmosSinkWriteConfig parseWriteConfig() { - boolean bulkEnabled = this.getBoolean(BULK_ENABLED_CONF); - int bulkMaxConcurrentCosmosPartitions = this.getInt(BULK_MAX_CONCURRENT_PARTITIONS_CONF); - int bulkInitialBatchSize = this.getInt(BULK_INITIAL_BATCH_SIZE_CONF); - ItemWriteStrategy writeStrategy = this.parseItemWriteStrategy(); - int maxRetryCount = this.getInt(MAX_RETRY_COUNT_CONF); - ToleranceOnErrorLevel toleranceOnErrorLevel = this.parseToleranceOnErrorLevel(); - - return new CosmosSinkWriteConfig( - bulkEnabled, - bulkMaxConcurrentCosmosPartitions, - bulkInitialBatchSize, - writeStrategy, - maxRetryCount, - toleranceOnErrorLevel); - } - - private CosmosSinkContainersConfig parseContainersConfig() { - String databaseName = this.getString(DATABASE_NAME_CONF); - Map<String, String> topicToContainerMap = this.getTopicToContainerMap(); - - return new CosmosSinkContainersConfig(databaseName, topicToContainerMap); - } - - private Map<String, String> getTopicToContainerMap() { - List<String> containersTopicMapList = convertToList(this.getString(CONTAINERS_TOPIC_MAP_CONF)); - return containersTopicMapList - .stream() - .map(containerTopicMapString -> containerTopicMapString.split("#")) - .collect( - Collectors.toMap( - containerTopicMapArray -> containerTopicMapArray[0], - containerTopicMapArray -> containerTopicMapArray[1])); - } - - private ItemWriteStrategy parseItemWriteStrategy() { - return ItemWriteStrategy.fromName(this.getString(WRITE_STRATEGY_CONF)); - } - - private ToleranceOnErrorLevel parseToleranceOnErrorLevel() { - return ToleranceOnErrorLevel.fromName(this.getString(TOLERANCE_ON_ERROR_CONFIG)); - } - - private IdStrategies parseIdStrategy() { - return IdStrategies.fromName(this.getString(ID_STRATEGY_CONF)); - } - - public CosmosSinkWriteConfig getWriteConfig() { - return writeConfig; - } - - public CosmosSinkContainersConfig getContainersConfig() { - return containersConfig; - } - - public IdStrategies getIdStrategy() { - return idStrategy; - } - - public static class ItemWriteStrategyValidator implements ConfigDef.Validator { - @Override - @SuppressWarnings("unchecked") - public void ensureValid(String name, Object o) { - String itemWriteStrategyString = (String) o; - if (StringUtils.isEmpty(itemWriteStrategyString)) { - throw new ConfigException(name, o, "WriteStrategy can not be empty or null"); - } - - ItemWriteStrategy itemWriteStrategy = ItemWriteStrategy.fromName(itemWriteStrategyString); - if (itemWriteStrategy == null) { - throw new ConfigException(name, o, "Invalid ItemWriteStrategy. Allowed values " + ItemWriteStrategy.values()); - } - } - - @Override - public String toString() { - return "ItemWriteStrategy. Only allow " + ItemWriteStrategy.values(); - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkContainersConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkContainersConfig.java deleted file mode 100644 index 9cb3273f8ebd..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkContainersConfig.java +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import java.util.Map; - -public class CosmosSinkContainersConfig { - private final String databaseName; - private final Map<String, String> topicToContainerMap; - - public CosmosSinkContainersConfig(String databaseName, Map<String, String> topicToContainerMap) { - this.databaseName = databaseName; - this.topicToContainerMap = topicToContainerMap; - } - - public String getDatabaseName() { - return databaseName; - } - - public Map<String, String> getTopicToContainerMap() { - return topicToContainerMap; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTask.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTask.java deleted file mode 100644 index 79d881169b3d..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTask.java +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConstants; -import org.apache.kafka.connect.sink.SinkRecord; -import org.apache.kafka.connect.sink.SinkTask; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class CosmosSinkTask extends SinkTask { - private static final Logger LOGGER = LoggerFactory.getLogger(CosmosSinkTask.class); - private CosmosSinkTaskConfig sinkTaskConfig; - private CosmosAsyncClient cosmosClient; - private SinkRecordTransformer sinkRecordTransformer; - private IWriter cosmosWriter; - - @Override - public String version() { - return KafkaCosmosConstants.CURRENT_VERSION; - } - - @Override - public void start(Map<String, String> props) { - LOGGER.info("Starting the kafka cosmos sink task..."); - this.sinkTaskConfig = new CosmosSinkTaskConfig(props); - this.cosmosClient = CosmosClientStore.getCosmosClient(this.sinkTaskConfig.getAccountConfig()); - this.sinkRecordTransformer = new SinkRecordTransformer(this.sinkTaskConfig); - - if (this.sinkTaskConfig.getWriteConfig().isBulkEnabled()) { - this.cosmosWriter = - new KafkaCosmosBulkWriter(this.sinkTaskConfig.getWriteConfig(), this.context.errantRecordReporter()); - } else { - this.cosmosWriter = - new KafkaCosmosPointWriter(this.sinkTaskConfig.getWriteConfig(), context.errantRecordReporter()); - } - - // TODO[public preview]: in V1, it will create the database if does not exists, but why? - } - - @Override - public void put(Collection<SinkRecord> records) { - if (records == null || records.isEmpty()) { - LOGGER.debug("No records to be written"); - return; - } - - LOGGER.debug("Sending {} records to be written", records.size()); - - // group by container - Map<String, List<SinkRecord>> recordsByContainer = - records.stream().collect( - Collectors.groupingBy( - record -> this.sinkTaskConfig - .getContainersConfig() - .getTopicToContainerMap() - .getOrDefault(record.topic(), StringUtils.EMPTY))); - - if (recordsByContainer.containsKey(StringUtils.EMPTY)) { - throw new IllegalStateException("There is no container defined for topics " + recordsByContainer.get(StringUtils.EMPTY)); - } - - for (Map.Entry<String, List<SinkRecord>> entry : recordsByContainer.entrySet()) { - String containerName = entry.getKey(); - CosmosAsyncContainer container = - this.cosmosClient - .getDatabase(this.sinkTaskConfig.getContainersConfig().getDatabaseName()) - .getContainer(containerName); - - // transform sink records, for example populating id - List<SinkRecord> transformedRecords = sinkRecordTransformer.transform(containerName, entry.getValue()); - this.cosmosWriter.write(container, transformedRecords); - } - } - - @Override - public void stop() { - LOGGER.info("Stopping Kafka CosmosDB sink task..."); - if (this.cosmosClient != null) { - this.cosmosClient.close(); - } - - this.cosmosClient = null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskConfig.java deleted file mode 100644 index f438f3f620eb..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskConfig.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import java.util.Map; - -// Currently the sink task config shares the same config as sink connector config -public class CosmosSinkTaskConfig extends CosmosSinkConfig { - public CosmosSinkTaskConfig(Map<String, ?> parsedConfig) { - super(parsedConfig); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkWriteConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkWriteConfig.java deleted file mode 100644 index 98d758deb6e3..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkWriteConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -public class CosmosSinkWriteConfig { - private final boolean bulkEnabled; - private final int bulkMaxConcurrentCosmosPartitions; - private final int bulkInitialBatchSize; - private final ItemWriteStrategy itemWriteStrategy; - private final int maxRetryCount; - - private final ToleranceOnErrorLevel toleranceOnErrorLevel; - - public CosmosSinkWriteConfig( - boolean bulkEnabled, - int bulkMaxConcurrentCosmosPartitions, - int bulkInitialBatchSize, - ItemWriteStrategy itemWriteStrategy, - int maxRetryCount, - ToleranceOnErrorLevel toleranceOnErrorLevel) { - - this.bulkEnabled = bulkEnabled; - this.bulkMaxConcurrentCosmosPartitions = bulkMaxConcurrentCosmosPartitions; - this.bulkInitialBatchSize = bulkInitialBatchSize; - this.itemWriteStrategy = itemWriteStrategy; - this.maxRetryCount = maxRetryCount; - this.toleranceOnErrorLevel = toleranceOnErrorLevel; - } - - public boolean isBulkEnabled() { - return bulkEnabled; - } - - public int getBulkMaxConcurrentCosmosPartitions() { - return bulkMaxConcurrentCosmosPartitions; - } - - public int getBulkInitialBatchSize() { - return bulkInitialBatchSize; - } - - public ItemWriteStrategy getItemWriteStrategy() { - return itemWriteStrategy; - } - - public int getMaxRetryCount() { - return maxRetryCount; - } - - public ToleranceOnErrorLevel getToleranceOnErrorLevel() { - return toleranceOnErrorLevel; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IWriter.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IWriter.java deleted file mode 100644 index 18cee5577b7d..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IWriter.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.CosmosAsyncContainer; -import org.apache.kafka.connect.sink.SinkRecord; - -import java.util.List; - -public interface IWriter { - void write(CosmosAsyncContainer container, List<SinkRecord> sinkRecords); -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IdStrategies.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IdStrategies.java deleted file mode 100644 index dcc3568dc2fb..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/IdStrategies.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -public enum IdStrategies { - TEMPLATE_STRATEGY("TemplateStrategy"), - FULL_KEY_STRATEGY("FullKeyStrategy"), - KAFKA_METADATA_STRATEGY("KafkaMetadataStrategy"), - PROVIDED_IN_KEY_STRATEGY("ProvidedInKeyStrategy"), - PROVIDED_IN_VALUE_STRATEGY("ProvidedInValueStrategy"); - - private final String name; - - IdStrategies(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public static IdStrategies fromName(String name) { - for (IdStrategies mode : IdStrategies.values()) { - if (mode.getName().equalsIgnoreCase(name)) { - return mode; - } - } - return null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ItemWriteStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ItemWriteStrategy.java deleted file mode 100644 index 988f577b5a26..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ItemWriteStrategy.java +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -public enum ItemWriteStrategy { - ITEM_OVERWRITE("ItemOverwrite"), - ITEM_APPEND("ItemAppend"), - ITEM_DELETE("ItemDelete"), - ITEM_DELETE_IF_NOT_MODIFIED("ItemDeleteIfNotModified"), - ITEM_OVERWRITE_IF_NOT_MODIFIED("ItemOverwriteIfNotModified"); - - // TODO[Public Preview] Add ItemPatch, ItemBulkUpdate - private final String name; - - ItemWriteStrategy(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public static ItemWriteStrategy fromName(String name) { - for (ItemWriteStrategy mode : ItemWriteStrategy.values()) { - if (mode.getName().equalsIgnoreCase(name)) { - return mode; - } - } - return null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosBulkWriter.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosBulkWriter.java deleted file mode 100644 index 97515fa530fe..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosBulkWriter.java +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.BridgeInternal; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.CosmosException; -import com.azure.cosmos.implementation.HttpConstants; -import com.azure.cosmos.implementation.ImplementationBridgeHelpers; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosSchedulers; -import com.azure.cosmos.models.CosmosBulkExecutionOptions; -import com.azure.cosmos.models.CosmosBulkItemRequestOptions; -import com.azure.cosmos.models.CosmosBulkItemResponse; -import com.azure.cosmos.models.CosmosBulkOperationResponse; -import com.azure.cosmos.models.CosmosBulkOperations; -import com.azure.cosmos.models.CosmosItemOperation; -import com.azure.cosmos.models.PartitionKeyDefinition; -import org.apache.kafka.connect.sink.ErrantRecordReporter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.core.publisher.SignalType; -import reactor.core.publisher.Sinks; - -import java.time.Duration; -import java.util.List; -import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -public class KafkaCosmosBulkWriter extends KafkaCosmosWriterBase { - private static final Logger LOGGER = LoggerFactory.getLogger(KafkaCosmosBulkWriter.class); - private static final int MAX_DELAY_ON_408_REQUEST_TIMEOUT_IN_MS = 10000; - private static final int MIN_DELAY_ON_408_REQUEST_TIMEOUT_IN_MS = 1000; - private static final Random RANDOM = new Random(); - - private final CosmosSinkWriteConfig writeConfig; - private final Sinks.EmitFailureHandler emitFailureHandler; - - public KafkaCosmosBulkWriter( - CosmosSinkWriteConfig writeConfig, - ErrantRecordReporter errantRecordReporter) { - super(errantRecordReporter); - checkNotNull(writeConfig, "Argument 'writeConfig' can not be null"); - - this.writeConfig = writeConfig; - this.emitFailureHandler = new KafkaCosmosEmitFailureHandler(); - } - - @Override - public void writeCore(CosmosAsyncContainer container, List<SinkOperation> sinkOperations) { - Sinks.Many<CosmosItemOperation> bulkRetryEmitter = Sinks.many().unicast().onBackpressureBuffer(); - CosmosBulkExecutionOptions bulkExecutionOptions = this.getBulkExecutionOperations(); - AtomicInteger totalPendingRecords = new AtomicInteger(sinkOperations.size()); - Runnable onTaskCompleteCheck = () -> { - if (totalPendingRecords.decrementAndGet() <= 0) { - bulkRetryEmitter.emitComplete(emitFailureHandler); - } - }; - - Flux.fromIterable(sinkOperations) - .flatMap(sinkOperation -> this.getBulkOperation(container, sinkOperation)) - .collectList() - .flatMapMany(itemOperations -> { - - Flux<CosmosBulkOperationResponse<Object>> cosmosBulkOperationResponseFlux = - container - .executeBulkOperations( - Flux.fromIterable(itemOperations) - .mergeWith(bulkRetryEmitter.asFlux()) - .publishOn(KafkaCosmosSchedulers.SINK_BOUNDED_ELASTIC), - bulkExecutionOptions); - return cosmosBulkOperationResponseFlux; - }) - .flatMap(itemResponse -> { - SinkOperation sinkOperation = itemResponse.getOperation().getContext(); - checkNotNull(sinkOperation, "sinkOperation should not be null"); - - if (itemResponse.getResponse() != null && itemResponse.getResponse().isSuccessStatusCode()) { - // success - this.completeSinkOperation(sinkOperation, onTaskCompleteCheck); - } else { - BulkOperationFailedException exception = handleErrorStatusCode( - itemResponse.getResponse(), - itemResponse.getException(), - sinkOperation); - - if (shouldIgnore(exception)) { - this.completeSinkOperation(sinkOperation, onTaskCompleteCheck); - } else { - if (shouldRetry(exception, sinkOperation.getRetryCount(), this.writeConfig.getMaxRetryCount())) { - sinkOperation.setException(exception); - return this.scheduleRetry(container, itemResponse.getOperation().getContext(), bulkRetryEmitter, exception); - } else { - // operation failed after exhausting all retries - this.completeSinkOperationWithFailure(sinkOperation, exception, onTaskCompleteCheck); - if (this.writeConfig.getToleranceOnErrorLevel() == ToleranceOnErrorLevel.ALL) { - LOGGER.warn( - "Could not upload record {} to CosmosDB after exhausting all retries, " - + "but ToleranceOnErrorLevel is all, will only log the error message. ", - sinkOperation.getSinkRecord().key(), - sinkOperation.getException()); - return Mono.empty(); - } else { - return Mono.error(exception); - } - } - } - } - - return Mono.empty(); - }) - .subscribeOn(KafkaCosmosSchedulers.SINK_BOUNDED_ELASTIC) - .blockLast(); - } - - private CosmosBulkExecutionOptions getBulkExecutionOperations() { - CosmosBulkExecutionOptions bulkExecutionOptions = new CosmosBulkExecutionOptions(); - bulkExecutionOptions.setInitialMicroBatchSize(this.writeConfig.getBulkInitialBatchSize()); - if (this.writeConfig.getBulkMaxConcurrentCosmosPartitions() > 0) { - ImplementationBridgeHelpers - .CosmosBulkExecutionOptionsHelper - .getCosmosBulkExecutionOptionsAccessor() - .setMaxConcurrentCosmosPartitions(bulkExecutionOptions, this.writeConfig.getBulkMaxConcurrentCosmosPartitions()); - } - - return bulkExecutionOptions; - } - - private Mono<CosmosItemOperation> getBulkOperation( - CosmosAsyncContainer container, - SinkOperation sinkOperation) { - - return this.getPartitionKeyDefinition(container) - .flatMap(partitionKeyDefinition -> { - CosmosItemOperation cosmosItemOperation; - - switch (this.writeConfig.getItemWriteStrategy()) { - case ITEM_OVERWRITE: - cosmosItemOperation = this.getUpsertItemOperation(sinkOperation, partitionKeyDefinition); - break; - case ITEM_OVERWRITE_IF_NOT_MODIFIED: - String etag = getEtag(sinkOperation.getSinkRecord().value()); - if (StringUtils.isEmpty(etag)) { - cosmosItemOperation = this.getCreateItemOperation(sinkOperation, partitionKeyDefinition); - } else { - cosmosItemOperation = this.getReplaceItemOperation(sinkOperation, partitionKeyDefinition, etag); - } - break; - case ITEM_APPEND: - cosmosItemOperation = this.getCreateItemOperation(sinkOperation, partitionKeyDefinition); - break; - case ITEM_DELETE: - cosmosItemOperation = this.getDeleteItemOperation(sinkOperation, partitionKeyDefinition, null); - break; - case ITEM_DELETE_IF_NOT_MODIFIED: - String itemDeleteEtag = getEtag(sinkOperation.getSinkRecord().value()); - cosmosItemOperation = this.getDeleteItemOperation(sinkOperation, partitionKeyDefinition, itemDeleteEtag); - break; - default: - return Mono.error(new IllegalArgumentException(this.writeConfig.getItemWriteStrategy() + " is not supported")); - } - - return Mono.just(cosmosItemOperation); - }); - } - - private CosmosItemOperation getUpsertItemOperation( - SinkOperation sinkOperation, - PartitionKeyDefinition partitionKeyDefinition) { - - return CosmosBulkOperations.getUpsertItemOperation( - sinkOperation.getSinkRecord().value(), - this.getPartitionKeyValue(sinkOperation.getSinkRecord().value(), partitionKeyDefinition), - sinkOperation); - } - - private CosmosItemOperation getCreateItemOperation( - SinkOperation sinkOperation, - PartitionKeyDefinition partitionKeyDefinition) { - return CosmosBulkOperations.getCreateItemOperation( - sinkOperation.getSinkRecord().value(), - this.getPartitionKeyValue(sinkOperation.getSinkRecord().value(), partitionKeyDefinition), - sinkOperation); - } - - private CosmosItemOperation getReplaceItemOperation( - SinkOperation sinkOperation, - PartitionKeyDefinition partitionKeyDefinition, - String etag) { - - CosmosBulkItemRequestOptions itemRequestOptions = new CosmosBulkItemRequestOptions(); - if (StringUtils.isNotEmpty(etag)) { - itemRequestOptions.setIfMatchETag(etag); - } - - return CosmosBulkOperations.getReplaceItemOperation( - getId(sinkOperation.getSinkRecord().value()), - sinkOperation.getSinkRecord().value(), - this.getPartitionKeyValue(sinkOperation.getSinkRecord().value(), partitionKeyDefinition), - new CosmosBulkItemRequestOptions().setIfMatchETag(etag), - sinkOperation); - } - - private CosmosItemOperation getDeleteItemOperation( - SinkOperation sinkOperation, - PartitionKeyDefinition partitionKeyDefinition, - String etag) { - - CosmosBulkItemRequestOptions itemRequestOptions = new CosmosBulkItemRequestOptions(); - if (StringUtils.isNotEmpty(etag)) { - itemRequestOptions.setIfMatchETag(etag); - } - - return CosmosBulkOperations.getDeleteItemOperation( - this.getId(sinkOperation.getSinkRecord().value()), - this.getPartitionKeyValue(sinkOperation.getSinkRecord().value(), partitionKeyDefinition), - itemRequestOptions, - sinkOperation); - } - - private Mono<Void> scheduleRetry( - CosmosAsyncContainer container, - SinkOperation sinkOperation, - Sinks.Many<CosmosItemOperation> bulkRetryEmitter, - BulkOperationFailedException exception) { - - sinkOperation.retry(); - Mono<Void> retryMono = - getBulkOperation(container, sinkOperation) - .flatMap(itemOperation -> { - bulkRetryEmitter.emitNext(itemOperation, emitFailureHandler); - return Mono.empty(); - }); - - if (KafkaCosmosExceptionsHelper.isTimeoutException(exception)) { - Duration delayDuration = Duration.ofMillis( - MIN_DELAY_ON_408_REQUEST_TIMEOUT_IN_MS - + RANDOM.nextInt(MAX_DELAY_ON_408_REQUEST_TIMEOUT_IN_MS - MIN_DELAY_ON_408_REQUEST_TIMEOUT_IN_MS)); - - return retryMono.delaySubscription(delayDuration); - } - - return retryMono; - } - - BulkOperationFailedException handleErrorStatusCode( - CosmosBulkItemResponse itemResponse, - Exception exception, - SinkOperation sinkOperationContext) { - - int effectiveStatusCode = - itemResponse != null - ? itemResponse.getStatusCode() - : (exception != null && exception instanceof CosmosException ? ((CosmosException) exception).getStatusCode() : HttpConstants.StatusCodes.REQUEST_TIMEOUT); - int effectiveSubStatusCode = - itemResponse != null - ? itemResponse.getSubStatusCode() - : (exception != null && exception instanceof CosmosException ? ((CosmosException) exception).getSubStatusCode() : 0); - - String errorMessage = - String.format( - "Request failed with effectiveStatusCode: {%s}, effectiveSubStatusCode: {%s}, kafkaOffset: {%s}, kafkaPartition: {%s}, topic: {%s}", - effectiveStatusCode, - effectiveSubStatusCode, - sinkOperationContext.getKafkaOffset(), - sinkOperationContext.getKafkaPartition(), - sinkOperationContext.getTopic()); - - - return new BulkOperationFailedException(effectiveStatusCode, effectiveSubStatusCode, errorMessage, exception); - } - - private boolean shouldIgnore(BulkOperationFailedException failedException) { - switch (this.writeConfig.getItemWriteStrategy()) { - case ITEM_APPEND: - return KafkaCosmosExceptionsHelper.isResourceExistsException(failedException); - case ITEM_DELETE: - return KafkaCosmosExceptionsHelper.isNotFoundException(failedException); - case ITEM_DELETE_IF_NOT_MODIFIED: - return KafkaCosmosExceptionsHelper.isNotFoundException(failedException) - || KafkaCosmosExceptionsHelper.isPreconditionFailedException(failedException); - case ITEM_OVERWRITE_IF_NOT_MODIFIED: - return KafkaCosmosExceptionsHelper.isResourceExistsException(failedException) - || KafkaCosmosExceptionsHelper.isNotFoundException(failedException) - || KafkaCosmosExceptionsHelper.isPreconditionFailedException(failedException); - default: - return false; - } - } - - private void completeSinkOperation(SinkOperation sinkOperationContext, Runnable onCompleteRunnable) { - sinkOperationContext.complete(); - onCompleteRunnable.run(); - } - - public void completeSinkOperationWithFailure( - SinkOperation sinkOperationContext, - Exception exception, - Runnable onCompleteRunnable) { - - sinkOperationContext.setException(exception); - sinkOperationContext.complete(); - onCompleteRunnable.run(); - - this.sendToDlqIfConfigured(sinkOperationContext); - } - - private static class BulkOperationFailedException extends CosmosException { - protected BulkOperationFailedException(int statusCode, int subStatusCode, String message, Throwable cause) { - super(statusCode, message, null, cause); - BridgeInternal.setSubStatusCode(this, subStatusCode); - } - } - - private static class KafkaCosmosEmitFailureHandler implements Sinks.EmitFailureHandler { - - @Override - public boolean onEmitFailure(SignalType signalType, Sinks.EmitResult emitResult) { - if (emitResult.equals(Sinks.EmitResult.FAIL_NON_SERIALIZED)) { - LOGGER.debug("emitFailureHandler - Signal: {}, Result: {}", signalType, emitResult.toString()); - return true; - } else { - LOGGER.error("emitFailureHandler - Signal: {}, Result: {}", signalType, emitResult.toString()); - return false; - } - } - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosPointWriter.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosPointWriter.java deleted file mode 100644 index a13c5d1a5266..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosPointWriter.java +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.guava25.base.Function; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosSchedulers; -import com.azure.cosmos.models.CosmosItemRequestOptions; -import org.apache.kafka.connect.sink.ErrantRecordReporter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Mono; - -import java.util.List; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -public class KafkaCosmosPointWriter extends KafkaCosmosWriterBase { - private static final Logger LOGGER = LoggerFactory.getLogger(KafkaCosmosPointWriter.class); - - private final CosmosSinkWriteConfig writeConfig; - - public KafkaCosmosPointWriter( - CosmosSinkWriteConfig writeConfig, - ErrantRecordReporter errantRecordReporter) { - super(errantRecordReporter); - checkNotNull(writeConfig, "Argument 'writeConfig' can not be null"); - this.writeConfig = writeConfig; - } - - @Override - public void writeCore(CosmosAsyncContainer container, List<SinkOperation> sinkOperations) { - for (SinkOperation sinkOperation : sinkOperations) { - switch (this.writeConfig.getItemWriteStrategy()) { - case ITEM_OVERWRITE: - this.upsertWithRetry(container, sinkOperation); - break; - case ITEM_OVERWRITE_IF_NOT_MODIFIED: - String etag = this.getEtag(sinkOperation.getSinkRecord().value()); - if (StringUtils.isNotEmpty(etag)) { - this.replaceIfNotModifiedWithRetry(container, sinkOperation, etag); - } else { - this.createWithRetry(container, sinkOperation); - } - break; - case ITEM_APPEND: - this.createWithRetry(container, sinkOperation); - break; - case ITEM_DELETE: - this.deleteWithRetry(container, sinkOperation, false); - break; - case ITEM_DELETE_IF_NOT_MODIFIED: - this.deleteWithRetry(container, sinkOperation, true); - break; - default: - throw new IllegalArgumentException(this.writeConfig.getItemWriteStrategy() + " is not supported"); - } - } - } - - private void upsertWithRetry(CosmosAsyncContainer container, SinkOperation sinkOperation) { - executeWithRetry( - (operation) -> container.upsertItem(operation.getSinkRecord().value()).then(), - (throwable) -> false, // no exceptions should be ignored - sinkOperation - ); - } - - private void createWithRetry(CosmosAsyncContainer container, SinkOperation sinkOperation) { - executeWithRetry( - (operation) -> container.createItem(operation.getSinkRecord().value()).then(), - (throwable) -> KafkaCosmosExceptionsHelper.isResourceExistsException(throwable), - sinkOperation - ); - } - - private void replaceIfNotModifiedWithRetry(CosmosAsyncContainer container, SinkOperation sinkOperation, String etag) { - executeWithRetry( - (operation) -> { - CosmosItemRequestOptions itemRequestOptions = new CosmosItemRequestOptions(); - itemRequestOptions.setIfMatchETag(etag); - - return this.getPartitionKeyDefinition(container) - .flatMap(partitionKeyDefinition -> { - return container.replaceItem( - operation.getSinkRecord().value(), - getId(operation.getSinkRecord().value()), - getPartitionKeyValue(operation.getSinkRecord().value(), partitionKeyDefinition), - itemRequestOptions).then(); - }); - }, - (throwable) -> { - return KafkaCosmosExceptionsHelper.isNotFoundException(throwable) - || KafkaCosmosExceptionsHelper.isPreconditionFailedException(throwable); - }, - sinkOperation - ); - } - - private void deleteWithRetry(CosmosAsyncContainer container, SinkOperation sinkOperation, boolean onlyIfModified) { - executeWithRetry( - (operation) -> { - CosmosItemRequestOptions itemRequestOptions = new CosmosItemRequestOptions(); - if (onlyIfModified) { - String etag = this.getEtag(operation.getSinkRecord().value()); - if (StringUtils.isNotEmpty(etag)) { - itemRequestOptions.setIfMatchETag(etag); - } - } - - return this.getPartitionKeyDefinition(container) - .flatMap(partitionKeyDefinition -> { - return container.deleteItem( - getId(operation.getSinkRecord().value()), - getPartitionKeyValue(operation.getSinkRecord().value(), partitionKeyDefinition), - itemRequestOptions - ); - }).then(); - }, - (throwable) -> { - return KafkaCosmosExceptionsHelper.isNotFoundException(throwable) - || KafkaCosmosExceptionsHelper.isPreconditionFailedException(throwable); - }, - sinkOperation - ); - } - - private void executeWithRetry( - Function<SinkOperation, Mono<Void>> execution, - Function<Throwable, Boolean> shouldIgnoreFunc, - SinkOperation sinkOperation) { - - Mono.just(this) - .flatMap(data -> { - if (sinkOperation.getRetryCount() > 0) { - LOGGER.debug("Retry for sinkRecord {}", sinkOperation.getSinkRecord().key()); - } - return execution.apply(sinkOperation); - }) - .doOnSuccess(response -> sinkOperation.complete()) - .onErrorResume(throwable -> { - if (shouldIgnoreFunc.apply(throwable)) { - sinkOperation.complete(); - return Mono.empty(); - } - - if (shouldRetry(throwable, sinkOperation.getRetryCount(), this.writeConfig.getMaxRetryCount())) { - sinkOperation.setException(throwable); - sinkOperation.retry(); - - return Mono.empty(); - } else { - // request failed after exhausted all retries - this.sendToDlqIfConfigured(sinkOperation); - - sinkOperation.setException(throwable); - sinkOperation.complete(); - - if (this.writeConfig.getToleranceOnErrorLevel() == ToleranceOnErrorLevel.ALL) { - LOGGER.warn( - "Could not upload record {} to CosmosDB after exhausting all retries, but ToleranceOnErrorLevel is all, will only log the error message. ", - sinkOperation.getSinkRecord().key(), - sinkOperation.getException()); - return Mono.empty(); - } else { - return Mono.error(sinkOperation.getException()); - } - } - }) - .repeat(() -> !sinkOperation.isCompleted()) - .then() - .subscribeOn(KafkaCosmosSchedulers.SINK_BOUNDED_ELASTIC) - .block(); - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosWriterBase.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosWriterBase.java deleted file mode 100644 index 108f3fefa7ca..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/KafkaCosmosWriterBase.java +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.CosmosBridgeInternal; -import com.azure.cosmos.implementation.DocumentCollection; -import com.azure.cosmos.implementation.ImplementationBridgeHelpers; -import com.azure.cosmos.implementation.Strings; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.routing.PartitionKeyInternal; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; -import com.azure.cosmos.models.PartitionKey; -import com.azure.cosmos.models.PartitionKeyDefinition; -import org.apache.kafka.connect.sink.ErrantRecordReporter; -import org.apache.kafka.connect.sink.SinkRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Mono; - -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; - -public abstract class KafkaCosmosWriterBase implements IWriter { - private static final Logger LOGGER = LoggerFactory.getLogger(KafkaCosmosWriterBase.class); - private static final String ID = "id"; - private static final String ETAG = "_etag"; - private final ErrantRecordReporter errantRecordReporter; - - public KafkaCosmosWriterBase(ErrantRecordReporter errantRecordReporter) { - this.errantRecordReporter = errantRecordReporter; - } - - abstract void writeCore(CosmosAsyncContainer container, List<SinkOperation> sinkOperations); - - @Override - public void write(CosmosAsyncContainer container, List<SinkRecord> sinkRecords) { - if (sinkRecords == null || sinkRecords.isEmpty()) { - LOGGER.debug("No records to be written to container {}", container.getId()); - return; - } - LOGGER.debug("Write {} records to container {}", sinkRecords.size(), container.getId()); - - // For each sinkRecord, it has a 1:1 mapping SinkOperation which contains sinkRecord and related context: retryCount, succeeded or failure. - List<SinkOperation> sinkOperations = - sinkRecords - .stream() - .map(sinkRecord -> new SinkOperation(sinkRecord)) - .collect(Collectors.toList()); - - try { - writeCore(container, sinkOperations); - } catch (Exception e) { - LOGGER.error("Write failed. ", e); - throw new CosmosDBWriteException(e.getMessage()); - } - } - - @SuppressWarnings("unchecked") - protected String getId(Object recordValue) { - checkArgument(recordValue instanceof Map, "Argument 'recordValue' is not valid map format."); - return ((Map<String, Object>) recordValue).get(ID).toString(); - } - - @SuppressWarnings("unchecked") - protected String getEtag(Object recordValue) { - checkArgument(recordValue instanceof Map, "Argument 'recordValue' is not valid map format."); - return ((Map<String, Object>) recordValue).getOrDefault(ETAG, Strings.Emtpy).toString(); - } - - @SuppressWarnings("unchecked") - protected PartitionKey getPartitionKeyValue(Object recordValue, PartitionKeyDefinition partitionKeyDefinition) { - checkArgument(recordValue instanceof Map, "Argument 'recordValue' is not valid map format."); - - //TODO[Public Preview]: add support for sub-partition - String partitionKeyPath = StringUtils.join(partitionKeyDefinition.getPaths(), ""); - Map<String, Object> recordMap = (Map<String, Object>) recordValue; - Object partitionKeyValue = recordMap.get(partitionKeyPath.substring(1)); - PartitionKeyInternal partitionKeyInternal = PartitionKeyInternal.fromObjectArray(Collections.singletonList(partitionKeyValue), false); - - return ImplementationBridgeHelpers - .PartitionKeyHelper - .getPartitionKeyAccessor() - .toPartitionKey(partitionKeyInternal); - } - - protected boolean shouldRetry(Throwable exception, int attemptedCount, int maxRetryCount) { - if (attemptedCount >= maxRetryCount) { - return false; - } - - return KafkaCosmosExceptionsHelper.isTransientFailure(exception); - } - - protected Mono<PartitionKeyDefinition> getPartitionKeyDefinition(CosmosAsyncContainer container) { - return Mono.just(CosmosBridgeInternal.getAsyncDocumentClient(container.getDatabase()).getCollectionCache()) - .flatMap(collectionCache -> { - return collectionCache - .resolveByNameAsync( - null, - ImplementationBridgeHelpers - .CosmosAsyncContainerHelper - .getCosmosAsyncContainerAccessor() - .getLinkWithoutTrailingSlash(container), - null, - new DocumentCollection()) - .map(documentCollection -> documentCollection.getPartitionKey()); - }); - } - - protected void sendToDlqIfConfigured(SinkOperation sinkOperationContext) { - if (this.errantRecordReporter != null) { - errantRecordReporter.report(sinkOperationContext.getSinkRecord(), sinkOperationContext.getException()); - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkOperation.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkOperation.java deleted file mode 100644 index 599a71ec9984..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkOperation.java +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import org.apache.kafka.connect.sink.SinkRecord; - -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; - -public class SinkOperation { - private final SinkRecord sinkRecord; - private final AtomicInteger retryCount; - private final AtomicReference<Throwable> exception; - private final AtomicBoolean completed; - - public SinkOperation(SinkRecord sinkRecord) { - this.sinkRecord = sinkRecord; - this.retryCount = new AtomicInteger(0); - this.exception = new AtomicReference<>(null); - this.completed = new AtomicBoolean(false); - } - - public SinkRecord getSinkRecord() { - return this.sinkRecord; - } - - public long getKafkaOffset() { - return this.sinkRecord.kafkaOffset(); - } - - public Integer getKafkaPartition() { - return this.sinkRecord.kafkaPartition(); - } - - public String getTopic() { - return this.sinkRecord.topic(); - } - - public int getRetryCount() { - return this.retryCount.get(); - } - - public void retry() { - this.retryCount.incrementAndGet(); - } - - public Throwable getException() { - return this.exception.get(); - } - - public void setException(Throwable exception) { - this.exception.set(exception); - } - - public boolean isCompleted() { - return this.completed.get(); - } - - public void complete() { - this.completed.set(true); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkRecordTransformer.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkRecordTransformer.java deleted file mode 100644 index 007d09bb793d..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/SinkRecordTransformer.java +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.AbstractIdStrategyConfig; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.FullKeyStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.IdStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.KafkaMetadataStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.ProvidedInKeyStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.ProvidedInValueStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.TemplateStrategy; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.sink.SinkRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class SinkRecordTransformer { - private static final Logger LOGGER = LoggerFactory.getLogger(SinkRecordTransformer.class); - - private final IdStrategy idStrategy; - - public SinkRecordTransformer(CosmosSinkTaskConfig sinkTaskConfig) { - this.idStrategy = this.createIdStrategy(sinkTaskConfig); - } - - @SuppressWarnings("unchecked") - public List<SinkRecord> transform(String containerName, List<SinkRecord> sinkRecords) { - List<SinkRecord> toBeWrittenRecordList = new ArrayList<>(); - for (SinkRecord record : sinkRecords) { - if (record.key() != null) { - MDC.put(String.format("CosmosDbSink-%s", containerName), record.key().toString()); - } - - LOGGER.trace( - "Key Schema [{}], Key [{}], Value type [{}], Value schema [{}]", - record.keySchema(), - record.key(), - record.value() == null ? null : record.value().getClass().getName(), - record.value() == null ? null : record.valueSchema()); - - Object recordValue; - if (record.value() instanceof Struct) { - recordValue = StructToJsonMap.toJsonMap((Struct) record.value()); - } else if (record.value() instanceof Map) { - recordValue = StructToJsonMap.handleMap((Map<String, Object>) record.value()); - } else { - recordValue = record.value(); - } - - maybeInsertId(recordValue, record); - - // Create an updated record with from the current record and the updated record value - final SinkRecord updatedRecord = new SinkRecord(record.topic(), - record.kafkaPartition(), - record.keySchema(), - record.key(), - record.valueSchema(), - recordValue, - record.kafkaOffset(), - record.timestamp(), - record.timestampType(), - record.headers()); - - toBeWrittenRecordList.add(updatedRecord); - } - - return toBeWrittenRecordList; - } - - @SuppressWarnings("unchecked") - private void maybeInsertId(Object recordValue, SinkRecord sinkRecord) { - if (!(recordValue instanceof Map)) { - return; - } - Map<String, Object> recordMap = (Map<String, Object>) recordValue; - recordMap.put(AbstractIdStrategyConfig.ID, this.idStrategy.generateId(sinkRecord)); - } - - private IdStrategy createIdStrategy(CosmosSinkTaskConfig sinkTaskConfig) { - IdStrategy idStrategyClass; - switch (sinkTaskConfig.getIdStrategy()) { - case FULL_KEY_STRATEGY: - idStrategyClass = new FullKeyStrategy(); - break; - case TEMPLATE_STRATEGY: - idStrategyClass = new TemplateStrategy(); - break; - case KAFKA_METADATA_STRATEGY: - idStrategyClass = new KafkaMetadataStrategy(); - break; - case PROVIDED_IN_VALUE_STRATEGY: - idStrategyClass = new ProvidedInValueStrategy(); - break; - case PROVIDED_IN_KEY_STRATEGY: - idStrategyClass = new ProvidedInKeyStrategy(); - break; - default: - throw new IllegalArgumentException(sinkTaskConfig.getIdStrategy() + " is not supported"); - } - - idStrategyClass.configure(sinkTaskConfig.originalsWithPrefix(AbstractIdStrategyConfig.PREFIX)); - return idStrategyClass; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/StructToJsonMap.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/StructToJsonMap.java deleted file mode 100644 index 388baa3778f8..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/StructToJsonMap.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import org.apache.kafka.connect.data.Date; -import org.apache.kafka.connect.data.Field; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.data.Time; -import org.apache.kafka.connect.data.Timestamp; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -// TODO[Public Preview]: Double check logic here, copied over from V1 -public class StructToJsonMap { - - public static Map<String, Object> toJsonMap(Struct struct) { - if (struct == null) { - return null; - } - Map<String, Object> jsonMap = new HashMap<String, Object>(0); - List<Field> fields = struct.schema().fields(); - for (Field field : fields) { - String fieldName = field.name(); - Schema.Type fieldType = field.schema().type(); - String schemaName = field.schema().name(); - switch (fieldType) { - case STRING: - jsonMap.put(fieldName, struct.getString(fieldName)); - break; - case INT32: - if (Date.LOGICAL_NAME.equals(schemaName) || Time.LOGICAL_NAME.equals(schemaName)) { - jsonMap.put(fieldName, (java.util.Date) struct.get(fieldName)); - } else { - jsonMap.put(fieldName, struct.getInt32(fieldName)); - } - break; - case INT16: - jsonMap.put(fieldName, struct.getInt16(fieldName)); - break; - case INT64: - if (Timestamp.LOGICAL_NAME.equals(schemaName)) { - jsonMap.put(fieldName, (java.util.Date) struct.get(fieldName)); - } else { - jsonMap.put(fieldName, struct.getInt64(fieldName)); - } - break; - case FLOAT32: - jsonMap.put(fieldName, struct.getFloat32(fieldName)); - break; - case FLOAT64: - jsonMap.put(fieldName, struct.getFloat64(fieldName)); - break; - case BOOLEAN: - jsonMap.put(fieldName, struct.getBoolean(fieldName)); - break; - case ARRAY: - List<Object> fieldArray = struct.getArray(fieldName); - if (fieldArray != null && !fieldArray.isEmpty() && fieldArray.get(0) instanceof Struct) { - // If Array contains list of Structs - List<Object> jsonArray = new ArrayList<>(); - fieldArray.forEach(item -> { - jsonArray.add(toJsonMap((Struct) item)); - }); - jsonMap.put(fieldName, jsonArray); - } else { - jsonMap.put(fieldName, fieldArray); - } - break; - case STRUCT: - jsonMap.put(fieldName, toJsonMap(struct.getStruct(fieldName))); - break; - case MAP: - jsonMap.put(fieldName, handleMap(struct.getMap(fieldName))); - break; - default: - jsonMap.put(fieldName, struct.get(fieldName)); - break; - } - } - return jsonMap; - } - - @SuppressWarnings("unchecked") - public static Map<String, Object> handleMap(Map<String, Object> map) { - if (map == null) { - return null; - } - Map<String, Object> cacheMap = new HashMap<>(); - map.forEach((key, value) -> { - if (value instanceof Map) { - cacheMap.put(key, handleMap((Map<String, Object>) value)); - } else if (value instanceof Struct) { - cacheMap.put(key, toJsonMap((Struct) value)); - } else if (value instanceof List) { - List<Object> list = (List<Object>) value; - List<Object> jsonArray = new ArrayList<>(); - list.forEach(item -> { - if (item instanceof Struct) { - jsonArray.add(toJsonMap((Struct) item)); - } else if (item instanceof Map) { - jsonArray.add(handleMap((Map<String, Object>) item)); - } else { - jsonArray.add(item); - } - }); - cacheMap.put(key, jsonArray); - } else { - cacheMap.put(key, value); - } - }); - return cacheMap; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ToleranceOnErrorLevel.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ToleranceOnErrorLevel.java deleted file mode 100644 index d169fd2484b0..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/ToleranceOnErrorLevel.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -public enum ToleranceOnErrorLevel { - NONE("None"), - ALL("All"); - - private final String name; - - ToleranceOnErrorLevel(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public static ToleranceOnErrorLevel fromName(String name) { - for (ToleranceOnErrorLevel mode : ToleranceOnErrorLevel.values()) { - if (mode.getName().equalsIgnoreCase(name)) { - return mode; - } - } - return null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategy.java deleted file mode 100644 index 41bc401b8136..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategy.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import java.util.Map; -import java.util.regex.Pattern; - -public abstract class AbstractIdStrategy implements IdStrategy { - private static final String SANITIZED_CHAR = "_"; - private static final Pattern SANITIZE_ID_PATTERN = Pattern.compile("[/\\\\?#]"); - - protected Map<String, ?> configs; - - @Override - public void configure(Map<String, ?> configs) { - this.configs = configs; - } - - /** - * Replaces all characters that cannot be part of the ID with {@value SANITIZED_CHAR}. - * <p>The following characters are restricted and cannot be used in the Id property: '/', '\\', '?', '#' - */ - public static String sanitizeId(String unsanitized) { - return SANITIZE_ID_PATTERN.matcher(unsanitized).replaceAll(SANITIZED_CHAR); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategyConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategyConfig.java deleted file mode 100644 index 1713ecc79636..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/AbstractIdStrategyConfig.java +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.common.config.AbstractConfig; -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Map; - -public class AbstractIdStrategyConfig extends AbstractConfig { - public static final String ID = "id"; - public static final String ID_STRATEGY = ID + ".strategy"; - public static final String PREFIX = ID_STRATEGY + "."; - - public AbstractIdStrategyConfig(ConfigDef definition, Map<?, ?> originals) { - super(definition, originals); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/FullKeyStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/FullKeyStrategy.java deleted file mode 100644 index 62fc6f72a340..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/FullKeyStrategy.java +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import java.util.HashMap; -import java.util.Map; - -public class FullKeyStrategy extends TemplateStrategy { - @Override - public void configure(Map<String, ?> configs) { - Map<String, Object> conf = new HashMap<>(configs); - conf.put(TemplateStrategyConfig.TEMPLATE_CONFIG, "${key}"); - super.configure(conf); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/IdStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/IdStrategy.java deleted file mode 100644 index b4ce03d4f73e..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/IdStrategy.java +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.common.Configurable; -import org.apache.kafka.connect.sink.SinkRecord; - -public interface IdStrategy extends Configurable { - String generateId(SinkRecord record); -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategy.java deleted file mode 100644 index 99d705f062f4..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategy.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import java.util.HashMap; -import java.util.Map; - -public class KafkaMetadataStrategy extends TemplateStrategy { - private KafkaMetadataStrategyConfig config; - - @Override - public void configure(Map<String, ?> configs) { - config = new KafkaMetadataStrategyConfig(configs); - Map<String, Object> conf = new HashMap<>(configs); - conf.put(TemplateStrategyConfig.TEMPLATE_CONFIG, - "${topic}" + config.delimiter() - + "${partition}" + config.delimiter() + "${offset}"); - - super.configure(conf); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategyConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategyConfig.java deleted file mode 100644 index b29d59e4409c..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/KafkaMetadataStrategyConfig.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Map; - -public class KafkaMetadataStrategyConfig extends AbstractIdStrategyConfig { - public static final String DELIMITER_CONFIG = "delimiter"; - public static final String DELIMITER_CONFIG_DEFAULT = "-"; - public static final String DELIMITER_CONFIG_DOC = "The delimiter between metadata components"; - public static final String DELIMITER_CONFIG_DISPLAY = "Kafka Metadata"; - - private String delimiter; - - public KafkaMetadataStrategyConfig(Map<String, ?> props) { - this(getConfig(), props); - } - - public KafkaMetadataStrategyConfig(ConfigDef definition, Map<?, ?> originals) { - super(definition, originals); - - this.delimiter = getString(DELIMITER_CONFIG); - } - - public static ConfigDef getConfig() { - ConfigDef result = new ConfigDef(); - - final String groupName = "Kafka Metadata Parameters"; - int groupOrder = 0; - - result.define( - DELIMITER_CONFIG, - ConfigDef.Type.STRING, - DELIMITER_CONFIG_DEFAULT, - ConfigDef.Importance.MEDIUM, - DELIMITER_CONFIG_DOC, - groupName, - groupOrder++, - ConfigDef.Width.MEDIUM, - DELIMITER_CONFIG_DISPLAY - ); - - return result; - } - - public String delimiter() { - return delimiter; - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInConfig.java deleted file mode 100644 index de3892baa1d2..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInConfig.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Map; - -public class ProvidedInConfig extends AbstractIdStrategyConfig { - public static final String JSON_PATH_CONFIG = "jsonPath"; - public static final String JSON_PATH_CONFIG_DEFAULT = "$.id"; - public static final String JSON_PATH_CONFIG_DOC = "A JsonPath expression to select the desired component to use as ``id``"; - public static final String JSON_PATH_CONFIG_DISPLAY = "JSON Path"; - private final String jsonPath; - - public ProvidedInConfig(Map<String, ?> props) { - this(getConfig(), props); - } - - public ProvidedInConfig(ConfigDef definition, Map<String, ?> originals) { - super(definition, originals); - - this.jsonPath = getString(JSON_PATH_CONFIG); - } - - - public static ConfigDef getConfig() { - ConfigDef result = new ConfigDef(); - - final String groupName = "JsonPath Parameters"; - int groupOrder = 0; - - result.define( - JSON_PATH_CONFIG, - ConfigDef.Type.STRING, - JSON_PATH_CONFIG_DEFAULT, - ConfigDef.Importance.MEDIUM, - JSON_PATH_CONFIG_DOC, - groupName, - groupOrder++, - ConfigDef.Width.MEDIUM, - JSON_PATH_CONFIG_DISPLAY - ); - - return result; - } - - public String jsonPath() { - return jsonPath; - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInKeyStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInKeyStrategy.java deleted file mode 100644 index 930e2435351e..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInKeyStrategy.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -public class ProvidedInKeyStrategy extends ProvidedInStrategy { - public ProvidedInKeyStrategy() { - super(ProvidedIn.KEY); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInStrategy.java deleted file mode 100644 index 79b4ed19f655..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInStrategy.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.connect.data.Values; -import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.sink.SinkRecord; -import com.jayway.jsonpath.JsonPath; - -import java.util.Map; - -class ProvidedInStrategy extends AbstractIdStrategy { - protected enum ProvidedIn { - KEY, - VALUE - } - - private final ProvidedIn where; - - private ProvidedInConfig config; - - ProvidedInStrategy(ProvidedIn where) { - this.where = where; - } - - @Override - public String generateId(SinkRecord record) { - String value = where == ProvidedIn.KEY - ? Values.convertToString(record.keySchema(), record.key()) - : Values.convertToString(record.valueSchema(), record.value()); - try { - Object object = JsonPath.parse(value).read(config.jsonPath()); - return sanitizeId(Values.convertToString(null, object)); - } catch (Exception e) { - throw new ConnectException("Could not evaluate JsonPath " + config.jsonPath(), e); - } - } - - @Override - public void configure(Map<String, ?> configs) { - config = new ProvidedInConfig(configs); - super.configure(configs); - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInValueStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInValueStrategy.java deleted file mode 100644 index ca5b794fc8bd..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/ProvidedInValueStrategy.java +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -public class ProvidedInValueStrategy extends ProvidedInStrategy { - public ProvidedInValueStrategy() { - super(ProvidedIn.VALUE); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategy.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategy.java deleted file mode 100644 index 1f40827050dc..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategy.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import com.azure.cosmos.implementation.guava25.collect.ImmutableMap; -import org.apache.kafka.connect.data.Values; -import org.apache.kafka.connect.sink.SinkRecord; - -import java.util.Map; -import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -public class TemplateStrategy extends AbstractIdStrategy { - private static final String KEY = "key"; - private static final String TOPIC = "topic"; - private static final String PARTITION = "partition"; - private static final String OFFSET = "offset"; - - private static final String PATTERN_TEMPLATE = "\\$\\{(%s)\\}"; - private static final Pattern PATTERN; - - private TemplateStrategyConfig config; - - private static final Map<String, Function<SinkRecord, String>> METHODS_BY_VARIABLE; - - static { - ImmutableMap.Builder<String, Function<SinkRecord, String>> builder = ImmutableMap.builder(); - builder.put(KEY, (r) -> Values.convertToString(r.keySchema(), r.key())); - builder.put(TOPIC, SinkRecord::topic); - builder.put(PARTITION, (r) -> r.kafkaPartition().toString()); - builder.put(OFFSET, (r) -> Long.toString(r.kafkaOffset())); - METHODS_BY_VARIABLE = builder.build(); - - String pattern = String.format(PATTERN_TEMPLATE, - METHODS_BY_VARIABLE.keySet().stream().collect(Collectors.joining("|"))); - PATTERN = Pattern.compile(pattern); - } - - @Override - public String generateId(SinkRecord record) { - String template = config.template(); - return sanitizeId(resolveAll(template, record)); - } - - @Override - public void configure(Map<String, ?> configs) { - config = new TemplateStrategyConfig(configs); - - super.configure(configs); - } - - private String resolveAll(String template, SinkRecord record) { - int lastIndex = 0; - StringBuilder output = new StringBuilder(); - Matcher matcher = PATTERN.matcher(template); - while (matcher.find()) { - output.append(template, lastIndex, matcher.start()) - .append(METHODS_BY_VARIABLE.get(matcher.group(1)).apply(record)); - - lastIndex = matcher.end(); - } - if (lastIndex < template.length()) { - output.append(template, lastIndex, template.length()); - } - return output.toString(); - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategyConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategyConfig.java deleted file mode 100644 index c03c605438d9..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/sink/idstrategy/TemplateStrategyConfig.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idstrategy; - -import org.apache.kafka.common.config.ConfigDef; - -import java.util.Map; - -public class TemplateStrategyConfig extends AbstractIdStrategyConfig { - public static final String TEMPLATE_CONFIG = "template"; - public static final String TEMPLATE_CONFIG_DEFAULT = ""; - public static final String TEMPLATE_CONFIG_DOC = - "The template string to use for determining the ``id``. The template can contain the " - + "following variables that are bound to their values on the Kafka record:" - + "${topic}, ${partition}, ${offset}, ${key}. For example, the template " - + "``${topic}-${key}`` would use the topic name and the entire key in the ``id``, " - + "separated by '-'"; - public static final String TEMPLATE_CONFIG_DISPLAY = "Template"; - private final String template; - - public TemplateStrategyConfig(Map<String, ?> props) { - this(getConfig(), props); - } - - public TemplateStrategyConfig(ConfigDef definition, Map<String, ?> originals) { - super(definition, originals); - - this.template = getString(TEMPLATE_CONFIG); - } - - public static ConfigDef getConfig() { - ConfigDef result = new ConfigDef(); - - final String groupName = "Template Parameters"; - int groupOrder = 0; - - result.define( - TEMPLATE_CONFIG, - ConfigDef.Type.STRING, - TEMPLATE_CONFIG_DEFAULT, - ConfigDef.Importance.MEDIUM, - TEMPLATE_CONFIG_DOC, - groupName, - groupOrder++, - ConfigDef.Width.MEDIUM, - TEMPLATE_CONFIG_DISPLAY - ); - - return result; - } - - public String template() { - return template; - } -} - diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ContainersMetadataTopicOffset.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ContainersMetadataTopicOffset.java deleted file mode 100644 index 10d7885bbde2..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ContainersMetadataTopicOffset.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.Utils; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -/** - * Containers metadata topic offset. - */ -public class ContainersMetadataTopicOffset { - public static final String CONTAINERS_RESOURCE_IDS_NAME_KEY = "cosmos.source.metadata.containerRids"; - public static final ObjectMapper OBJECT_MAPPER = Utils.getSimpleObjectMapper(); - - private final List<String> containerRids; - public ContainersMetadataTopicOffset(List<String> containerRids) { - checkNotNull(containerRids, "Argument 'containerRids' can not be null"); - this.containerRids = containerRids; - } - - public List<String> getContainerRids() { - return containerRids; - } - - public static Map<String, Object> toMap(ContainersMetadataTopicOffset offset) { - Map<String, Object> map = new HashMap<>(); - try { - map.put( - CONTAINERS_RESOURCE_IDS_NAME_KEY, - OBJECT_MAPPER.writeValueAsString(offset.getContainerRids())); - return map; - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - @SuppressWarnings("unchecked") - public static ContainersMetadataTopicOffset fromMap(Map<String, Object> offsetMap) { - if (offsetMap == null) { - return null; - } - - try { - List<String> containerRids = - OBJECT_MAPPER - .readValue(offsetMap.get(CONTAINERS_RESOURCE_IDS_NAME_KEY).toString(), new TypeReference<List<String>>() {}); - return new ContainersMetadataTopicOffset(containerRids); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ContainersMetadataTopicPartition.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ContainersMetadataTopicPartition.java deleted file mode 100644 index b2ae0e6de93e..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ContainersMetadataTopicPartition.java +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; - -public class ContainersMetadataTopicPartition { - public static final String DATABASE_NAME_KEY = "cosmos.source.metadata.database.name"; - - private final String databaseName; - - public ContainersMetadataTopicPartition(String databaseName) { - checkArgument(StringUtils.isNotEmpty(databaseName), "Argument 'databaseName' can not be null"); - - this.databaseName = databaseName; - } - - public String getDatabaseName() { - return databaseName; - } - - public static Map<String, Object> toMap(ContainersMetadataTopicPartition topicPartition) { - Map<String, Object> map = new HashMap<>(); - map.put(DATABASE_NAME_KEY, topicPartition.getDatabaseName()); - return map; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedModes.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedModes.java deleted file mode 100644 index 9917a19145c8..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedModes.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -public enum CosmosChangeFeedModes { - LATEST_VERSION("LatestVersion"), - ALL_VERSION_AND_DELETES("AllVersionsAndDeletes"); - - private final String name; - CosmosChangeFeedModes(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public static CosmosChangeFeedModes fromName(String name) { - for (CosmosChangeFeedModes mode : CosmosChangeFeedModes.values()) { - if (mode.getName().equalsIgnoreCase(name)) { - return mode; - } - } - return null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedStartFromModes.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedStartFromModes.java deleted file mode 100644 index 15be8610bcee..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosChangeFeedStartFromModes.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -public enum CosmosChangeFeedStartFromModes { - BEGINNING("Beginning"), - NOW("Now"), - POINT_IN_TIME("PointInTime"); - - private final String name; - CosmosChangeFeedStartFromModes(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public static CosmosChangeFeedStartFromModes fromName(String name) { - for (CosmosChangeFeedStartFromModes startFromModes : CosmosChangeFeedStartFromModes.values()) { - if (startFromModes.getName().equalsIgnoreCase(name)) { - return startFromModes; - } - } - return null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosMetadataConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosMetadataConfig.java deleted file mode 100644 index 91f54088cc31..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosMetadataConfig.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; - -public class CosmosMetadataConfig { - private final int metadataPollDelayInMs; - private final String metadataTopicName; - - public CosmosMetadataConfig(int metadataPollDelayInMs, String metadataTopicName) { - checkArgument(StringUtils.isNotEmpty(metadataTopicName), "Argument 'metadataTopicName' can not be null"); - checkArgument(metadataPollDelayInMs > 0, "Argument 'metadataPollDelayInMs' should be larger than 0"); - - this.metadataPollDelayInMs = metadataPollDelayInMs; - this.metadataTopicName = metadataTopicName; - } - - public int getMetadataPollDelayInMs() { - return metadataPollDelayInMs; - } - - public String getMetadataTopicName() { - return metadataTopicName; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceChangeFeedConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceChangeFeedConfig.java deleted file mode 100644 index cff4305fcaa2..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceChangeFeedConfig.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import java.time.Instant; - -public class CosmosSourceChangeFeedConfig { - private final CosmosChangeFeedModes changeFeedModes; - private final CosmosChangeFeedStartFromModes changeFeedStartFromModes; - private final Instant startFrom; - private final int maxItemCountHint; - - public CosmosSourceChangeFeedConfig( - CosmosChangeFeedModes changeFeedModes, - CosmosChangeFeedStartFromModes changeFeedStartFromModes, - Instant startFrom, - int maxItemCountHint) { - this.changeFeedModes = changeFeedModes; - this.changeFeedStartFromModes = changeFeedStartFromModes; - this.startFrom = startFrom; - this.maxItemCountHint = maxItemCountHint; - } - - public CosmosChangeFeedModes getChangeFeedModes() { - return changeFeedModes; - } - - public CosmosChangeFeedStartFromModes getChangeFeedStartFromModes() { - return changeFeedStartFromModes; - } - - public Instant getStartFrom() { - return startFrom; - } - - public int getMaxItemCountHint() { - return maxItemCountHint; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceConfig.java deleted file mode 100644 index 1365d1e53575..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceConfig.java +++ /dev/null @@ -1,444 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.Strings; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConfig; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigException; - -import java.time.Instant; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Common Configuration for Cosmos DB Kafka source connector. - */ -public class CosmosSourceConfig extends KafkaCosmosConfig { - - // configuration only targets to source connector - private static final String SOURCE_CONF_PREFIX = "kafka.connect.cosmos.source."; - - // database name - private static final String DATABASE_NAME_CONF = SOURCE_CONF_PREFIX + "database.name"; - private static final String DATABASE_NAME_CONF_DOC = "Cosmos DB database name."; - private static final String DATABASE_NAME_CONF_DISPLAY = "Cosmos DB database name."; - - // Source containers config - private static final String CONTAINERS_INCLUDE_ALL_CONF = SOURCE_CONF_PREFIX + "containers.includeAll"; - private static final String CONTAINERS_INCLUDE_ALL_CONF_DOC = "Flag to indicate whether reading from all containers."; - private static final String CONTAINERS_INCLUDE_ALL_CONF_DISPLAY = "Include all containers."; - private static final boolean DEFAULT_CONTAINERS_INCLUDE_ALL = false; - - private static final String CONTAINERS_INCLUDED_LIST_CONF = SOURCE_CONF_PREFIX + "containers.includedList"; - private static final String CONTAINERS_INCLUDED_LIST_CONF_DOC = - "Containers included. This config will be ignored if kafka.connect.cosmos.source.includeAllContainers is true."; - private static final String CONTAINERS_INCLUDED_LIST_CONF_DISPLAY = "Containers included."; - - private static final String CONTAINERS_TOPIC_MAP_CONF = SOURCE_CONF_PREFIX + "containers.topicMap"; - private static final String CONTAINERS_TOPIC_MAP_CONF_DOC = - "A comma delimited list of Kafka topics mapped to Cosmos containers. For example: topic1#con1,topic2#con2. " - + "By default, container name is used as the name of the kafka topic to publish data to, " - + "can use this property to override the default config "; - private static final String CONTAINERS_TOPIC_MAP_CONF_DISPLAY = "Cosmos container topic map."; - - // changeFeed config - private static final String CHANGE_FEED_START_FROM_CONF = SOURCE_CONF_PREFIX + "changeFeed.startFrom"; - private static final String CHANGE_FEED_START_FROM_CONF_DOC = "ChangeFeed Start from settings (Now, Beginning " - + "or a certain point in time (UTC) for example 2020-02-10T14:15:03) - the default value is 'Beginning'. "; - private static final String CHANGE_FEED_START_FROM_CONF_DISPLAY = "Change feed start from."; - private static final String DEFAULT_CHANGE_FEED_START_FROM = CosmosChangeFeedStartFromModes.BEGINNING.getName(); - - private static final String CHANGE_FEED_MODE_CONF = SOURCE_CONF_PREFIX + "changeFeed.mode"; - private static final String CHANGE_FEED_MODE_CONF_DOC = "ChangeFeed mode (LatestVersion or AllVersionsAndDeletes)"; - private static final String CHANGE_FEED_MODE_CONF_DISPLAY = "ChangeFeed mode (LatestVersion or AllVersionsAndDeletes)"; - private static final String DEFAULT_CHANGE_FEED_MODE = CosmosChangeFeedModes.LATEST_VERSION.getName(); - - private static final String CHANGE_FEED_MAX_ITEM_COUNT_CONF = SOURCE_CONF_PREFIX + "changeFeed.maxItemCountHint"; - private static final String CHANGE_FEED_MAX_ITEM_COUNT_CONF_DOC = - "The maximum number of documents returned in a single change feed request." - + " But the number of items received might be higher than the specified value if multiple items are changed by the same transaction." - + " The default is 1000."; - private static final String CHANGE_FEED_MAX_ITEM_COUNT_CONF_DISPLAY = "The maximum number hint of documents returned in a single request. "; - private static final int DEFAULT_CHANGE_FEED_MAX_ITEM_COUNT = 1000; - - // Metadata config - private static final String METADATA_POLL_DELAY_MS_CONF = SOURCE_CONF_PREFIX + "metadata.poll.delay.ms"; - private static final String METADATA_POLL_DELAY_MS_CONF_DOC = - "Indicates how often to check the metadata changes (including container split/merge, adding/removing/recreated containers). " - + "When changes are detected, it will reconfigure the tasks. Default is 5 minutes."; - private static final String METADATA_POLL_DELAY_MS_CONF_DISPLAY = "Metadata polling delay in ms."; - private static final int DEFAULT_METADATA_POLL_DELAY_MS = 5 * 60 * 1000; // default is every 5 minutes - - private static final String METADATA_STORAGE_TOPIC_CONF = SOURCE_CONF_PREFIX + "metadata.storage.topic"; - private static final String METADATA_STORAGE_TOPIC_CONF_DOC = "The name of the topic where the metadata are stored. " - + "The metadata topic will be created if it does not already exist, else it will use the pre-created topic."; - private static final String METADATA_STORAGE_TOPIC_CONF_DISPLAY = "Metadata storage topic."; - private static final String DEFAULT_METADATA_STORAGE_TOPIC = "_cosmos.metadata.topic"; - - // messageKey - private static final String MESSAGE_KEY_ENABLED_CONF = SOURCE_CONF_PREFIX + "messageKey.enabled"; - private static final String MESSAGE_KEY_ENABLED_CONF_DOC = "Whether to set the kafka record message key."; - private static final String MESSAGE_KEY_ENABLED_CONF_DISPLAY = "Kafka record message key enabled."; - private static final boolean DEFAULT_MESSAGE_KEY_ENABLED = true; - - private static final String MESSAGE_KEY_FIELD_CONF = SOURCE_CONF_PREFIX + "messageKey.field"; - private static final String MESSAGE_KEY_FIELD_CONF_DOC = "The field to use as the message key."; - private static final String MESSAGE_KEY_FIELD_CONF_DISPLAY = "Kafka message key field."; - private static final String DEFAULT_MESSAGE_KEY_FIELD = "id"; - - private final CosmosSourceContainersConfig containersConfig; - private final CosmosMetadataConfig metadataConfig; - private final CosmosSourceChangeFeedConfig changeFeedConfig; - private final CosmosSourceMessageKeyConfig messageKeyConfig; - - public CosmosSourceConfig(Map<String, ?> parsedConfigs) { - this(getConfigDef(), parsedConfigs); - } - - public CosmosSourceConfig(ConfigDef configDef, Map<String, ?> parsedConfigs) { - super(configDef, parsedConfigs); - this.containersConfig = this.parseContainersConfig(); - this.metadataConfig = this.parseMetadataConfig(); - this.changeFeedConfig = this.parseChangeFeedConfig(); - this.messageKeyConfig = this.parseMessageKeyConfig(); - } - - public static ConfigDef getConfigDef() { - ConfigDef configDef = KafkaCosmosConfig.getConfigDef(); - - defineContainersConfig(configDef); - defineMetadataConfig(configDef); - defineChangeFeedConfig(configDef); - defineMessageKeyConfig(configDef); - - return configDef; - } - - private static void defineContainersConfig(ConfigDef result) { - final String containersGroupName = "Containers"; - int containersGroupOrder = 0; - - result - .define( - DATABASE_NAME_CONF, - ConfigDef.Type.STRING, - ConfigDef.NO_DEFAULT_VALUE, - NON_EMPTY_STRING, - ConfigDef.Importance.HIGH, - DATABASE_NAME_CONF_DOC, - containersGroupName, - containersGroupOrder++, - ConfigDef.Width.LONG, - DATABASE_NAME_CONF_DISPLAY - ) - .define( - CONTAINERS_INCLUDE_ALL_CONF, - ConfigDef.Type.BOOLEAN, - DEFAULT_CONTAINERS_INCLUDE_ALL, - ConfigDef.Importance.HIGH, - CONTAINERS_INCLUDE_ALL_CONF_DOC, - containersGroupName, - containersGroupOrder++, - ConfigDef.Width.MEDIUM, - CONTAINERS_INCLUDE_ALL_CONF_DISPLAY - ) - .define( - CONTAINERS_INCLUDED_LIST_CONF, - ConfigDef.Type.STRING, - Strings.Emtpy, - ConfigDef.Importance.MEDIUM, - CONTAINERS_INCLUDED_LIST_CONF_DOC, - containersGroupName, - containersGroupOrder++, - ConfigDef.Width.LONG, - CONTAINERS_INCLUDED_LIST_CONF_DISPLAY - ) - .define( - CONTAINERS_TOPIC_MAP_CONF, - ConfigDef.Type.STRING, - Strings.Emtpy, - new ContainersTopicMapValidator(), - ConfigDef.Importance.MEDIUM, - CONTAINERS_TOPIC_MAP_CONF_DOC, - containersGroupName, - containersGroupOrder++, - ConfigDef.Width.LONG, - CONTAINERS_TOPIC_MAP_CONF_DISPLAY - ); - } - - private static void defineMetadataConfig(ConfigDef result) { - final String metadataGroupName = "Metadata"; - int metadataGroupOrder = 0; - - result - .define( - METADATA_POLL_DELAY_MS_CONF, - ConfigDef.Type.INT, - DEFAULT_METADATA_POLL_DELAY_MS, - new PositiveValueValidator(), - ConfigDef.Importance.MEDIUM, - METADATA_POLL_DELAY_MS_CONF_DOC, - metadataGroupName, - metadataGroupOrder++, - ConfigDef.Width.MEDIUM, - METADATA_POLL_DELAY_MS_CONF_DISPLAY - ) - .define( - METADATA_STORAGE_TOPIC_CONF, - ConfigDef.Type.STRING, - DEFAULT_METADATA_STORAGE_TOPIC, - NON_EMPTY_STRING, - ConfigDef.Importance.HIGH, - METADATA_STORAGE_TOPIC_CONF_DOC, - metadataGroupName, - metadataGroupOrder++, - ConfigDef.Width.LONG, - METADATA_STORAGE_TOPIC_CONF_DISPLAY - ); - } - - private static void defineChangeFeedConfig(ConfigDef result) { - final String changeFeedGroupName = "ChangeFeed"; - int changeFeedGroupOrder = 0; - - result - .define( - CHANGE_FEED_MODE_CONF, - ConfigDef.Type.STRING, - DEFAULT_CHANGE_FEED_MODE, - new ChangeFeedModeValidator(), - ConfigDef.Importance.HIGH, - CHANGE_FEED_MODE_CONF_DOC, - changeFeedGroupName, - changeFeedGroupOrder++, - ConfigDef.Width.MEDIUM, - CHANGE_FEED_MODE_CONF_DISPLAY - ) - .define( - CHANGE_FEED_START_FROM_CONF, - ConfigDef.Type.STRING, - DEFAULT_CHANGE_FEED_START_FROM, - new ChangeFeedStartFromValidator(), - ConfigDef.Importance.HIGH, - CHANGE_FEED_START_FROM_CONF_DOC, - changeFeedGroupName, - changeFeedGroupOrder++, - ConfigDef.Width.MEDIUM, - CHANGE_FEED_START_FROM_CONF_DISPLAY - ) - .define( - CHANGE_FEED_MAX_ITEM_COUNT_CONF, - ConfigDef.Type.INT, - DEFAULT_CHANGE_FEED_MAX_ITEM_COUNT, - new PositiveValueValidator(), - ConfigDef.Importance.MEDIUM, - CHANGE_FEED_MAX_ITEM_COUNT_CONF_DOC, - changeFeedGroupName, - changeFeedGroupOrder++, - ConfigDef.Width.MEDIUM, - CHANGE_FEED_MAX_ITEM_COUNT_CONF_DISPLAY - ); - } - - private static void defineMessageKeyConfig(ConfigDef result) { - final String messageGroupName = "Message Key"; - int messageGroupOrder = 0; - - result - .define( - MESSAGE_KEY_ENABLED_CONF, - ConfigDef.Type.BOOLEAN, - DEFAULT_MESSAGE_KEY_ENABLED, - ConfigDef.Importance.MEDIUM, - MESSAGE_KEY_ENABLED_CONF_DOC, - messageGroupName, - messageGroupOrder++, - ConfigDef.Width.SHORT, - MESSAGE_KEY_ENABLED_CONF_DISPLAY - ) - .define( - MESSAGE_KEY_FIELD_CONF, - ConfigDef.Type.STRING, - DEFAULT_MESSAGE_KEY_FIELD, - ConfigDef.Importance.HIGH, - MESSAGE_KEY_FIELD_CONF_DOC, - messageGroupName, - messageGroupOrder++, - ConfigDef.Width.MEDIUM, - MESSAGE_KEY_FIELD_CONF_DISPLAY - ); - } - - private CosmosSourceContainersConfig parseContainersConfig() { - String databaseName = this.getString(DATABASE_NAME_CONF); - boolean includeAllContainers = this.getBoolean(CONTAINERS_INCLUDE_ALL_CONF); - List<String> containersIncludedList = this.getContainersIncludedList(); - Map<String, String> containersTopicMap = this.getContainerToTopicMap(); - - return new CosmosSourceContainersConfig( - databaseName, - includeAllContainers, - containersIncludedList, - containersTopicMap - ); - } - - private List<String> getContainersIncludedList() { - return convertToList(this.getString(CONTAINERS_INCLUDED_LIST_CONF)); - } - - private Map<String, String> getContainerToTopicMap() { - List<String> containerTopicMapList = convertToList(this.getString(CONTAINERS_TOPIC_MAP_CONF)); - return containerTopicMapList - .stream() - .map(containerTopicMapString -> containerTopicMapString.split("#")) - .collect( - Collectors.toMap( - containerTopicMapArray -> containerTopicMapArray[1], - containerTopicMapArray -> containerTopicMapArray[0])); - } - - private CosmosMetadataConfig parseMetadataConfig() { - int metadataPollDelayInMs = this.getInt(METADATA_POLL_DELAY_MS_CONF); - String metadataTopicName = this.getString(METADATA_STORAGE_TOPIC_CONF); - - return new CosmosMetadataConfig(metadataPollDelayInMs, metadataTopicName); - } - - private CosmosSourceChangeFeedConfig parseChangeFeedConfig() { - CosmosChangeFeedModes changeFeedModes = this.parseChangeFeedMode(); - CosmosChangeFeedStartFromModes changeFeedStartFromMode = this.parseChangeFeedStartFromMode(); - Instant changeFeedStartFrom = this.parseChangeFeedStartFrom(changeFeedStartFromMode); - Integer changeFeedMaxItemCountHint = this.getInt(CHANGE_FEED_MAX_ITEM_COUNT_CONF); - - return new CosmosSourceChangeFeedConfig( - changeFeedModes, - changeFeedStartFromMode, - changeFeedStartFrom, - changeFeedMaxItemCountHint); - } - - private CosmosSourceMessageKeyConfig parseMessageKeyConfig() { - boolean messageKeyEnabled = this.getBoolean(MESSAGE_KEY_ENABLED_CONF); - String messageKeyField = this.getString(MESSAGE_KEY_FIELD_CONF); - - return new CosmosSourceMessageKeyConfig(messageKeyEnabled, messageKeyField); - } - private CosmosChangeFeedStartFromModes parseChangeFeedStartFromMode() { - String changeFeedStartFrom = this.getString(CHANGE_FEED_START_FROM_CONF); - if (changeFeedStartFrom.equalsIgnoreCase(CosmosChangeFeedStartFromModes.BEGINNING.getName())) { - return CosmosChangeFeedStartFromModes.BEGINNING; - } - - if (changeFeedStartFrom.equalsIgnoreCase(CosmosChangeFeedStartFromModes.NOW.getName())) { - return CosmosChangeFeedStartFromModes.NOW; - } - - return CosmosChangeFeedStartFromModes.POINT_IN_TIME; - } - - private Instant parseChangeFeedStartFrom(CosmosChangeFeedStartFromModes startFromMode) { - if (startFromMode == CosmosChangeFeedStartFromModes.POINT_IN_TIME) { - String changeFeedStartFrom = this.getString(CHANGE_FEED_START_FROM_CONF); - return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(changeFeedStartFrom.trim())); - } - - return null; - } - - private CosmosChangeFeedModes parseChangeFeedMode() { - String changeFeedMode = this.getString(CHANGE_FEED_MODE_CONF); - return CosmosChangeFeedModes.fromName(changeFeedMode); - } - - public CosmosSourceContainersConfig getContainersConfig() { - return containersConfig; - } - - public CosmosMetadataConfig getMetadataConfig() { - return metadataConfig; - } - - public CosmosSourceChangeFeedConfig getChangeFeedConfig() { - return changeFeedConfig; - } - - public CosmosSourceMessageKeyConfig getMessageKeyConfig() { - return messageKeyConfig; - } - - public static class ChangeFeedModeValidator implements ConfigDef.Validator { - @Override - @SuppressWarnings("unchecked") - public void ensureValid(String name, Object o) { - String changeFeedModeString = (String) o; - if (StringUtils.isEmpty(changeFeedModeString)) { - throw new ConfigException(name, o, "ChangeFeedMode can not be empty or null"); - } - - CosmosChangeFeedModes changeFeedMode = CosmosChangeFeedModes.fromName(changeFeedModeString); - if (changeFeedMode == null) { - throw new ConfigException(name, o, "Invalid ChangeFeedMode, only allow LatestVersion or AllVersionsAndDeletes"); - } - } - - @Override - public String toString() { - return "ChangeFeedMode. Only allow " + CosmosChangeFeedModes.values(); - } - } - - public static class ChangeFeedStartFromValidator implements ConfigDef.Validator { - @Override - @SuppressWarnings("unchecked") - public void ensureValid(String name, Object o) { - String changeFeedStartFromString = (String) o; - if (StringUtils.isEmpty(changeFeedStartFromString)) { - throw new ConfigException(name, o, "ChangeFeedStartFrom can not be empty or null"); - } - - CosmosChangeFeedStartFromModes changeFeedStartFromModes = - CosmosChangeFeedStartFromModes.fromName(changeFeedStartFromString); - if (changeFeedStartFromModes == null) { - try { - Instant.parse(changeFeedStartFromString); - } catch (DateTimeParseException dateTimeParseException) { - throw new ConfigException( - name, - o, - "Invalid changeFeedStartFrom." - + " only allow Now, Beginning or a certain point in time (UTC) for example 2020-02-10T14:15:03 "); - } - } - } - - @Override - public String toString() { - return "ChangeFeedStartFrom. Only allow Now, Beginning or a certain point in time (UTC) for example 2020-02-10T14:15:03"; - } - } - - public static class PositiveValueValidator implements ConfigDef.Validator { - @Override - @SuppressWarnings("unchecked") - public void ensureValid(String name, Object o) { - int value = Integer.parseInt(o.toString()); - - if (value <= 0) { - throw new ConfigException(name, o, "Invalid value, need to be >= 0"); - } - } - - @Override - public String toString() { - return "Value need to be >= 0"; - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceContainersConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceContainersConfig.java deleted file mode 100644 index 7f9a2d4284e8..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceContainersConfig.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; - -import java.util.List; -import java.util.Map; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -public class CosmosSourceContainersConfig { - public static final String CONTAINER_TOPIC_MAP_SEPARATOR = "#"; - - private final String databaseName; - private final boolean includeAllContainers; - private final List<String> includedContainers; - private final Map<String, String> containerToTopicMap; - - public CosmosSourceContainersConfig( - String databaseName, - boolean includeAllContainers, - List<String> includedContainers, - Map<String, String> containerToTopicMap) { - - checkArgument(StringUtils.isNotEmpty(databaseName), "Argument 'databaseName' can not be null"); - checkNotNull(includedContainers, "Argument 'includedContainers' can not be null"); - - this.databaseName = databaseName; - this.includeAllContainers = includeAllContainers; - this.includedContainers = includedContainers; - this.containerToTopicMap = containerToTopicMap; - } - - public String getDatabaseName() { - return databaseName; - } - - public boolean isIncludeAllContainers() { - return includeAllContainers; - } - - public List<String> getIncludedContainers() { - return includedContainers; - } - - public Map<String, String> getContainerToTopicMap() { - return containerToTopicMap; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceMessageKeyConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceMessageKeyConfig.java deleted file mode 100644 index 82c59531d059..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceMessageKeyConfig.java +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -public class CosmosSourceMessageKeyConfig { - private final boolean messageKeyEnabled; - private final String messageKeyField; - - public CosmosSourceMessageKeyConfig(boolean messageKeyEnabled, String messageKeyField) { - this.messageKeyEnabled = messageKeyEnabled; - this.messageKeyField = messageKeyField; - } - - public boolean isMessageKeyEnabled() { - return messageKeyEnabled; - } - - public String getMessageKeyField() { - return messageKeyField; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceOffsetStorageReader.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceOffsetStorageReader.java deleted file mode 100644 index b7c4118379e5..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceOffsetStorageReader.java +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.routing.Range; -import org.apache.kafka.connect.storage.OffsetStorageReader; - -import java.util.Map; - -public class CosmosSourceOffsetStorageReader { - private final OffsetStorageReader offsetStorageReader; - - public CosmosSourceOffsetStorageReader(OffsetStorageReader offsetStorageReader) { - this.offsetStorageReader = offsetStorageReader; - } - - public FeedRangesMetadataTopicOffset getFeedRangesMetadataOffset(String databaseName, String containerRid) { - Map<String, Object> topicOffsetMap = - this.offsetStorageReader - .offset( - FeedRangesMetadataTopicPartition.toMap( - new FeedRangesMetadataTopicPartition(databaseName, containerRid))); - - return FeedRangesMetadataTopicOffset.fromMap(topicOffsetMap); - } - - public ContainersMetadataTopicOffset getContainersMetadataOffset(String databaseName) { - Map<String, Object> topicOffsetMap = - this.offsetStorageReader - .offset( - ContainersMetadataTopicPartition.toMap( - new ContainersMetadataTopicPartition(databaseName))); - - return ContainersMetadataTopicOffset.fromMap(topicOffsetMap); - } - - public FeedRangeContinuationTopicOffset getFeedRangeContinuationOffset( - String databaseName, - String collectionRid, - Range<String> feedRange) { - - Map<String, Object> topicOffsetMap = - this.offsetStorageReader - .offset( - FeedRangeContinuationTopicPartition.toMap( - new FeedRangeContinuationTopicPartition(databaseName, collectionRid, feedRange))); - - return FeedRangeContinuationTopicOffset.fromMap(topicOffsetMap); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTask.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTask.java deleted file mode 100644 index ec358965e765..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTask.java +++ /dev/null @@ -1,341 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.BridgeInternal; -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.CosmosBridgeInternal; -import com.azure.cosmos.implementation.AsyncDocumentClient; -import com.azure.cosmos.implementation.PartitionKeyRange; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.apachecommons.lang.tuple.Pair; -import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; -import com.azure.cosmos.implementation.guava25.base.Stopwatch; -import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosConstants; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; -import com.azure.cosmos.models.CosmosChangeFeedRequestOptions; -import com.azure.cosmos.models.FeedRange; -import com.azure.cosmos.models.FeedResponse; -import com.azure.cosmos.models.ModelBridgeInternal; -import com.fasterxml.jackson.databind.JsonNode; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaAndValue; -import org.apache.kafka.connect.source.SourceRecord; -import org.apache.kafka.connect.source.SourceTask; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Mono; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.stream.Collectors; - -public class CosmosSourceTask extends SourceTask { - private static final Logger LOGGER = LoggerFactory.getLogger(CosmosSourceTask.class); - private static final String LSN_ATTRIBUTE_NAME = "_lsn"; - - private CosmosSourceTaskConfig taskConfig; - private CosmosAsyncClient cosmosClient; - private Queue<ITaskUnit> taskUnitsQueue = new LinkedList<>(); - - @Override - public String version() { - return KafkaCosmosConstants.CURRENT_VERSION; - } - - @Override - public void start(Map<String, String> map) { - LOGGER.info("Starting the kafka cosmos source task..."); - - this.taskConfig = new CosmosSourceTaskConfig(map); - if (this.taskConfig.getMetadataTaskUnit() != null) { - // adding metadata task units into the head of the queue - this.taskUnitsQueue.add(this.taskConfig.getMetadataTaskUnit()); - } - - this.taskUnitsQueue.addAll(this.taskConfig.getFeedRangeTaskUnits()); - LOGGER.info("Creating the cosmos client"); - - // TODO[GA]: optimize the client creation, client metadata cache? - this.cosmosClient = CosmosClientStore.getCosmosClient(this.taskConfig.getAccountConfig()); - } - - @Override - public List<SourceRecord> poll() { - // do not poll it from the queue yet - // we need to make sure not losing tasks for failure cases - ITaskUnit taskUnit = this.taskUnitsQueue.poll(); - try { - if (taskUnit == null) { - // there is no task to do - return new ArrayList<>(); - } - - List<SourceRecord> results = new ArrayList<>(); - if (taskUnit instanceof MetadataTaskUnit) { - results.addAll(executeMetadataTask((MetadataTaskUnit) taskUnit)); - LOGGER.info( - "Return {} metadata records, databaseName {}", results.size(), ((MetadataTaskUnit) taskUnit).getDatabaseName()); - - } else { - Stopwatch stopwatch = Stopwatch.createStarted(); - - LOGGER.trace("Polling for task {}", taskUnit); - Pair<List<SourceRecord>, Boolean> feedRangeTaskResults = executeFeedRangeTask((FeedRangeTaskUnit) taskUnit); - results.addAll(feedRangeTaskResults.getLeft()); - - // for split, new feedRangeTaskUnit will be created, so we do not need to add the original taskUnit back to the queue - if (!feedRangeTaskResults.getRight()) { - LOGGER.trace("Adding task {} back to queue", taskUnit); - this.taskUnitsQueue.add(taskUnit); - } - - stopwatch.stop(); - LOGGER.debug( - "Return {} records, databaseName {}, containerName {}, containerRid {}, feedRange {}, durationInMs {}", - results.size(), - ((FeedRangeTaskUnit) taskUnit).getDatabaseName(), - ((FeedRangeTaskUnit) taskUnit).getContainerName(), - ((FeedRangeTaskUnit) taskUnit).getContainerRid(), - ((FeedRangeTaskUnit) taskUnit).getFeedRange(), - stopwatch.elapsed().toMillis()); - } - return results; - } catch (Exception e) { - // for error cases, we should always the task back to the queue - this.taskUnitsQueue.add(taskUnit); - - // TODO[Public Preview]: add checking for max retries checking - throw KafkaCosmosExceptionsHelper.convertToConnectException(e, "PollTask failed"); - } - } - - private List<SourceRecord> executeMetadataTask(MetadataTaskUnit taskUnit) { - List<SourceRecord> sourceRecords = new ArrayList<>(); - - // add the containers metadata record - it track the databaseName -> List[containerRid] mapping - ContainersMetadataTopicPartition metadataTopicPartition = - new ContainersMetadataTopicPartition(taskUnit.getDatabaseName()); - ContainersMetadataTopicOffset metadataTopicOffset = - new ContainersMetadataTopicOffset(taskUnit.getContainerRids()); - - sourceRecords.add( - new SourceRecord( - ContainersMetadataTopicPartition.toMap(metadataTopicPartition), - ContainersMetadataTopicOffset.toMap(metadataTopicOffset), - taskUnit.getTopic(), - SchemaAndValue.NULL.schema(), - SchemaAndValue.NULL.value())); - - // add the container feedRanges metadata record - it tracks the containerRid -> List[FeedRange] mapping - for (String containerRid : taskUnit.getContainersEffectiveRangesMap().keySet()) { - FeedRangesMetadataTopicPartition feedRangesMetadataTopicPartition = - new FeedRangesMetadataTopicPartition(taskUnit.getDatabaseName(), containerRid); - FeedRangesMetadataTopicOffset feedRangesMetadataTopicOffset = - new FeedRangesMetadataTopicOffset(taskUnit.getContainersEffectiveRangesMap().get(containerRid)); - - sourceRecords.add( - new SourceRecord( - FeedRangesMetadataTopicPartition.toMap(feedRangesMetadataTopicPartition), - FeedRangesMetadataTopicOffset.toMap(feedRangesMetadataTopicOffset), - taskUnit.getTopic(), - SchemaAndValue.NULL.schema(), - SchemaAndValue.NULL.value())); - } - - LOGGER.info("There are {} metadata records being created/updated", sourceRecords.size()); - return sourceRecords; - } - - private Pair<List<SourceRecord>, Boolean> executeFeedRangeTask(FeedRangeTaskUnit feedRangeTaskUnit) { - // each time we will only pull one page - CosmosChangeFeedRequestOptions changeFeedRequestOptions = - this.getChangeFeedRequestOptions(feedRangeTaskUnit); - - // split/merge will be handled in source task - ModelBridgeInternal.getChangeFeedIsSplitHandlingDisabled(changeFeedRequestOptions); - - CosmosAsyncContainer container = - this.cosmosClient - .getDatabase(feedRangeTaskUnit.getDatabaseName()) - .getContainer(feedRangeTaskUnit.getContainerName()); - - return container.queryChangeFeed(changeFeedRequestOptions, JsonNode.class) - .byPage(this.taskConfig.getChangeFeedConfig().getMaxItemCountHint()) - .next() - .map(feedResponse -> { - List<SourceRecord> records = handleSuccessfulResponse(feedResponse, feedRangeTaskUnit); - return Pair.of(records, false); - }) - .onErrorResume(throwable -> { - if (KafkaCosmosExceptionsHelper.isFeedRangeGoneException(throwable)) { - return this.handleFeedRangeGone(feedRangeTaskUnit) - .map(shouldRemoveOriginalTaskUnit -> Pair.of(new ArrayList<>(), shouldRemoveOriginalTaskUnit)); - } - - return Mono.error(throwable); - }) - .block(); - } - - private List<SourceRecord> handleSuccessfulResponse( - FeedResponse<JsonNode> feedResponse, - FeedRangeTaskUnit feedRangeTaskUnit) { - - List<SourceRecord> sourceRecords = new ArrayList<>(); - for (JsonNode item : feedResponse.getResults()) { - FeedRangeContinuationTopicPartition feedRangeContinuationTopicPartition = - new FeedRangeContinuationTopicPartition( - feedRangeTaskUnit.getDatabaseName(), - feedRangeTaskUnit.getContainerRid(), - feedRangeTaskUnit.getFeedRange()); - FeedRangeContinuationTopicOffset feedRangeContinuationTopicOffset = - new FeedRangeContinuationTopicOffset( - feedResponse.getContinuationToken(), - getItemLsn(item)); - - // Set the Kafka message key if option is enabled and field is configured in document - String messageKey = this.getMessageKey(item); - - // Convert JSON to Kafka Connect struct and JSON schema - SchemaAndValue schemaAndValue = JsonToStruct.recordToSchemaAndValue(item); - - sourceRecords.add( - new SourceRecord( - FeedRangeContinuationTopicPartition.toMap(feedRangeContinuationTopicPartition), - FeedRangeContinuationTopicOffset.toMap(feedRangeContinuationTopicOffset), - feedRangeTaskUnit.getTopic(), - Schema.STRING_SCHEMA, - messageKey, - schemaAndValue.schema(), - schemaAndValue.value())); - } - - // Important: track the continuationToken - feedRangeTaskUnit.setContinuationState(feedResponse.getContinuationToken()); - return sourceRecords; - } - - private Mono<Boolean> handleFeedRangeGone(FeedRangeTaskUnit feedRangeTaskUnit) { - // need to find out whether it is split or merge - AsyncDocumentClient asyncDocumentClient = CosmosBridgeInternal.getAsyncDocumentClient(this.cosmosClient); - CosmosAsyncContainer container = - this.cosmosClient - .getDatabase(feedRangeTaskUnit.getDatabaseName()) - .getContainer(feedRangeTaskUnit.getContainerName()); - return asyncDocumentClient - .getCollectionCache() - .resolveByNameAsync(null, BridgeInternal.extractContainerSelfLink(container), null) - .flatMap(collection -> { - return asyncDocumentClient.getPartitionKeyRangeCache().tryGetOverlappingRangesAsync( - null, - collection.getResourceId(), - feedRangeTaskUnit.getFeedRange(), - true, - null); - }) - .flatMap(pkRangesValueHolder -> { - if (pkRangesValueHolder == null || pkRangesValueHolder.v == null) { - return Mono.error(new IllegalStateException("There are no overlapping ranges for the range")); - } - - List<PartitionKeyRange> partitionKeyRanges = pkRangesValueHolder.v; - if (partitionKeyRanges.size() == 1) { - // merge happens - LOGGER.info( - "FeedRange {} is merged into {}, but we will continue polling data from feedRange {}", - feedRangeTaskUnit.getFeedRange(), - partitionKeyRanges.get(0).toRange(), - feedRangeTaskUnit.getFeedRange()); - - // Continue using polling data from the current task unit feedRange - return Mono.just(false); - } else { - LOGGER.info( - "FeedRange {} is split into {}. Will create new task units. ", - feedRangeTaskUnit.getFeedRange(), - partitionKeyRanges.stream().map(PartitionKeyRange::toRange).collect(Collectors.toList()) - ); - - for (PartitionKeyRange pkRange : partitionKeyRanges) { - FeedRangeTaskUnit childTaskUnit = - new FeedRangeTaskUnit( - feedRangeTaskUnit.getDatabaseName(), - feedRangeTaskUnit.getContainerName(), - feedRangeTaskUnit.getContainerRid(), - pkRange.toRange(), - feedRangeTaskUnit.getContinuationState(), - feedRangeTaskUnit.getTopic()); - this.taskUnitsQueue.add(childTaskUnit); - } - - // remove the current task unit from the queue - return Mono.just(true); - } - }); - } - - private String getItemLsn(JsonNode item) { - return item.get(LSN_ATTRIBUTE_NAME).asText(); - } - - private String getMessageKey(JsonNode item) { - String messageKey = ""; - if (this.taskConfig.getMessageKeyConfig().isMessageKeyEnabled()) { - JsonNode messageKeyFieldNode = item.get(this.taskConfig.getMessageKeyConfig().getMessageKeyField()); - if (messageKeyFieldNode != null) { - messageKey = messageKeyFieldNode.asText(); - } - } - - return messageKey; - } - - private CosmosChangeFeedRequestOptions getChangeFeedRequestOptions(FeedRangeTaskUnit feedRangeTaskUnit) { - CosmosChangeFeedRequestOptions changeFeedRequestOptions = null; - FeedRange changeFeedRange = new FeedRangeEpkImpl(feedRangeTaskUnit.getFeedRange()); - if (StringUtils.isEmpty(feedRangeTaskUnit.getContinuationState())) { - switch (this.taskConfig.getChangeFeedConfig().getChangeFeedStartFromModes()) { - case BEGINNING: - changeFeedRequestOptions = - CosmosChangeFeedRequestOptions.createForProcessingFromBeginning(changeFeedRange); - break; - case NOW: - changeFeedRequestOptions = - CosmosChangeFeedRequestOptions.createForProcessingFromNow(changeFeedRange); - break; - case POINT_IN_TIME: - changeFeedRequestOptions = - CosmosChangeFeedRequestOptions - .createForProcessingFromPointInTime( - this.taskConfig.getChangeFeedConfig().getStartFrom(), - changeFeedRange); - break; - default: - throw new IllegalArgumentException(feedRangeTaskUnit.getContinuationState() + " is not supported"); - } - - if (this.taskConfig.getChangeFeedConfig().getChangeFeedModes() == CosmosChangeFeedModes.ALL_VERSION_AND_DELETES) { - changeFeedRequestOptions.allVersionsAndDeletes(); - } - } else { - changeFeedRequestOptions = - CosmosChangeFeedRequestOptions.createForProcessingFromContinuation(feedRangeTaskUnit.getContinuationState()); - } - - return changeFeedRequestOptions; - } - - @Override - public void stop() { - if (this.cosmosClient != null) { - this.cosmosClient.close(); - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTaskConfig.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTaskConfig.java deleted file mode 100644 index 147bb61814e9..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTaskConfig.java +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.Utils; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.kafka.common.config.ConfigDef; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class CosmosSourceTaskConfig extends CosmosSourceConfig { - private static final ObjectMapper OBJECT_MAPPER = Utils.getSimpleObjectMapper(); - private static final String SOURCE_TASK_CONFIG_PREFIX = "kafka.connect.cosmos.source.task."; - - public static final String SOURCE_METADATA_TASK_UNIT = SOURCE_TASK_CONFIG_PREFIX + "metadataTaskUnit"; - public static final String SOURCE_FEED_RANGE_TASK_UNITS = SOURCE_TASK_CONFIG_PREFIX + "feedRangeTaskUnits"; - - private final List<FeedRangeTaskUnit> feedRangeTaskUnits; - private MetadataTaskUnit metadataTaskUnit; - - public CosmosSourceTaskConfig(Map<String, String> parsedConfigs) { - super(getConfigDef(), parsedConfigs); - - this.feedRangeTaskUnits = this.parseFeedRangeTaskUnits(); - this.metadataTaskUnit = this.parseMetadataTaskUnit(); - } - - public static ConfigDef getConfigDef() { - ConfigDef configDef = CosmosSourceConfig.getConfigDef(); - defineTaskUnitsConfig(configDef); - - return configDef; - } - - private static void defineTaskUnitsConfig(ConfigDef result) { - result - .defineInternal( - SOURCE_FEED_RANGE_TASK_UNITS, - ConfigDef.Type.STRING, - ConfigDef.NO_DEFAULT_VALUE, - ConfigDef.Importance.HIGH - ) - .defineInternal( - SOURCE_METADATA_TASK_UNIT, - ConfigDef.Type.STRING, - null, - ConfigDef.Importance.HIGH - ); - } - - private List<FeedRangeTaskUnit> parseFeedRangeTaskUnits() { - String feedRangesTaskUnitsConfig = this.getString(SOURCE_FEED_RANGE_TASK_UNITS); - - try { - if (!StringUtils.isEmpty(feedRangesTaskUnitsConfig)) { - return OBJECT_MAPPER - .readValue(feedRangesTaskUnitsConfig, new TypeReference<List<String>>() {}) - .stream() - .map(taskUnitConfigJson -> { - try { - return OBJECT_MAPPER.readValue(taskUnitConfigJson, FeedRangeTaskUnit.class); - } catch (JsonProcessingException e) { - throw new IllegalArgumentException("Failed to parseFeedRangeTaskUnit[" + taskUnitConfigJson + "]", e); - } - }) - .collect(Collectors.toList()); - } - - return new ArrayList<>(); - } catch (JsonProcessingException e) { - throw new IllegalArgumentException("Failed to parseFeedRangeTaskUnits[" + feedRangesTaskUnitsConfig + "]", e); - } - - } - - private MetadataTaskUnit parseMetadataTaskUnit() { - String metadataTaskUnitConfig = this.getString(SOURCE_METADATA_TASK_UNIT); - if (!StringUtils.isEmpty(metadataTaskUnitConfig)) { - try { - return OBJECT_MAPPER.readValue(metadataTaskUnitConfig, MetadataTaskUnit.class); - } catch (JsonProcessingException e) { - throw new IllegalArgumentException("Failed to parseMetadataTaskUnit[" + metadataTaskUnitConfig + "]", e); - } - } - - return null; - } - - public static Map<String, String> getFeedRangeTaskUnitsConfigMap(List<FeedRangeTaskUnit> feedRangeTaskUnits) { - try { - Map<String, String> taskConfigMap = new HashMap<>(); - taskConfigMap.put( - SOURCE_FEED_RANGE_TASK_UNITS, - OBJECT_MAPPER.writeValueAsString( - feedRangeTaskUnits - .stream() - .map(taskUnit -> { - try { - return OBJECT_MAPPER.writeValueAsString(taskUnit); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()) - )); - return taskConfigMap; - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - public static Map<String, String> getMetadataTaskUnitConfigMap(MetadataTaskUnit metadataTaskUnit) { - try { - Map<String, String> taskConfigMap = new HashMap<>(); - if (metadataTaskUnit != null) { - taskConfigMap.put(SOURCE_METADATA_TASK_UNIT, OBJECT_MAPPER.writeValueAsString(metadataTaskUnit)); - } - return taskConfigMap; - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - public List<FeedRangeTaskUnit> getFeedRangeTaskUnits() { - return feedRangeTaskUnits; - } - - public MetadataTaskUnit getMetadataTaskUnit() { - return metadataTaskUnit; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeContinuationTopicOffset.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeContinuationTopicOffset.java deleted file mode 100644 index 72d294262aed..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeContinuationTopicOffset.java +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; - -public class FeedRangeContinuationTopicOffset { - private static final String ITEM_LSN_KEY = "cosmos.source.feedRange.item.lsn"; - private static final String CONTINUATION_KEY = "cosmos.source.feedRange.continuation"; - - private final String continuationState; - private final String itemLsn; - - public FeedRangeContinuationTopicOffset( - String continuationState, - String itemLsn) { - checkArgument(StringUtils.isNotEmpty(continuationState), "Argument 'continuationState' should not be null"); - checkArgument(StringUtils.isNotEmpty(itemLsn), "Argument 'itemLsn' should not be null"); - - this.itemLsn = itemLsn; - this.continuationState = continuationState; - } - - public String getContinuationState() { - return continuationState; - } - - public String getItemLsn() { - return itemLsn; - } - - public static Map<String, Object> toMap(FeedRangeContinuationTopicOffset offset) { - Map<String, Object> map = new HashMap<>(); - map.put(CONTINUATION_KEY, offset.getContinuationState()); - map.put(ITEM_LSN_KEY, offset.getItemLsn()); - - return map; - } - - public static FeedRangeContinuationTopicOffset fromMap(Map<String, Object> offsetMap) { - if (offsetMap == null) { - return null; - } - - String continuationState = offsetMap.get(CONTINUATION_KEY).toString(); - String itemLsn = offsetMap.get(ITEM_LSN_KEY).toString(); - return new FeedRangeContinuationTopicOffset(continuationState, itemLsn); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeContinuationTopicPartition.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeContinuationTopicPartition.java deleted file mode 100644 index c981c6b9fe6d..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeContinuationTopicPartition.java +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.routing.Range; - -import java.util.HashMap; -import java.util.Map; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -public class FeedRangeContinuationTopicPartition { - private static final String DATABASE_NAME_KEY = "cosmos.source.database.name"; - public static final String CONTAINER_RESOURCE_ID_KEY = "cosmos.source.container.resourceId"; - private static final String CONTAINER_FEED_RANGE_KEY = "cosmos.source.feedRange"; - - private final String databaseName; - private final String containerRid; - private final Range<String> feedRange; - - public FeedRangeContinuationTopicPartition( - String databaseName, - String containerRid, - Range<String> feedRange) { - checkArgument(StringUtils.isNotEmpty(databaseName), "Argument 'databaseName' should not be null"); - checkArgument(StringUtils.isNotEmpty(containerRid), "Argument 'containerRid' should not be null"); - checkNotNull(feedRange, "Argument 'feedRange' can not be null"); - - this.databaseName = databaseName; - this.containerRid = containerRid; - this.feedRange = feedRange; - } - - public String getDatabaseName() { - return databaseName; - } - - public String getContainerRid() { - return containerRid; - } - - public Range<String> getFeedRange() { - return feedRange; - } - - public static Map<String, Object> toMap(FeedRangeContinuationTopicPartition partition) { - Map<String, Object> map = new HashMap<>(); - map.put(DATABASE_NAME_KEY, partition.getDatabaseName()); - map.put(CONTAINER_RESOURCE_ID_KEY, partition.getContainerRid()); - map.put(CONTAINER_FEED_RANGE_KEY, partition.getFeedRange().toJson()); - - return map; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeTaskUnit.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeTaskUnit.java deleted file mode 100644 index 8a184afbc2fb..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangeTaskUnit.java +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.routing.Range; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; - -import java.io.IOException; -import java.util.Objects; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -@JsonSerialize(using = FeedRangeTaskUnit.FeedRangeTaskUnitSerializer.class) -@JsonDeserialize(using = FeedRangeTaskUnit.FeedRangeTaskUnitDeserializer.class) -public class FeedRangeTaskUnit implements ITaskUnit { - private String databaseName; - private String containerName; - private String containerRid; - private Range<String> feedRange; - private String continuationState; - private String topic; - - public FeedRangeTaskUnit() {} - - public FeedRangeTaskUnit( - String databaseName, - String containerName, - String containerRid, - Range<String> feedRange, - String continuationState, - String topic) { - - checkArgument(StringUtils.isNotEmpty(databaseName), "Argument 'databaseName' should not be null"); - checkArgument(StringUtils.isNotEmpty(containerName), "Argument 'containerName' should not be null"); - checkArgument(StringUtils.isNotEmpty(containerRid), "Argument 'containerRid' should not be null"); - checkNotNull(feedRange, "Argument 'feedRange' can not be null"); - checkArgument(StringUtils.isNotEmpty(topic), "Argument 'topic' should not be null"); - - this.databaseName = databaseName; - this.containerName = containerName; - this.containerRid = containerRid; - this.feedRange = feedRange; - this.continuationState = continuationState; - this.topic = topic; - } - - public String getDatabaseName() { - return databaseName; - } - - public String getContainerName() { - return containerName; - } - - public String getContainerRid() { - return containerRid; - } - - public Range<String> getFeedRange() { - return feedRange; - } - - public String getContinuationState() { - return continuationState; - } - - public void setContinuationState(String continuationState) { - this.continuationState = continuationState; - } - - public String getTopic() { - return topic; - } - - @Override - public String toString() { - return "FeedRangeTaskUnit{" - + "databaseName='" + databaseName + '\'' - + ", containerName='" + containerName + '\'' - + ", containerRid='" + containerRid + '\'' - + ", feedRange=" + feedRange - + ", continuationState='" + continuationState + '\'' - + ", topic='" + topic + '\'' - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - FeedRangeTaskUnit that = (FeedRangeTaskUnit) o; - return databaseName.equals(that.databaseName) - && containerName.equals(that.containerName) - && containerRid.equals(that.containerRid) - && feedRange.equals(that.feedRange) - && Objects.equals(continuationState, that.continuationState) - && topic.equals(that.topic); - } - - @Override - public int hashCode() { - return Objects.hash( - databaseName, - containerName, - containerRid, - feedRange, - continuationState, - topic); - } - - public static class FeedRangeTaskUnitSerializer extends com.fasterxml.jackson.databind.JsonSerializer<FeedRangeTaskUnit> { - @Override - public void serialize(FeedRangeTaskUnit feedRangeTaskUnit, - JsonGenerator writer, - SerializerProvider serializerProvider) throws IOException { - writer.writeStartObject(); - writer.writeStringField("databaseName", feedRangeTaskUnit.getDatabaseName()); - writer.writeStringField("containerName", feedRangeTaskUnit.getContainerName()); - writer.writeStringField("containerRid", feedRangeTaskUnit.getContainerRid()); - writer.writeStringField("feedRange", feedRangeTaskUnit.getFeedRange().toString()); - if (!StringUtils.isEmpty(feedRangeTaskUnit.getContinuationState())) { - writer.writeStringField("continuationState", feedRangeTaskUnit.getContinuationState()); - } - writer.writeStringField("topic", feedRangeTaskUnit.getTopic()); - writer.writeEndObject(); - } - } - - static class FeedRangeTaskUnitDeserializer extends StdDeserializer<FeedRangeTaskUnit> { - FeedRangeTaskUnitDeserializer() { - super(FeedRangeTaskUnit.class); - } - - @Override - public FeedRangeTaskUnit deserialize( - JsonParser jsonParser, - DeserializationContext deserializationContext) throws IOException { - - final JsonNode rootNode = jsonParser.getCodec().readTree(jsonParser); - String databaseName = rootNode.get("databaseName").asText(); - String containerName = rootNode.get("containerName").asText(); - String containerRid = rootNode.get("containerRid").asText(); - Range<String> feedRange = new Range<String>(rootNode.get("feedRange").asText()); - String continuationState = null; - if (rootNode.has("continuationState")) { - continuationState = rootNode.get("continuationState").asText(); - } - - String topic = rootNode.get("topic").asText(); - - return new FeedRangeTaskUnit(databaseName, containerName, containerRid, feedRange, continuationState, topic); - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangesMetadataTopicOffset.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangesMetadataTopicOffset.java deleted file mode 100644 index 5350a47ec744..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangesMetadataTopicOffset.java +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.Utils; -import com.azure.cosmos.implementation.routing.Range; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -public class FeedRangesMetadataTopicOffset { - public static final String CONTAINER_FEED_RANGES_KEY = "cosmos.source.metadata.container.feedRanges"; - public static final ObjectMapper OBJECT_MAPPER = Utils.getSimpleObjectMapper(); - - private final List<Range<String>> feedRanges; - - public FeedRangesMetadataTopicOffset(List<Range<String>> feedRanges) { - checkNotNull(feedRanges, "Argument 'feedRanges' can not be null"); - - this.feedRanges = feedRanges; - } - - public List<Range<String>> getFeedRanges() { - return feedRanges; - } - - public static Map<String, Object> toMap(FeedRangesMetadataTopicOffset offset) { - try { - Map<String, Object> map = new HashMap<>(); - - // offset can only contain primitive types - map.put( - CONTAINER_FEED_RANGES_KEY, - OBJECT_MAPPER - .writeValueAsString( - offset.getFeedRanges().stream().map(range -> range.toJson()).collect(Collectors.toList()))); - - return map; - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } - - public static FeedRangesMetadataTopicOffset fromMap(Map<String, Object> offsetMap) { - if (offsetMap == null) { - return null; - } - - String feedRangesValue = offsetMap.get(CONTAINER_FEED_RANGES_KEY).toString(); - try { - List<Range<String>> feedRanges = - OBJECT_MAPPER - .readValue(feedRangesValue, new TypeReference<List<String>>() {}) - .stream() - .map(rangeJson -> new Range<String>(rangeJson)) - .collect(Collectors.toList()); - - return new FeedRangesMetadataTopicOffset(feedRanges); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangesMetadataTopicPartition.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangesMetadataTopicPartition.java deleted file mode 100644 index 03c0c50e8c7e..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/FeedRangesMetadataTopicPartition.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; - -import java.util.HashMap; -import java.util.Map; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; - -public class FeedRangesMetadataTopicPartition { - public static final String DATABASE_NAME_KEY = "cosmos.source.metadata.database.name"; - public static final String CONTAINER_RESOURCE_ID_KEY = "cosmos.source.metadata.container.resourceId"; - private final String databaseName; - private final String containerRid; - - public FeedRangesMetadataTopicPartition(String databaseName, String containerRid) { - checkArgument(StringUtils.isNotEmpty(databaseName), "Argument 'databaseName' should not be null"); - checkArgument(StringUtils.isNotEmpty(containerRid), "Argument 'containerRid' should not be null"); - - this.databaseName = databaseName; - this.containerRid = containerRid; - } - - public String getDatabaseName() { - return databaseName; - } - - public String getContainerRid() { - return containerRid; - } - - public static Map<String, Object> toMap(FeedRangesMetadataTopicPartition partition) { - Map<String, Object> map = new HashMap<>(); - map.put(DATABASE_NAME_KEY, partition.getDatabaseName()); - map.put(CONTAINER_RESOURCE_ID_KEY, partition.getContainerRid()); - return map; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ITaskUnit.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ITaskUnit.java deleted file mode 100644 index 2c7f6efdff16..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/ITaskUnit.java +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -public interface ITaskUnit { -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/JsonToStruct.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/JsonToStruct.java deleted file mode 100644 index c73af26327b5..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/JsonToStruct.java +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.fasterxml.jackson.databind.JsonNode; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaAndValue; -import org.apache.kafka.connect.data.SchemaBuilder; -import org.apache.kafka.connect.data.Struct; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static java.lang.String.format; -import static org.apache.kafka.connect.data.Values.convertToByte; -import static org.apache.kafka.connect.data.Values.convertToDouble; -import static org.apache.kafka.connect.data.Values.convertToFloat; -import static org.apache.kafka.connect.data.Values.convertToInteger; -import static org.apache.kafka.connect.data.Values.convertToLong; -import static org.apache.kafka.connect.data.Values.convertToShort; - -public class JsonToStruct { - private static final Logger LOGGER = LoggerFactory.getLogger(JsonToStruct.class); - private static final String SCHEMA_NAME_TEMPLATE = "inferred_name_%s"; - - public static SchemaAndValue recordToSchemaAndValue(final JsonNode node) { - Schema nodeSchema = inferSchema(node); - Struct struct = new Struct(nodeSchema); - - if (nodeSchema != null) { - nodeSchema.fields().forEach(field -> { - JsonNode fieldValue = node.get(field.name()); - if (fieldValue != null) { - SchemaAndValue schemaAndValue = toSchemaAndValue(field.schema(), fieldValue); - struct.put(field, schemaAndValue.value()); - } else { - boolean optionalField = field.schema().isOptional(); - Object defaultValue = field.schema().defaultValue(); - if (optionalField || defaultValue != null) { - struct.put(field, defaultValue); - } else { - LOGGER.error("Missing value for field {}", field.name()); - } - } - }); - } - return new SchemaAndValue(nodeSchema, struct); - } - - private static Schema inferSchema(JsonNode jsonNode) { - switch (jsonNode.getNodeType()) { - case NULL: - return Schema.OPTIONAL_STRING_SCHEMA; - case BOOLEAN: - return Schema.BOOLEAN_SCHEMA; - case NUMBER: - if (jsonNode.isIntegralNumber()) { - return Schema.INT64_SCHEMA; - } else { - return Schema.FLOAT64_SCHEMA; - } - case ARRAY: - List<JsonNode> jsonValues = new ArrayList<>(); - SchemaBuilder arrayBuilder; - jsonNode.forEach(jn -> jsonValues.add(jn)); - - Schema firstItemSchema = jsonValues.isEmpty() ? Schema.OPTIONAL_STRING_SCHEMA - : inferSchema(jsonValues.get(0)); - if (jsonValues.isEmpty() || jsonValues.stream() - .anyMatch(jv -> !Objects.equals(inferSchema(jv), firstItemSchema))) { - // If array is emtpy or it contains elements with different schema types - arrayBuilder = SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA); - arrayBuilder.name(generateName(arrayBuilder)); - return arrayBuilder.optional().build(); - } - arrayBuilder = SchemaBuilder.array(inferSchema(jsonValues.get(0))); - arrayBuilder.name(generateName(arrayBuilder)); - return arrayBuilder.optional().build(); - case OBJECT: - SchemaBuilder structBuilder = SchemaBuilder.struct(); - Iterator<Map.Entry<String, JsonNode>> it = jsonNode.fields(); - while (it.hasNext()) { - Map.Entry<String, JsonNode> entry = it.next(); - structBuilder.field(entry.getKey(), inferSchema(entry.getValue())); - } - structBuilder.name(generateName(structBuilder)); - return structBuilder.build(); - case STRING: - return Schema.STRING_SCHEMA; - // TODO[GA]: do we need to support binary/pojo? - case BINARY: - case MISSING: - case POJO: - default: - return null; - } - } - - // Generate Unique Schema Name - private static String generateName(final SchemaBuilder builder) { - return format(SCHEMA_NAME_TEMPLATE, Objects.hashCode(builder.build())).replace("-", "_"); - } - - private static SchemaAndValue toSchemaAndValue(final Schema schema, final JsonNode node) { - SchemaAndValue schemaAndValue = new SchemaAndValue(schema, node); - if (schema.isOptional() && node.isNull()) { - return new SchemaAndValue(schema, null); - } - switch (schema.type()) { - case INT8: - case INT16: - case INT32: - case INT64: - case FLOAT32: - case FLOAT64: - schemaAndValue = numberToSchemaAndValue(schema, node); - break; - case BOOLEAN: - schemaAndValue = new SchemaAndValue(schema, node.asBoolean()); - break; - case STRING: - schemaAndValue = stringToSchemaAndValue(schema, node); - break; - case BYTES: - schemaAndValue = new SchemaAndValue(schema, node); - break; - case ARRAY: - schemaAndValue = arrayToSchemaAndValue(schema, node); - break; - case MAP: - schemaAndValue = new SchemaAndValue(schema, node); - break; - case STRUCT: - schemaAndValue = recordToSchemaAndValue(node); - break; - default: - LOGGER.error("Unsupported Schema type: {}", schema.type()); - } - return schemaAndValue; - } - - private static SchemaAndValue stringToSchemaAndValue(final Schema schema, final JsonNode nodeValue) { - String value; - if (nodeValue.isTextual()) { - value = nodeValue.asText(); - } else { - value = nodeValue.toString(); - } - return new SchemaAndValue(schema, value); - } - - private static SchemaAndValue arrayToSchemaAndValue(final Schema schema, final JsonNode nodeValue) { - if (!nodeValue.isArray()) { - LOGGER.error("Unexpected array value for schema {}", schema); - } - List<Object> values = new ArrayList<>(); - nodeValue.forEach(v -> - values.add(toSchemaAndValue(schema.valueSchema(), v).value()) - ); - return new SchemaAndValue(schema, values); - } - - private static SchemaAndValue numberToSchemaAndValue(final Schema schema, final JsonNode nodeValue) { - Object value = null; - if (nodeValue.isNumber()) { - if (nodeValue.isInt()) { - value = nodeValue.intValue(); - } else if (nodeValue.isDouble()) { - value = nodeValue.doubleValue(); - } else if (nodeValue.isLong()) { - value = nodeValue.longValue(); - } - } else { - LOGGER.error("Unexpected value for schema {}", schema); - } - - switch (schema.type()) { - case INT8: - value = convertToByte(schema, value); - break; - case INT16: - value = convertToShort(schema, value); - break; - case INT32: - value = convertToInteger(schema, value); - break; - case INT64: - value = convertToLong(schema, value); - break; - case FLOAT32: - value = convertToFloat(schema, value); - break; - case FLOAT64: - value = convertToDouble(schema, value); - break; - default: - LOGGER.error("Unsupported Schema type: {}", schema.type()); - } - return new SchemaAndValue(schema, value); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThread.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThread.java deleted file mode 100644 index 699b3ad9375d..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThread.java +++ /dev/null @@ -1,338 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.BridgeInternal; -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.implementation.AsyncDocumentClient; -import com.azure.cosmos.implementation.PartitionKeyRange; -import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; -import com.azure.cosmos.implementation.feedranges.FeedRangeInternal; -import com.azure.cosmos.implementation.routing.Range; -import com.azure.cosmos.kafka.connect.implementation.KafkaCosmosExceptionsHelper; -import com.azure.cosmos.models.CosmosContainerProperties; -import com.azure.cosmos.models.SqlParameter; -import com.azure.cosmos.models.SqlQuerySpec; -import org.apache.kafka.connect.source.SourceConnectorContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Scheduler; -import reactor.core.scheduler.Schedulers; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -public class MetadataMonitorThread extends Thread { - private static final Logger LOGGER = LoggerFactory.getLogger(MetadataMonitorThread.class); - - // TODO[Public Preview]: using a threadPool with less threads or single thread - public static final Scheduler CONTAINERS_MONITORING_SCHEDULER = Schedulers.newBoundedElastic( - Schedulers.DEFAULT_BOUNDED_ELASTIC_SIZE, - Schedulers.DEFAULT_BOUNDED_ELASTIC_QUEUESIZE, - "cosmos-source-metadata-monitoring-bounded-elastic", - 60, - true - ); - - private final CosmosSourceContainersConfig sourceContainersConfig; - private final CosmosMetadataConfig metadataConfig; - private final SourceConnectorContext connectorContext; - private final CosmosSourceOffsetStorageReader offsetStorageReader; - private final CosmosAsyncClient cosmosClient; - private final SqlQuerySpec containersQuerySpec; - private final ContainersMetadataTopicPartition containersMetadataTopicPartition; - private final AtomicBoolean isRunning = new AtomicBoolean(true); - - public MetadataMonitorThread( - CosmosSourceContainersConfig containersConfig, - CosmosMetadataConfig metadataConfig, - SourceConnectorContext connectorContext, - CosmosSourceOffsetStorageReader offsetStorageReader, - CosmosAsyncClient cosmosClient) { - - checkNotNull(containersConfig, "Argument 'containersConfig' can not be null"); - checkNotNull(metadataConfig, "Argument 'metadataConfig' can not be null"); - checkNotNull(connectorContext, "Argument 'connectorContext' can not be null"); - checkNotNull(offsetStorageReader, "Argument 'offsetStorageReader' can not be null"); - checkNotNull(cosmosClient, "Argument 'cosmosClient' can not be null"); - - this.sourceContainersConfig = containersConfig; - this.metadataConfig = metadataConfig; - this.connectorContext = connectorContext; - this.offsetStorageReader = offsetStorageReader; - this.cosmosClient = cosmosClient; - this.containersQuerySpec = this.getContainersQuerySpec(); - this.containersMetadataTopicPartition = new ContainersMetadataTopicPartition(containersConfig.getDatabaseName()); - } - - @Override - public void run() { - LOGGER.info("Start containers monitoring task"); - - int containersPollDelayInMs = this.metadataConfig.getMetadataPollDelayInMs(); - if (containersPollDelayInMs >= 0) { - Mono - .delay(Duration.ofMillis(containersPollDelayInMs)) - .flatMap(t -> { - if (this.isRunning.get()) { - LOGGER.trace("ValidateContainersMetadataChange..."); - return shouldRequestTaskReconfiguration(); - } - return Mono.empty(); - }) - .doOnNext(shouldRequestReconfiguration -> { - if (shouldRequestReconfiguration) { - LOGGER.info("Changes detected, requestTaskReconfiguration"); - this.connectorContext.requestTaskReconfiguration(); - } - }) - .onErrorResume(throwable -> { - LOGGER.warn("Containers metadata checking failed. Will retry in next polling cycle", throwable); - // TODO: only allow continue for transient errors, for others raiseError - return Mono.empty(); - }) - .repeat(() -> this.isRunning.get()) - .subscribeOn(CONTAINERS_MONITORING_SCHEDULER) - .subscribe(); - } - - LOGGER.info("Containers monitoring task not started due to negative containers poll delay"); - } - - private Mono<Boolean> shouldRequestTaskReconfiguration() { - // First check any containers to be copied changes - // Container re-created, add or remove will request task reconfiguration - // If there are no changes on the containers, then check for each container any feedRanges change need to request task reconfiguration - if (containersMetadataOffsetExists()) { - return this.getAllContainers() - .flatMap(containersList -> { - if (hasContainersChange(containersList)) { - return Mono.just(true); - } - - return shouldRequestTaskReconfigurationOnFeedRanges(containersList); - }); - } - - // there is no existing containers offset for comparison. - // Could be this is the first time for the connector to start and the metadata task has not been initialized. - // will skip and validate in next cycle. - return Mono.just(false); - } - - public boolean containersMetadataOffsetExists() { - return this.offsetStorageReader.getContainersMetadataOffset(this.sourceContainersConfig.getDatabaseName()) != null; - } - - public Mono<List<CosmosContainerProperties>> getAllContainers() { - return this.cosmosClient - .getDatabase(this.sourceContainersConfig.getDatabaseName()) - .queryContainers(this.containersQuerySpec) - .byPage() - .flatMapIterable(response -> response.getResults()) - .collectList() - .onErrorMap(throwable -> KafkaCosmosExceptionsHelper.convertToConnectException(throwable, "getAllContainers failed.")); - } - - public List<String> getContainerRidsFromOffset() { - ContainersMetadataTopicOffset topicOffset = - this.offsetStorageReader - .getContainersMetadataOffset(this.sourceContainersConfig.getDatabaseName()); - return topicOffset == null ? new ArrayList<>() : topicOffset.getContainerRids(); - } - - private boolean hasContainersChange(List<CosmosContainerProperties> allContainers) { - List<String> containerRidsFromOffset = this.getContainerRidsFromOffset(); - - List<String> containersRidToBeCopied = - allContainers - .stream() - .map(CosmosContainerProperties::getResourceId) - .collect(Collectors.toList()); - - return !(containerRidsFromOffset.size() == containersRidToBeCopied.size() - && containerRidsFromOffset.containsAll(containersRidToBeCopied)); - } - - private Mono<Boolean> shouldRequestTaskReconfigurationOnFeedRanges(List<CosmosContainerProperties> allContainers) { - AtomicBoolean shouldRequestTaskReconfiguration = new AtomicBoolean(false); - AtomicInteger containerIndex = new AtomicInteger(0); - - // loop through containers to check any feedRanges change - return Mono.just(allContainers.get(containerIndex.get())) - .flatMap(containerProperties -> shouldRequestTaskReconfigurationOnFeedRanges(containerProperties)) - .doOnNext(hasChange -> { - shouldRequestTaskReconfiguration.set(hasChange); - containerIndex.incrementAndGet(); - }) - .repeat(() -> !shouldRequestTaskReconfiguration.get() && containerIndex.get() < allContainers.size()) - .then(Mono.defer(() -> Mono.just(shouldRequestTaskReconfiguration.get()))); - } - - private Mono<Boolean> shouldRequestTaskReconfigurationOnFeedRanges(CosmosContainerProperties containerProperties) { - if (feedRangesMetadataOffsetExists(containerProperties)) { - CosmosAsyncContainer container = - this.cosmosClient - .getDatabase(this.sourceContainersConfig.getDatabaseName()) - .getContainer(containerProperties.getId()); - - return container - .getFeedRanges() - .map(feedRanges -> { - return feedRanges - .stream() - .map(feedRange -> FeedRangeInternal.normalizeRange(((FeedRangeEpkImpl) feedRange).getRange())) - .collect(Collectors.toList()); - }) - .flatMap(range -> { - FeedRangesMetadataTopicOffset topicOffset = - this.offsetStorageReader - .getFeedRangesMetadataOffset( - this.sourceContainersConfig.getDatabaseName(), - containerProperties.getResourceId()); - - if (topicOffset == null) { - // the container may have recreated - return Mono.just(true); - } - - List<Range<String>> differences = - topicOffset - .getFeedRanges() - .stream() - .filter(normalizedFeedRange -> !range.contains(normalizedFeedRange)) - .collect(Collectors.toList()); - - if (differences.size() == 0) { - // the feedRanges are exact the same - return Mono.just(false); - } - - // There are feedRanges change, but not all changes need to trigger a reconfiguration - // Merge should not trigger task reconfiguration as we will continue pulling the data from the pre-merge feed ranges - // Split should trigger task reconfiguration for load-balancing - return shouldRequestTaskReconfigurationOnFeedRangeChanges(containerProperties, differences); - }); - } - - // there is no existing feedRanges offset for comparison. - // Could be this is the first time for the connector to start and the metadata task has not been initialized. - // will skip and validate in next cycle. - return Mono.just(false); - } - - private boolean feedRangesMetadataOffsetExists(CosmosContainerProperties containerProperties) { - return this.offsetStorageReader - .getFeedRangesMetadataOffset( - this.sourceContainersConfig.getDatabaseName(), - containerProperties.getResourceId()) != null; - } - - private Mono<Boolean> shouldRequestTaskReconfigurationOnFeedRangeChanges( - CosmosContainerProperties containerProperties, - List<Range<String>> changes) { - if (changes == null || changes.isEmpty()) { - return Mono.just(false); - } - - AtomicBoolean shouldRequestTaskReconfiguration = new AtomicBoolean(false); - AtomicInteger feedRangeIndex = new AtomicInteger(0); - - return Mono.just(changes.get(feedRangeIndex.get())) - .flatMap(feedRangeChanged -> shouldRequestTaskReconfigurationOnFeedRangeChange(containerProperties, feedRangeChanged)) - .doOnNext(shouldReconfig -> { - shouldRequestTaskReconfiguration.compareAndSet(false, shouldReconfig); - feedRangeIndex.incrementAndGet(); - }) - .repeat(() -> (!shouldRequestTaskReconfiguration.get()) && feedRangeIndex.get() < changes.size()) - .then(Mono.defer(() -> Mono.just(shouldRequestTaskReconfiguration.get()))); - } - - private Mono<Boolean> shouldRequestTaskReconfigurationOnFeedRangeChange( - CosmosContainerProperties containerProperties, - Range<String> feedRangeChanged) { - - AsyncDocumentClient asyncDocumentClient = BridgeInternal.getContextClient(this.cosmosClient); - - // find out whether it is a split or merge - // for split, we are going to request a task reconfiguration for load-balancing - // for merge, ignore as we are going to continue consuming base on the current feed range - return asyncDocumentClient - .getPartitionKeyRangeCache() - .tryGetOverlappingRangesAsync( - null, - containerProperties.getResourceId(), - feedRangeChanged, - false, - null - ) - .map(pkRangesValueHolder -> { - List<PartitionKeyRange> matchedPkRanges = - (pkRangesValueHolder == null || pkRangesValueHolder.v == null) ? new ArrayList<>() : pkRangesValueHolder.v; - - if (matchedPkRanges.size() == 0) { - LOGGER.warn( - "FeedRang {} on container {} is gone but we failed to find at least one matching pkRange", - feedRangeChanged, - containerProperties.getResourceId()); - - return true; - } - - if (matchedPkRanges.size() == 1) { - LOGGER.info( - "FeedRange {} is merged into {} on container {}", - feedRangeChanged, - matchedPkRanges.get(0).toRange(), - containerProperties.getResourceId()); - return false; - } - - LOGGER.info( - "FeedRange {} is split into [{}] on container {}", - feedRangeChanged, - matchedPkRanges.stream().map(PartitionKeyRange::toRange).collect(Collectors.toList()), - containerProperties.getResourceId() - ); - return true; - }); - } - - private SqlQuerySpec getContainersQuerySpec() { - boolean includeAllContainers = sourceContainersConfig.isIncludeAllContainers(); - if (includeAllContainers) { - return new SqlQuerySpec("SELECT * FROM c"); - } - - StringBuilder queryStringBuilder = new StringBuilder(); - List<SqlParameter> parameters = new ArrayList<>(); - - queryStringBuilder.append("SELECT * FROM c WHERE c.id IN ( "); - for (int i = 0; i < sourceContainersConfig.getIncludedContainers().size(); i++) { - String idValue = sourceContainersConfig.getIncludedContainers().get(i); - String idParamName = "@param" + i; - - parameters.add(new SqlParameter(idParamName, idValue)); - queryStringBuilder.append(idParamName); - - if (i < sourceContainersConfig.getIncludedContainers().size() - 1) { - queryStringBuilder.append(", "); - } - } - queryStringBuilder.append(" )"); - return new SqlQuerySpec(queryStringBuilder.toString(), parameters); - } - - public void close() { - this.isRunning.set(false); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataTaskUnit.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataTaskUnit.java deleted file mode 100644 index 18a9e593567a..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataTaskUnit.java +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.routing.Range; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import com.fasterxml.jackson.databind.node.ArrayNode; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkArgument; -import static com.azure.cosmos.implementation.guava25.base.Preconditions.checkNotNull; - -@JsonSerialize(using = MetadataTaskUnit.MetadataTaskUnitSerializer.class) -@JsonDeserialize(using = MetadataTaskUnit.MetadataTaskUnitDeserializer.class) -public class MetadataTaskUnit implements ITaskUnit { - private final String databaseName; - private final List<String> containerRids; - private final Map<String, List<Range<String>>> containersEffectiveRangesMap; - private final String topic; - - public MetadataTaskUnit( - String databaseName, - List<String> containerRids, - Map<String, List<Range<String>>> containersEffectiveRangesMap, - String topic) { - - checkArgument(StringUtils.isNotEmpty(databaseName), "Argument 'databaseName' should not be null"); - checkNotNull(containerRids, "Argument 'containerRids' can not be null"); - checkNotNull(containersEffectiveRangesMap, "Argument 'containersEffectiveRangesMap' can not be null"); - checkArgument(StringUtils.isNotEmpty(topic), "Argument 'topic' should not be null"); - - this.databaseName = databaseName; - this.containerRids = containerRids; - this.containersEffectiveRangesMap = containersEffectiveRangesMap; - this.topic = topic; - } - - public String getDatabaseName() { - return databaseName; - } - - public List<String> getContainerRids() { - return containerRids; - } - - public Map<String, List<Range<String>>> getContainersEffectiveRangesMap() { - return containersEffectiveRangesMap; - } - - public String getTopic() { - return topic; - } - - @Override - public String toString() { - return "MetadataTaskUnit{" - + "databaseName='" - + databaseName - + '\'' - + ", containerRids=" - + containerRids - + ", containersEffectiveRangesMap=" - + containersEffectiveRangesMap - + ", topic='" + topic + '\'' - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - MetadataTaskUnit that = (MetadataTaskUnit) o; - return databaseName.equals(that.databaseName) - && containerRids.equals(that.containerRids) - && Objects.equals(containersEffectiveRangesMap, that.containersEffectiveRangesMap) - && topic.equals(that.topic); - } - - @Override - public int hashCode() { - return Objects.hash(databaseName, containerRids, containersEffectiveRangesMap, topic); - } - - public static class MetadataTaskUnitSerializer extends com.fasterxml.jackson.databind.JsonSerializer<MetadataTaskUnit> { - @Override - public void serialize(MetadataTaskUnit metadataTaskUnit, - JsonGenerator writer, - SerializerProvider serializerProvider) throws IOException { - ObjectMapper objectMapper = new ObjectMapper(); - writer.writeStartObject(); - writer.writeStringField("databaseName", metadataTaskUnit.getDatabaseName()); - writer.writeStringField( - "containerRids", - objectMapper.writeValueAsString(metadataTaskUnit.getContainerRids())); - - writer.writeArrayFieldStart("containersEffectiveRangesMap"); - for (String containerRid : metadataTaskUnit.getContainersEffectiveRangesMap().keySet()) { - writer.writeStartObject(); - writer.writeStringField("containerRid", containerRid); - writer.writeStringField( - "effectiveFeedRanges", - objectMapper.writeValueAsString( - metadataTaskUnit. - getContainersEffectiveRangesMap(). - get(containerRid) - .stream() - .map(range -> range.toJson()) - .collect(Collectors.toList()))); - writer.writeEndObject(); - } - writer.writeEndArray(); - - writer.writeStringField("topic", metadataTaskUnit.getTopic()); - writer.writeEndObject(); - } - } - - static class MetadataTaskUnitDeserializer extends StdDeserializer<MetadataTaskUnit> { - MetadataTaskUnitDeserializer() { - super(MetadataTaskUnit.class); - } - - @Override - public MetadataTaskUnit deserialize( - JsonParser jsonParser, - DeserializationContext deserializationContext) throws IOException { - - final JsonNode rootNode = jsonParser.getCodec().readTree(jsonParser); - final ObjectMapper mapper = (ObjectMapper) jsonParser.getCodec(); - - String databaseName = rootNode.get("databaseName").asText(); - List<String> containerRids = mapper.readValue(rootNode.get("containerRids").asText(), new TypeReference<List<String>>() {}); - ArrayNode arrayNode = (ArrayNode) rootNode.get("containersEffectiveRangesMap"); - - Map<String, List<Range<String>>> containersEffectiveRangesMap = new HashMap<>(); - for (JsonNode jsonNode : arrayNode) { - String containerRid = jsonNode.get("containerRid").asText(); - List<Range<String>> effectiveRanges = - mapper - .readValue( - jsonNode.get("effectiveFeedRanges").asText(), - new TypeReference<List<String>>() {}) - .stream().map(rangeJson -> new Range<String>(rangeJson)) - .collect(Collectors.toList()); - containersEffectiveRangesMap.put(containerRid, effectiveRanges); - } - String topic = rootNode.get("topic").asText(); - - return new MetadataTaskUnit(databaseName, containerRids, containersEffectiveRangesMap, topic); - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/module-info.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/module-info.java index cfea2ab8a321..0557d9e6bc83 100644 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/module-info.java +++ b/sdk/cosmos/azure-cosmos-kafka-connect/src/main/java/module-info.java @@ -6,11 +6,10 @@ requires transitive com.azure.cosmos; requires kafka.clients; - requires connect.api; - requires json.path; // public API surface area exports com.azure.cosmos.kafka.connect; + uses com.azure.core.util.tracing.Tracer; } diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/connectorPlugins/build.ps1 b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/connectorPlugins/build.ps1 deleted file mode 100644 index 600429c8488a..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/connectorPlugins/build.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env pwsh -$ErrorActionPreference='Stop' -cd $PSScriptRoot - -Write-Host "Deleting prior Cosmos DB connectors..." -rm -rf "$PSScriptRoot/src/test/connectorPlugins/connectors" -New-Item -Path "$PSScriptRoot/src/test/connectorPlugins" -ItemType "directory" -Name "connectors" -Force | Out-Null - -Write-Host "Rebuilding Cosmos DB connectors..." -mvn clean package -DskipTests -Dmaven.javadoc.skip -copy target\*-jar-with-dependencies.jar $PSScriptRoot/src/test/connectorPlugins/connectors -cd $PSScriptRoot/src/test/connectorPlugins - -Write-Host "Adding custom Insert UUID SMT" -cd $PSScriptRoot/src/test/connectorPlugins/connectors -git clone https://github.com/confluentinc/kafka-connect-insert-uuid.git insertuuid -q && cd insertuuid -mvn clean package -DskipTests=true -copy target\*.jar $PSScriptRoot/src/test/connectorPlugins/connectors -rm -rf "$PSScriptRoot/src/test/connectorPlugins/connectors/insertuuid" -cd $PSScriptRoot \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/connectorPlugins/build.sh b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/connectorPlugins/build.sh deleted file mode 100755 index bda724c262d4..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/connectorPlugins/build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -echo "Deleting prior Cosmos DB connectors..." -rm -rf src/test/connectorPlugins/connectors -mkdir src/test/connectorPlugins/connectors - -echo "Rebuilding Cosmos DB connectors..." -mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true -cp target/*-jar-with-dependencies.jar src/test/connectorPlugins/connectors -cd src/test/connectorPlugins - -echo "Adding custom Insert UUID SMT" -cd connectors -git clone https://github.com/confluentinc/kafka-connect-insert-uuid.git insertuuid -q && cd insertuuid -mvn clean package -DskipTests=true -cp target/*.jar ../ -cd .. && rm -rf insertuuid -cd ../ \ No newline at end of file diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnectorTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnectorTest.java deleted file mode 100644 index cc332096b5e1..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSinkConnectorTest.java +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.implementation.Strings; -import com.azure.cosmos.implementation.TestConfigurations; -import com.azure.cosmos.kafka.connect.implementation.sink.CosmosSinkTask; -import com.azure.cosmos.kafka.connect.implementation.sink.IdStrategies; -import com.azure.cosmos.kafka.connect.implementation.sink.ItemWriteStrategy; -import org.apache.kafka.common.config.Config; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigValue; -import org.testng.annotations.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -import static com.azure.cosmos.kafka.connect.CosmosDBSinkConnectorTest.SinkConfigs.ALL_VALID_CONFIGS; -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.testng.Assert.assertEquals; - -public class CosmosDBSinkConnectorTest extends KafkaCosmosTestSuiteBase { - @Test(groups = "unit") - public void taskClass() { - CosmosDBSinkConnector sinkConnector = new CosmosDBSinkConnector(); - assertEquals(sinkConnector.taskClass(), CosmosSinkTask.class); - } - - @Test(groups = "unit") - public void config() { - CosmosDBSinkConnector sinkConnector = new CosmosDBSinkConnector(); - ConfigDef configDef = sinkConnector.config(); - Map<String, ConfigDef.ConfigKey> configs = configDef.configKeys(); - List<KafkaCosmosConfigEntry<?>> allValidConfigs = ALL_VALID_CONFIGS; - - for (KafkaCosmosConfigEntry<?> sinkConfigEntry : allValidConfigs) { - assertThat(configs.containsKey(sinkConfigEntry.getName())).isTrue(); - - configs.containsKey(sinkConfigEntry.getName()); - if (sinkConfigEntry.isOptional()) { - assertThat(configs.get(sinkConfigEntry.getName()).defaultValue).isEqualTo(sinkConfigEntry.getDefaultValue()); - } else { - assertThat(configs.get(sinkConfigEntry.getName()).defaultValue).isEqualTo(ConfigDef.NO_DEFAULT_VALUE); - } - } - } - - @Test(groups = "unit") - public void requiredConfig() { - Config config = new CosmosDBSinkConnector().validate(Collections.emptyMap()); - Map<String, List<String>> errorMessages = config.configValues().stream() - .collect(Collectors.toMap(ConfigValue::name, ConfigValue::errorMessages)); - assertThat(errorMessages.get("kafka.connect.cosmos.accountEndpoint").size()).isGreaterThan(0); - assertThat(errorMessages.get("kafka.connect.cosmos.accountKey").size()).isGreaterThan(0); - assertThat(errorMessages.get("kafka.connect.cosmos.sink.database.name").size()).isGreaterThan(0); - assertThat(errorMessages.get("kafka.connect.cosmos.sink.containers.topicMap").size()).isGreaterThan(0); - } - - @Test(groups = "unit") - public void taskConfigs() { - CosmosDBSinkConnector sinkConnector = new CosmosDBSinkConnector(); - - Map<String, String> sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", singlePartitionContainerName + "#" + singlePartitionContainerName); - sinkConnector.start(sinkConfigMap); - - int maxTask = 2; - List<Map<String, String>> taskConfigs = sinkConnector.taskConfigs(maxTask); - assertThat(taskConfigs.size()).isEqualTo(maxTask); - - for (Map<String, String> taskConfig : taskConfigs) { - assertThat(taskConfig.get("kafka.connect.cosmos.accountEndpoint")).isEqualTo(TestConfigurations.HOST); - assertThat(taskConfig.get("kafka.connect.cosmos.accountKey")).isEqualTo(TestConfigurations.MASTER_KEY); - assertThat(taskConfig.get("kafka.connect.cosmos.sink.database.name")).isEqualTo(databaseName); - assertThat(taskConfig.get("kafka.connect.cosmos.sink.containers.topicMap")) - .isEqualTo(singlePartitionContainerName + "#" + singlePartitionContainerName); - } - } - - @Test(groups = "unit") - public void misFormattedConfig() { - CosmosDBSinkConnector sinkConnector = new CosmosDBSinkConnector(); - Map<String, String> sinkConfigMap = this.getValidSinkConfig(); - - String topicMapConfigName = "kafka.connect.cosmos.sink.containers.topicMap"; - sinkConfigMap.put(topicMapConfigName, UUID.randomUUID().toString()); - - Config validatedConfig = sinkConnector.validate(sinkConfigMap); - ConfigValue configValue = - validatedConfig - .configValues() - .stream() - .filter(config -> config.name().equalsIgnoreCase(topicMapConfigName)) - .findFirst() - .get(); - - assertThat(configValue.errorMessages()).isNotNull(); - assertThat( - configValue - .errorMessages() - .get(0) - .contains( - "The topic-container map should be a comma-delimited list of Kafka topic to Cosmos containers." + - " Each mapping should be a pair of Kafka topic and Cosmos container separated by '#'." + - " For example: topic1#con1,topic2#con2.")) - .isTrue(); - - // TODO[Public Preview]: add other config validations - } - - private Map<String, String> getValidSinkConfig() { - Map<String, String> sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", singlePartitionContainerName + "#" + singlePartitionContainerName); - - return sinkConfigMap; - } - - public static class SinkConfigs { - public static final List<KafkaCosmosConfigEntry<?>> ALL_VALID_CONFIGS = Arrays.asList( - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.accountEndpoint", null, false), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.accountKey", null, false), - new KafkaCosmosConfigEntry<Boolean>("kafka.connect.cosmos.useGatewayMode", false, true), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.preferredRegionsList", Strings.Emtpy, true), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.applicationName", Strings.Emtpy, true), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.sink.errors.tolerance", "None", true), - new KafkaCosmosConfigEntry<Boolean>("kafka.connect.cosmos.sink.bulk.enabled", true, true), - new KafkaCosmosConfigEntry<Integer>("kafka.connect.cosmos.sink.bulk.maxConcurrentCosmosPartitions", -1, true), - new KafkaCosmosConfigEntry<Integer>("kafka.connect.cosmos.sink.bulk.initialBatchSize", 1, true), - new KafkaCosmosConfigEntry<String>( - "kafka.connect.cosmos.sink.write.strategy", - ItemWriteStrategy.ITEM_OVERWRITE.getName(), - true), - new KafkaCosmosConfigEntry<Integer>("kafka.connect.cosmos.sink.maxRetryCount", 10, true), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.sink.database.name", null, false), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.sink.containers.topicMap", null, false), - new KafkaCosmosConfigEntry<String>( - "kafka.connect.cosmos.sink.id.strategy", - IdStrategies.PROVIDED_IN_VALUE_STRATEGY.getName(), - true) - ); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnectorTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnectorTest.java deleted file mode 100644 index a69da0cc6bf4..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDBSourceConnectorTest.java +++ /dev/null @@ -1,611 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.implementation.ImplementationBridgeHelpers; -import com.azure.cosmos.implementation.Strings; -import com.azure.cosmos.implementation.TestConfigurations; -import com.azure.cosmos.implementation.Utils; -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.changefeed.common.ChangeFeedMode; -import com.azure.cosmos.implementation.changefeed.common.ChangeFeedStartFromInternal; -import com.azure.cosmos.implementation.changefeed.common.ChangeFeedState; -import com.azure.cosmos.implementation.changefeed.common.ChangeFeedStateV1; -import com.azure.cosmos.implementation.feedranges.FeedRangeContinuation; -import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; -import com.azure.cosmos.implementation.query.CompositeContinuationToken; -import com.azure.cosmos.implementation.routing.Range; -import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosChangeFeedModes; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosChangeFeedStartFromModes; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceConfig; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceOffsetStorageReader; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceTask; -import com.azure.cosmos.kafka.connect.implementation.source.FeedRangeContinuationTopicOffset; -import com.azure.cosmos.kafka.connect.implementation.source.FeedRangeContinuationTopicPartition; -import com.azure.cosmos.kafka.connect.implementation.source.FeedRangeTaskUnit; -import com.azure.cosmos.kafka.connect.implementation.source.FeedRangesMetadataTopicOffset; -import com.azure.cosmos.kafka.connect.implementation.source.FeedRangesMetadataTopicPartition; -import com.azure.cosmos.kafka.connect.implementation.source.MetadataMonitorThread; -import com.azure.cosmos.kafka.connect.implementation.source.MetadataTaskUnit; -import com.azure.cosmos.models.CosmosContainerProperties; -import com.azure.cosmos.models.FeedRange; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import org.apache.kafka.common.config.Config; -import org.apache.kafka.common.config.ConfigDef; -import org.apache.kafka.common.config.ConfigValue; -import org.apache.kafka.connect.source.SourceConnectorContext; -import org.mockito.Mockito; -import org.testng.annotations.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -import static com.azure.cosmos.kafka.connect.CosmosDBSourceConnectorTest.SourceConfigs.ALL_VALID_CONFIGS; -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.testng.Assert.assertEquals; - -@Test -public class CosmosDBSourceConnectorTest extends KafkaCosmosTestSuiteBase { - @Test(groups = "unit") - public void taskClass() { - CosmosDBSourceConnector sourceConnector = new CosmosDBSourceConnector(); - assertEquals(sourceConnector.taskClass(), CosmosSourceTask.class); - } - - @Test(groups = "unit") - public void config() { - CosmosDBSourceConnector sourceConnector = new CosmosDBSourceConnector(); - ConfigDef configDef = sourceConnector.config(); - Map<String, ConfigDef.ConfigKey> configs = configDef.configKeys(); - List<KafkaCosmosConfigEntry<?>> allValidConfigs = ALL_VALID_CONFIGS; - - for (KafkaCosmosConfigEntry<?> sourceConfigEntry : allValidConfigs) { - assertThat(configs.containsKey(sourceConfigEntry.getName())).isTrue(); - - configs.containsKey(sourceConfigEntry.getName()); - if (sourceConfigEntry.isOptional()) { - assertThat(configs.get(sourceConfigEntry.getName()).defaultValue).isEqualTo(sourceConfigEntry.getDefaultValue()); - } else { - assertThat(configs.get(sourceConfigEntry.getName()).defaultValue).isEqualTo(ConfigDef.NO_DEFAULT_VALUE); - } - } - } - - @Test(groups = "{ kafka }", timeOut = TIMEOUT) - public void getTaskConfigsWithoutPersistedOffset() throws JsonProcessingException { - CosmosDBSourceConnector sourceConnector = new CosmosDBSourceConnector(); - try { - Map<String, Object> sourceConfigMap = new HashMap<>(); - sourceConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sourceConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sourceConfigMap.put("kafka.connect.cosmos.source.database.name", databaseName); - List<String> containersIncludedList = Arrays.asList( - singlePartitionContainerName, - multiPartitionContainerName - ); - sourceConfigMap.put("kafka.connect.cosmos.source.containers.includedList", containersIncludedList.toString()); - - String singlePartitionContainerTopicName = singlePartitionContainerName + "topic"; - List<String> containerTopicMapList = Arrays.asList(singlePartitionContainerTopicName + "#" + singlePartitionContainerName); - sourceConfigMap.put("kafka.connect.cosmos.source.containers.topicMap", containerTopicMapList.toString()); - - // setup the internal state - this.setupDefaultConnectorInternalStates(sourceConnector, sourceConfigMap); - CosmosAsyncClient cosmosAsyncClient = KafkaCosmosReflectionUtils.getCosmosClient(sourceConnector); - - int maxTask = 2; - List<Map<String, String>> taskConfigs = sourceConnector.taskConfigs(maxTask); - assertThat(taskConfigs.size()).isEqualTo(maxTask); - - // construct expected feed range task units - CosmosContainerProperties singlePartitionContainer = getSinglePartitionContainer(cosmosAsyncClient); - List<FeedRangeTaskUnit> singlePartitionContainerFeedRangeTasks = - getFeedRangeTaskUnits( - cosmosAsyncClient, - databaseName, - singlePartitionContainer, - null, - singlePartitionContainerTopicName); - assertThat(singlePartitionContainerFeedRangeTasks.size()).isEqualTo(1); - - CosmosContainerProperties multiPartitionContainer = getMultiPartitionContainer(cosmosAsyncClient); - List<FeedRangeTaskUnit> multiPartitionContainerFeedRangeTasks = - getFeedRangeTaskUnits( - cosmosAsyncClient, - databaseName, - multiPartitionContainer, - null, - multiPartitionContainer.getId()); - assertThat(multiPartitionContainerFeedRangeTasks.size()).isGreaterThan(1); - - List<List<FeedRangeTaskUnit>> expectedTaskUnits = new ArrayList<>(); - for (int i = 0; i < maxTask; i++) { - expectedTaskUnits.add(new ArrayList<>()); - } - - expectedTaskUnits.get(0).add(singlePartitionContainerFeedRangeTasks.get(0)); - for (int i = 0; i < multiPartitionContainerFeedRangeTasks.size(); i++) { - int index = ( i + 1) % 2; - expectedTaskUnits.get(index).add(multiPartitionContainerFeedRangeTasks.get(i)); - } - - validateFeedRangeTasks(expectedTaskUnits, taskConfigs); - - MetadataTaskUnit expectedMetadataTaskUnit = - getMetadataTaskUnit( - cosmosAsyncClient, - databaseName, - Arrays.asList(singlePartitionContainer, multiPartitionContainer)); - validateMetadataTask(expectedMetadataTaskUnit, taskConfigs.get(1)); - } finally { - sourceConnector.stop(); - } - } - - @Test(groups = "{ kafka }", timeOut = TIMEOUT) - public void getTaskConfigsAfterSplit() throws JsonProcessingException { - // This test is to simulate after a split happen, the task resume with persisted offset - CosmosDBSourceConnector sourceConnector = new CosmosDBSourceConnector(); - - try { - Map<String, Object> sourceConfigMap = new HashMap<>(); - sourceConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sourceConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sourceConfigMap.put("kafka.connect.cosmos.source.database.name", databaseName); - List<String> containersIncludedList = Arrays.asList(multiPartitionContainerName); - sourceConfigMap.put("kafka.connect.cosmos.source.containers.includedList", containersIncludedList.toString()); - - // setup the internal state - this.setupDefaultConnectorInternalStates(sourceConnector, sourceConfigMap); - - // override the storage reader with initial offset - CosmosAsyncClient cosmosAsyncClient = KafkaCosmosReflectionUtils.getCosmosClient(sourceConnector); - CosmosSourceOffsetStorageReader sourceOffsetStorageReader = KafkaCosmosReflectionUtils.getSourceOffsetStorageReader(sourceConnector); - InMemoryStorageReader inMemoryStorageReader = - (InMemoryStorageReader) KafkaCosmosReflectionUtils.getOffsetStorageReader(sourceOffsetStorageReader); - - CosmosContainerProperties multiPartitionContainer = getMultiPartitionContainer(cosmosAsyncClient); - - // constructing feed range continuation offset - FeedRangeContinuationTopicPartition feedRangeContinuationTopicPartition = - new FeedRangeContinuationTopicPartition( - databaseName, - multiPartitionContainer.getResourceId(), - FeedRangeEpkImpl.forFullRange().getRange()); - - String initialContinuationState = new ChangeFeedStateV1( - multiPartitionContainer.getResourceId(), - FeedRangeEpkImpl.forFullRange(), - ChangeFeedMode.INCREMENTAL, - ChangeFeedStartFromInternal.createFromBeginning(), - FeedRangeContinuation.create( - multiPartitionContainer.getResourceId(), - FeedRangeEpkImpl.forFullRange(), - Arrays.asList(new CompositeContinuationToken("1", FeedRangeEpkImpl.forFullRange().getRange())))).toString(); - - FeedRangeContinuationTopicOffset feedRangeContinuationTopicOffset = - new FeedRangeContinuationTopicOffset(initialContinuationState, "1"); // using the same itemLsn as in the continuationToken - Map<Map<String, Object>, Map<String, Object>> initialOffsetMap = new HashMap<>(); - initialOffsetMap.put( - FeedRangeContinuationTopicPartition.toMap(feedRangeContinuationTopicPartition), - FeedRangeContinuationTopicOffset.toMap(feedRangeContinuationTopicOffset)); - - // constructing feedRange metadata offset - FeedRangesMetadataTopicPartition feedRangesMetadataTopicPartition = - new FeedRangesMetadataTopicPartition(databaseName, multiPartitionContainer.getResourceId()); - FeedRangesMetadataTopicOffset feedRangesMetadataTopicOffset = - new FeedRangesMetadataTopicOffset(Arrays.asList(FeedRangeEpkImpl.forFullRange().getRange())); - initialOffsetMap.put( - FeedRangesMetadataTopicPartition.toMap(feedRangesMetadataTopicPartition), - FeedRangesMetadataTopicOffset.toMap(feedRangesMetadataTopicOffset)); - - inMemoryStorageReader.populateOffset(initialOffsetMap); - - int maxTask = 2; - List<Map<String, String>> taskConfigs = sourceConnector.taskConfigs(maxTask); - assertThat(taskConfigs.size()).isEqualTo(maxTask); - - // construct expected feed range task units - List<FeedRangeTaskUnit> multiPartitionContainerFeedRangeTasks = - getFeedRangeTaskUnits( - cosmosAsyncClient, - databaseName, - multiPartitionContainer, - initialContinuationState, - multiPartitionContainer.getId()); - assertThat(multiPartitionContainerFeedRangeTasks.size()).isGreaterThan(1); - - List<List<FeedRangeTaskUnit>> expectedTaskUnits = new ArrayList<>(); - for (int i = 0; i < maxTask; i++) { - expectedTaskUnits.add(new ArrayList<>()); - } - - for (int i = 0; i < multiPartitionContainerFeedRangeTasks.size(); i++) { - expectedTaskUnits.get( i % 2).add(multiPartitionContainerFeedRangeTasks.get(i)); - } - - validateFeedRangeTasks(expectedTaskUnits, taskConfigs); - - MetadataTaskUnit expectedMetadataTaskUnit = - getMetadataTaskUnit( - cosmosAsyncClient, - databaseName, - Arrays.asList(multiPartitionContainer)); - validateMetadataTask(expectedMetadataTaskUnit, taskConfigs.get(1)); - } finally { - sourceConnector.stop(); - } - } - - @Test(groups = "{ kafka }", timeOut = TIMEOUT) - public void getTaskConfigsAfterMerge() throws JsonProcessingException { - // This test is to simulate after a merge happen, the task resume with previous feedRanges - CosmosDBSourceConnector sourceConnector = new CosmosDBSourceConnector(); - - try { - - Map<String, Object> sourceConfigMap = new HashMap<>(); - sourceConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sourceConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sourceConfigMap.put("kafka.connect.cosmos.source.database.name", databaseName); - List<String> containersIncludedList = Arrays.asList(singlePartitionContainerName); - sourceConfigMap.put("kafka.connect.cosmos.source.containers.includedList", containersIncludedList.toString()); - - // setup the internal state - this.setupDefaultConnectorInternalStates(sourceConnector, sourceConfigMap); - - // override the storage reader with initial offset - CosmosAsyncClient cosmosAsyncClient = KafkaCosmosReflectionUtils.getCosmosClient(sourceConnector); - CosmosSourceOffsetStorageReader sourceOffsetStorageReader = KafkaCosmosReflectionUtils.getSourceOffsetStorageReader(sourceConnector); - InMemoryStorageReader inMemoryStorageReader = - (InMemoryStorageReader) KafkaCosmosReflectionUtils.getOffsetStorageReader(sourceOffsetStorageReader); - - CosmosContainerProperties singlePartitionContainer = getSinglePartitionContainer(cosmosAsyncClient); - - // constructing feed range continuation offset - List<FeedRangeEpkImpl> childRanges = - ImplementationBridgeHelpers - .CosmosAsyncContainerHelper - .getCosmosAsyncContainerAccessor() - .trySplitFeedRange( - cosmosAsyncClient.getDatabase(databaseName).getContainer(singlePartitionContainer.getId()), - FeedRange.forFullRange(), - 2) - .block(); - - Map<Map<String, Object>, Map<String, Object>> initialOffsetMap = new HashMap<>(); - List<FeedRangeTaskUnit> singlePartitionFeedRangeTaskUnits = new ArrayList<>(); - - for (FeedRangeEpkImpl childRange : childRanges) { - FeedRangeContinuationTopicPartition feedRangeContinuationTopicPartition = - new FeedRangeContinuationTopicPartition( - databaseName, - singlePartitionContainer.getResourceId(), - childRange.getRange()); - - String childRangeContinuationState = new ChangeFeedStateV1( - singlePartitionContainer.getResourceId(), - childRange, - ChangeFeedMode.INCREMENTAL, - ChangeFeedStartFromInternal.createFromBeginning(), - FeedRangeContinuation.create( - singlePartitionContainer.getResourceId(), - childRange, - Arrays.asList(new CompositeContinuationToken("1", childRange.getRange())))).toString(); - - FeedRangeContinuationTopicOffset feedRangeContinuationTopicOffset = - new FeedRangeContinuationTopicOffset(childRangeContinuationState, "1"); - - initialOffsetMap.put( - FeedRangeContinuationTopicPartition.toMap(feedRangeContinuationTopicPartition), - FeedRangeContinuationTopicOffset.toMap(feedRangeContinuationTopicOffset)); - - singlePartitionFeedRangeTaskUnits.add( - new FeedRangeTaskUnit( - databaseName, - singlePartitionContainer.getId(), - singlePartitionContainer.getResourceId(), - childRange.getRange(), - childRangeContinuationState, - singlePartitionContainer.getId())); - } - - // constructing feedRange metadata offset - FeedRangesMetadataTopicPartition feedRangesMetadataTopicPartition = - new FeedRangesMetadataTopicPartition(databaseName, singlePartitionContainer.getResourceId()); - FeedRangesMetadataTopicOffset feedRangesMetadataTopicOffset = - new FeedRangesMetadataTopicOffset( - childRanges - .stream() - .map(childRange -> childRange.getRange()) - .collect(Collectors.toList())); - - initialOffsetMap.put( - FeedRangesMetadataTopicPartition.toMap(feedRangesMetadataTopicPartition), - FeedRangesMetadataTopicOffset.toMap(feedRangesMetadataTopicOffset)); - - inMemoryStorageReader.populateOffset(initialOffsetMap); - - int maxTask = 2; - List<Map<String, String>> taskConfigs = sourceConnector.taskConfigs(maxTask); - assertThat(taskConfigs.size()).isEqualTo(maxTask); - - // construct expected feed range task units - assertThat(singlePartitionFeedRangeTaskUnits.size()).isEqualTo(2); - - List<List<FeedRangeTaskUnit>> expectedTaskUnits = new ArrayList<>(); - for (int i = 0; i < maxTask; i++) { - expectedTaskUnits.add(new ArrayList<>()); - } - - for (int i = 0; i < singlePartitionFeedRangeTaskUnits.size(); i++) { - expectedTaskUnits.get( i % 2).add(singlePartitionFeedRangeTaskUnits.get(i)); - } - - validateFeedRangeTasks(expectedTaskUnits, taskConfigs); - - Map<String, List<Range<String>>> containersEffectiveRangesMap = new HashMap<>(); - containersEffectiveRangesMap.put( - singlePartitionContainer.getResourceId(), - childRanges.stream().map(FeedRangeEpkImpl::getRange).collect(Collectors.toList())); - - MetadataTaskUnit expectedMetadataTaskUnit = - new MetadataTaskUnit( - databaseName, - Arrays.asList(singlePartitionContainer.getResourceId()), - containersEffectiveRangesMap, - "_cosmos.metadata.topic" - ); - validateMetadataTask(expectedMetadataTaskUnit, taskConfigs.get(1)); - } finally { - sourceConnector.stop(); - } - } - - @Test(groups = "unit") - public void missingRequiredConfig() { - - List<KafkaCosmosConfigEntry<?>> requiredConfigs = - ALL_VALID_CONFIGS - .stream() - .filter(sourceConfigEntry -> !sourceConfigEntry.isOptional()) - .collect(Collectors.toList()); - - assertThat(requiredConfigs.size()).isGreaterThan(1); - CosmosDBSourceConnector sourceConnector = new CosmosDBSourceConnector(); - for (KafkaCosmosConfigEntry<?> configEntry : requiredConfigs) { - - Map<String, String> sourceConfigMap = this.getValidSourceConfig(); - sourceConfigMap.remove(configEntry.getName()); - Config validatedConfig = sourceConnector.validate(sourceConfigMap); - ConfigValue configValue = - validatedConfig - .configValues() - .stream() - .filter(config -> config.name().equalsIgnoreCase(configEntry.getName())) - .findFirst() - .get(); - - assertThat(configValue.errorMessages()).isNotNull(); - assertThat(configValue.errorMessages().size()).isGreaterThanOrEqualTo(1); - } - } - - @Test(groups = "unit") - public void misFormattedConfig() { - CosmosDBSourceConnector sourceConnector = new CosmosDBSourceConnector(); - Map<String, String> sourceConfigMap = this.getValidSourceConfig(); - - String topicMapConfigName = "kafka.connect.cosmos.source.containers.topicMap"; - sourceConfigMap.put(topicMapConfigName, UUID.randomUUID().toString()); - - Config validatedConfig = sourceConnector.validate(sourceConfigMap); - ConfigValue configValue = - validatedConfig - .configValues() - .stream() - .filter(config -> config.name().equalsIgnoreCase(topicMapConfigName)) - .findFirst() - .get(); - - assertThat(configValue.errorMessages()).isNotNull(); - assertThat( - configValue - .errorMessages() - .get(0) - .contains( - "The topic-container map should be a comma-delimited list of Kafka topic to Cosmos containers." + - " Each mapping should be a pair of Kafka topic and Cosmos container separated by '#'." + - " For example: topic1#con1,topic2#con2.")) - .isTrue(); - - // TODO[Public Preview]: add other config validations - } - - private Map<String, String> getValidSourceConfig() { - Map<String, String> sourceConfigMap = new HashMap<>(); - sourceConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sourceConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sourceConfigMap.put("kafka.connect.cosmos.source.database.name", databaseName); - List<String> containersIncludedList = Arrays.asList(singlePartitionContainerName); - sourceConfigMap.put("kafka.connect.cosmos.source.containers.includedList", containersIncludedList.toString()); - - return sourceConfigMap; - } - - private void setupDefaultConnectorInternalStates(CosmosDBSourceConnector sourceConnector, Map<String, Object> sourceConfigMap) { - CosmosSourceConfig cosmosSourceConfig = new CosmosSourceConfig(sourceConfigMap); - KafkaCosmosReflectionUtils.setCosmosSourceConfig(sourceConnector, cosmosSourceConfig); - - CosmosAsyncClient cosmosAsyncClient = CosmosClientStore.getCosmosClient(cosmosSourceConfig.getAccountConfig()); - KafkaCosmosReflectionUtils.setCosmosClient(sourceConnector, cosmosAsyncClient); - - InMemoryStorageReader inMemoryStorageReader = new InMemoryStorageReader(); - CosmosSourceOffsetStorageReader storageReader = new CosmosSourceOffsetStorageReader(inMemoryStorageReader); - KafkaCosmosReflectionUtils.setOffsetStorageReader(sourceConnector, storageReader); - - SourceConnectorContext connectorContext = Mockito.mock(SourceConnectorContext.class); - MetadataMonitorThread monitorThread = new MetadataMonitorThread( - cosmosSourceConfig.getContainersConfig(), - cosmosSourceConfig.getMetadataConfig(), - connectorContext, - storageReader, - cosmosAsyncClient); - - KafkaCosmosReflectionUtils.setMetadataMonitorThread(sourceConnector, monitorThread); - } - - private List<FeedRangeTaskUnit> getFeedRangeTaskUnits( - CosmosAsyncClient cosmosClient, - String databaseName, - CosmosContainerProperties containerProperties, - String continuationState, - String topicName) { - - List<FeedRange> feedRanges = - cosmosClient - .getDatabase(databaseName) - .getContainer(containerProperties.getId()) - .getFeedRanges() - .block(); - - return feedRanges - .stream() - .map(feedRange -> { - String feedRangeContinuationState = null; - if (StringUtils.isNotEmpty(continuationState)) { - ChangeFeedState changeFeedState = ChangeFeedStateV1.fromString(continuationState); - feedRangeContinuationState = - new ChangeFeedStateV1( - changeFeedState.getContainerRid(), - FeedRangeEpkImpl.forFullRange(), - ChangeFeedMode.INCREMENTAL, - ChangeFeedStartFromInternal.createFromBeginning(), - FeedRangeContinuation.create( - changeFeedState.getContainerRid(), - (FeedRangeEpkImpl)feedRange, - Arrays.asList( - new CompositeContinuationToken( - changeFeedState.getContinuation().getCurrentContinuationToken().getToken(), - ((FeedRangeEpkImpl)feedRange).getRange())))).toString(); - } - - return new FeedRangeTaskUnit( - databaseName, - containerProperties.getId(), - containerProperties.getResourceId(), - ((FeedRangeEpkImpl)feedRange).getRange(), - feedRangeContinuationState, - topicName); - }) - .collect(Collectors.toList()); - } - - private MetadataTaskUnit getMetadataTaskUnit( - CosmosAsyncClient cosmosAsyncClient, - String databaseName, - List<CosmosContainerProperties> containers) { - - Map<String, List<Range<String>>> containersEffectiveRangesMap = new HashMap<>(); - for (CosmosContainerProperties containerProperties : containers) { - List<FeedRange> feedRanges = - cosmosAsyncClient - .getDatabase(databaseName) - .getContainer(containerProperties.getId()) - .getFeedRanges() - .block(); - - containersEffectiveRangesMap.put( - containerProperties.getResourceId(), - feedRanges - .stream() - .map(feedRange -> ((FeedRangeEpkImpl)feedRange).getRange()) - .collect(Collectors.toList())); - } - - return new MetadataTaskUnit( - databaseName, - containers.stream().map(CosmosContainerProperties::getResourceId).collect(Collectors.toList()), - containersEffectiveRangesMap, - "_cosmos.metadata.topic" - ); - } - - private void validateFeedRangeTasks( - List<List<FeedRangeTaskUnit>> feedRangeTaskUnits, - List<Map<String, String>> taskConfig) throws JsonProcessingException { - - String taskUnitsKey = "kafka.connect.cosmos.source.task.feedRangeTaskUnits"; - for (int i = 0; i< feedRangeTaskUnits.size(); i++) { - List<FeedRangeTaskUnit> expectedTaskUnits = feedRangeTaskUnits.get(i); - assertThat(taskConfig.get(i).containsKey(taskUnitsKey)).isTrue(); - List<FeedRangeTaskUnit> taskUnitsFromTaskConfig = - Utils - .getSimpleObjectMapper() - .readValue(taskConfig.get(i).get(taskUnitsKey), new TypeReference<List<String>>() {}) - .stream() - .map(taskUnitString -> { - try { - return Utils.getSimpleObjectMapper().readValue(taskUnitString, FeedRangeTaskUnit.class); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toList()); - - assertThat(expectedTaskUnits.size()).isEqualTo(taskUnitsFromTaskConfig.size()); - assertThat(expectedTaskUnits.containsAll(taskUnitsFromTaskConfig)).isTrue(); - } - } - - private void validateMetadataTask( - MetadataTaskUnit expectedMetadataTaskUnit, - Map<String, String> taskConfig) throws JsonProcessingException { - - String taskUnitKey = "kafka.connect.cosmos.source.task.metadataTaskUnit"; - assertThat(taskConfig.containsKey(taskUnitKey)); - MetadataTaskUnit metadataTaskUnitFromTaskConfig = - Utils.getSimpleObjectMapper().readValue(taskConfig.get(taskUnitKey), MetadataTaskUnit.class); - - assertThat(expectedMetadataTaskUnit).isEqualTo(metadataTaskUnitFromTaskConfig); - } - - public static class SourceConfigs { - public static final List<KafkaCosmosConfigEntry<?>> ALL_VALID_CONFIGS = Arrays.asList( - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.accountEndpoint", null, false), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.accountKey", null, false), - new KafkaCosmosConfigEntry<Boolean>("kafka.connect.cosmos.useGatewayMode", false, true), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.preferredRegionsList", Strings.Emtpy, true), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.applicationName", Strings.Emtpy, true), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.source.database.name", null, false), - new KafkaCosmosConfigEntry<Boolean>("kafka.connect.cosmos.source.containers.includeAll", false, true), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.source.containers.includedList", Strings.Emtpy, true), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.source.containers.topicMap", Strings.Emtpy, true), - new KafkaCosmosConfigEntry<String>( - "kafka.connect.cosmos.source.changeFeed.startFrom", - CosmosChangeFeedStartFromModes.BEGINNING.getName(), - true), - new KafkaCosmosConfigEntry<String>( - "kafka.connect.cosmos.source.changeFeed.mode", - CosmosChangeFeedModes.LATEST_VERSION.getName(), - true), - new KafkaCosmosConfigEntry<Integer>("kafka.connect.cosmos.source.changeFeed.maxItemCountHint", 1000, true), - new KafkaCosmosConfigEntry<Integer>("kafka.connect.cosmos.source.metadata.poll.delay.ms", 5 * 60 * 1000, true), - new KafkaCosmosConfigEntry<String>( - "kafka.connect.cosmos.source.metadata.storage.topic", - "_cosmos.metadata.topic", - true), - new KafkaCosmosConfigEntry<Boolean>("kafka.connect.cosmos.source.messageKey.enabled", true, true), - new KafkaCosmosConfigEntry<String>("kafka.connect.cosmos.source.messageKey.field", "id", true) - ); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSinkConnectorITest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSinkConnectorITest.java deleted file mode 100644 index 9a0714e9f5d0..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSinkConnectorITest.java +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.implementation.Utils; -import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.kafka.connect.implementation.sink.CosmosSinkConfig; -import com.fasterxml.jackson.databind.JsonNode; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.connect.json.JsonConverter; -import org.apache.kafka.connect.storage.StringConverter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.annotations.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -public class CosmosDbSinkConnectorITest extends KafkaCosmosIntegrationTestSuiteBase { - private static final Logger logger = LoggerFactory.getLogger(CosmosDbSinkConnectorITest.class); - - // TODO[public preview]: add more integration tests - @Test(groups = { "kafka-integration"}, timeOut = TIMEOUT) - public void sinkToSingleContainer() throws InterruptedException { - Map<String, String> sinkConnectorConfig = new HashMap<>(); - - sinkConnectorConfig.put("topics", singlePartitionContainerName); - sinkConnectorConfig.put("value.converter", JsonConverter.class.getName()); - // TODO[Public Preview]: add tests for with schema - sinkConnectorConfig.put("value.converter.schemas.enable", "false"); - sinkConnectorConfig.put("key.converter", StringConverter.class.getName()); - sinkConnectorConfig.put("connector.class", "com.azure.cosmos.kafka.connect.CosmosDBSinkConnector"); - sinkConnectorConfig.put("kafka.connect.cosmos.accountEndpoint", KafkaCosmosTestConfigurations.HOST); - sinkConnectorConfig.put("kafka.connect.cosmos.accountKey", KafkaCosmosTestConfigurations.MASTER_KEY); - sinkConnectorConfig.put("kafka.connect.cosmos.applicationName", "Test"); - sinkConnectorConfig.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConnectorConfig.put("kafka.connect.cosmos.sink.containers.topicMap", singlePartitionContainerName + "#" + singlePartitionContainerName); - - // Create topic ahead of time - kafkaCosmosConnectContainer.createTopic(singlePartitionContainerName, 1); - - CosmosSinkConfig sinkConfig = new CosmosSinkConfig(sinkConnectorConfig); - CosmosAsyncClient client = CosmosClientStore.getCosmosClient(sinkConfig.getAccountConfig()); - CosmosAsyncContainer container = client.getDatabase(databaseName).getContainer(singlePartitionContainerName); - - String connectorName = "simpleTest-" + UUID.randomUUID(); - try { - // register the sink connector - kafkaCosmosConnectContainer.registerConnector(connectorName, sinkConnectorConfig); - - KafkaProducer<String, JsonNode> kafkaProducer = kafkaCosmosConnectContainer.getProducer(); - - // first create few records in the topic - logger.info("Creating sink records..."); - List<String> recordValueIds = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - TestItem testItem = TestItem.createNewItem(); - ProducerRecord<String, JsonNode> record = - new ProducerRecord<>(singlePartitionContainerName, testItem.getId(), Utils.getSimpleObjectMapper().valueToTree(testItem)); - kafkaProducer.send(record); - recordValueIds.add(testItem.getId()); - } - - // Wait for some time for the sink connector to process all records - Thread.sleep(5000); - // read from the container and verify all the items are created - String query = "select * from c"; - List<String> createdItemIds = container.queryItems(query, TestItem.class) - .byPage() - .flatMapIterable(response -> response.getResults()) - .map(TestItem::getId) - .collectList() - .block(); - assertThat(createdItemIds.size()).isEqualTo(recordValueIds.size()); - assertThat(createdItemIds.containsAll(recordValueIds)).isTrue(); - - } finally { - if (client != null) { - logger.info("cleaning container {}", singlePartitionContainerName); - cleanUpContainer(client, databaseName, singlePartitionContainerName); - client.close(); - } - - // IMPORTANT: remove the connector after use - if (kafkaCosmosConnectContainer != null) { - kafkaCosmosConnectContainer.deleteConnector(connectorName); - } - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSourceConnectorITest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSourceConnectorITest.java deleted file mode 100644 index 9c9ef85c9a33..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/CosmosDbSourceConnectorITest.java +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceConfig; -import com.fasterxml.jackson.databind.JsonNode; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.rnorth.ducttape.unreliables.Unreliables; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.annotations.Test; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -public class CosmosDbSourceConnectorITest extends KafkaCosmosIntegrationTestSuiteBase { - private static final Logger logger = LoggerFactory.getLogger(CosmosDbSourceConnectorITest.class); - - // TODO[public preview]: add more integration tests - @Test(groups = { "kafka-integration"}, timeOut = TIMEOUT) - public void readFromSingleContainer() { - Map<String, String> sourceConnectorConfig = new HashMap<>(); - sourceConnectorConfig.put("connector.class", "com.azure.cosmos.kafka.connect.CosmosDBSourceConnector"); - sourceConnectorConfig.put("kafka.connect.cosmos.accountEndpoint", KafkaCosmosTestConfigurations.HOST); - sourceConnectorConfig.put("kafka.connect.cosmos.accountKey", KafkaCosmosTestConfigurations.MASTER_KEY); - sourceConnectorConfig.put("kafka.connect.cosmos.applicationName", "Test"); - sourceConnectorConfig.put("kafka.connect.cosmos.source.database.name", databaseName); - sourceConnectorConfig.put("kafka.connect.cosmos.source.containers.includeAll", "false"); - sourceConnectorConfig.put("kafka.connect.cosmos.source.containers.includedList", singlePartitionContainerName); - - // Create topic ahead of time - kafkaCosmosConnectContainer.createTopic(singlePartitionContainerName, 1); - - CosmosSourceConfig sourceConfig = new CosmosSourceConfig(sourceConnectorConfig); - CosmosAsyncClient client = CosmosClientStore.getCosmosClient(sourceConfig.getAccountConfig()); - CosmosAsyncContainer container = client.getDatabase(databaseName).getContainer(singlePartitionContainerName); - - String connectorName = "simpleTest-" + UUID.randomUUID(); - try { - // create few items in the container - logger.info("creating items in container {}", singlePartitionContainerName); - List<String> createdItems = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - TestItem testItem = TestItem.createNewItem(); - container.createItem(testItem).block(); - createdItems.add(testItem.getId()); - } - - kafkaCosmosConnectContainer.registerConnector(connectorName, sourceConnectorConfig); - - logger.info("Getting consumer and subscribe to topic {}", singlePartitionContainerName); - KafkaConsumer<String, JsonNode> kafkaConsumer = kafkaCosmosConnectContainer.getConsumer(); - kafkaConsumer.subscribe( - Arrays.asList( - singlePartitionContainerName, - sourceConfig.getMetadataConfig().getMetadataTopicName())); - - List<ConsumerRecord<String, JsonNode>> metadataRecords = new ArrayList<>(); - List<ConsumerRecord<String, JsonNode>> itemRecords = new ArrayList<>(); - - Unreliables.retryUntilTrue(30, TimeUnit.SECONDS, () -> {; - kafkaConsumer.poll(Duration.ofMillis(1000)) - .iterator() - .forEachRemaining(consumerRecord -> { - if (consumerRecord.topic().equals(singlePartitionContainerName)) { - itemRecords.add(consumerRecord); - } else if (consumerRecord.topic().equals(sourceConfig.getMetadataConfig().getMetadataTopicName())) { - metadataRecords.add(consumerRecord); - } - }); - return metadataRecords.size() >= 2 && itemRecords.size() >= createdItems.size(); - }); - - //TODO[public preview]currently the metadata record value is null, populate it with metadata and validate the content here - assertThat(metadataRecords.size()).isEqualTo(2); - assertThat(itemRecords.size()).isEqualTo(createdItems.size()); - - List<String> receivedItems = - itemRecords.stream().map(consumerRecord -> { - JsonNode jsonNode = consumerRecord.value(); - return jsonNode.get("payload").get("id").asText(); - }).collect(Collectors.toList()); - - assertThat(receivedItems.containsAll(createdItems)).isTrue(); - - } finally { - if (client != null) { - logger.info("cleaning container {}", singlePartitionContainerName); - cleanUpContainer(client, databaseName, singlePartitionContainerName); - client.close(); - } - - // IMPORTANT: remove the connector after use - if (kafkaCosmosConnectContainer != null) { - kafkaCosmosConnectContainer.deleteConnector(connectorName); - } - } - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/InMemoryStorageReader.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/InMemoryStorageReader.java deleted file mode 100644 index c7903df4d640..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/InMemoryStorageReader.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import org.apache.kafka.connect.storage.OffsetStorageReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -/*** - * Only used for test. - */ -public class InMemoryStorageReader implements OffsetStorageReader { - private static final Logger logger = LoggerFactory.getLogger(InMemoryStorageReader.class); - private Map<Map<String, Object>, Map<String, Object>> offsetStore; - public InMemoryStorageReader() { - offsetStore = new HashMap<>(); - } - - public void populateOffset(Map<Map<String, Object>, Map<String, Object>> offsets) { - this.offsetStore.putAll(offsets); - } - - @Override - public <T> Map<String, Object> offset(Map<String, T> partition) { - return offsetStore.get(partition); - } - - @Override - public <T> Map<Map<String, T>, Map<String, Object>> offsets(Collection<Map<String, T>> partitions) { - Map<Map<String, T>, Map<String, Object>> results = new HashMap<>(); - for (Map<String, T> partition : partitions) { - results.put(partition, offsetStore.get(partition)); - } - - return results; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConfigEntry.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConfigEntry.java deleted file mode 100644 index ec0ee09b3e65..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConfigEntry.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -public class KafkaCosmosConfigEntry<T> { - private final String name; - private final T defaultValue; - private final boolean isOptional; - - public KafkaCosmosConfigEntry(String name, T defaultValue, boolean isOptional) { - this.name = name; - this.defaultValue = defaultValue; - this.isOptional = isOptional; - } - - public String getName() { - return name; - } - - public T getDefaultValue() { - return defaultValue; - } - - public boolean isOptional() { - return isOptional; - } - -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConnectContainer.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConnectContainer.java deleted file mode 100644 index f0bef9d3d65c..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosConnectContainer.java +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.core.exception.ResourceNotFoundException; -import com.fasterxml.jackson.databind.JsonNode; -import org.apache.kafka.clients.admin.AdminClient; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.KafkaConsumer; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.apache.kafka.connect.json.JsonDeserializer; -import org.apache.kafka.connect.json.JsonSerializer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sourcelab.kafka.connect.apiclient.Configuration; -import org.sourcelab.kafka.connect.apiclient.KafkaConnectClient; -import org.sourcelab.kafka.connect.apiclient.request.dto.ConnectorDefinition; -import org.sourcelab.kafka.connect.apiclient.request.dto.NewConnectorDefinition; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.KafkaContainer; -import org.testcontainers.utility.DockerImageName; - -import java.util.Arrays; -import java.util.Map; -import java.util.Properties; - -public class KafkaCosmosConnectContainer extends GenericContainer<KafkaCosmosConnectContainer> { - private static final Logger logger = LoggerFactory.getLogger(KafkaCosmosConnectContainer.class); - private static final int KAFKA_CONNECT_PORT = 8083; - private KafkaConsumer<String, JsonNode> kafkaConsumer; - private KafkaProducer<String, JsonNode> kafkaProducer; - private AdminClient adminClient; - private int replicationFactor = 1; - - public KafkaCosmosConnectContainer(final DockerImageName dockerImageName) { - super(dockerImageName); - defaultConfig(); - } - - private void defaultConfig() { - withEnv("CONNECT_GROUP_ID", KafkaCosmosTestConfigurations.CONNECT_GROUP_ID); - withEnv("CONNECT_CONFIG_STORAGE_TOPIC", KafkaCosmosTestConfigurations.CONNECT_CONFIG_STORAGE_TOPIC); - withEnv("CONNECT_OFFSET_STORAGE_TOPIC", KafkaCosmosTestConfigurations.CONNECT_OFFSET_STORAGE_TOPIC); - withEnv("CONNECT_STATUS_STORAGE_TOPIC", KafkaCosmosTestConfigurations.CONNECT_STATUS_STORAGE_TOPIC); - withEnv("CONNECT_KEY_CONVERTER", KafkaCosmosTestConfigurations.CONNECT_KEY_CONVERTER); - withEnv("CONNECT_VALUE_CONVERTER", KafkaCosmosTestConfigurations.CONNECT_VALUE_CONVERTER); - withEnv("CONNECT_PLUGIN_PATH", KafkaCosmosTestConfigurations.CONNECT_PLUGIN_PATH); - withEnv("CONNECT_REST_ADVERTISED_HOST_NAME", KafkaCosmosTestConfigurations.CONNECT_REST_ADVERTISED_HOST_NAME); - withEnv("CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR", KafkaCosmosTestConfigurations.CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR); - withEnv("CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR", KafkaCosmosTestConfigurations.CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR); - withEnv("CONNECT_STATUS_STORAGE_REPLICATION_FACTOR", KafkaCosmosTestConfigurations.CONNECT_STATUS_STORAGE_REPLICATION_FACTOR); -// withEnv("CONNECT_LOG4J_ROOT_LOGLEVEL", "DEBUG"); -// withEnv("CONNECT_LOG4J_LOGGERS", "org.apache.kafka=DEBUG,org.reflections=DEBUG,com.azure.cosmos.kafka=DEBUG"); - - withExposedPorts(KAFKA_CONNECT_PORT); - } - - private Properties defaultConsumerConfig() { - Properties kafkaConsumerProperties = new Properties(); - kafkaConsumerProperties.put("group.id", "IntegrationTest-Consumer"); - kafkaConsumerProperties.put("value.deserializer", JsonDeserializer.class.getName()); - kafkaConsumerProperties.put("key.deserializer", StringDeserializer.class.getName()); - kafkaConsumerProperties.put("sasl.mechanism", "PLAIN"); - kafkaConsumerProperties.put("client.dns.lookup", "use_all_dns_ips"); - kafkaConsumerProperties.put("session.timeout.ms", "45000"); - return kafkaConsumerProperties; - } - - private Properties defaultProducerConfig() { - Properties kafkaProducerProperties = new Properties(); - - kafkaProducerProperties.put(ProducerConfig.CLIENT_ID_CONFIG, "IntegrationTest-producer"); - kafkaProducerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); - kafkaProducerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class.getName()); - kafkaProducerProperties.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 2000L); - kafkaProducerProperties.put(ProducerConfig.ACKS_CONFIG, "all"); - kafkaProducerProperties.put("sasl.mechanism", "PLAIN"); - kafkaProducerProperties.put("client.dns.lookup", "use_all_dns_ips"); - kafkaProducerProperties.put("session.timeout.ms", "45000"); - - return kafkaProducerProperties; - } - - public KafkaCosmosConnectContainer withLocalKafkaContainer(final KafkaContainer kafkaContainer) { - withNetwork(kafkaContainer.getNetwork()); - - withEnv("CONNECT_BOOTSTRAP_SERVERS", kafkaContainer.getNetworkAliases().get(0) + ":9092"); - return self(); - } - - public KafkaCosmosConnectContainer withCloudKafkaContainer() { - withEnv("CONNECT_BOOTSTRAP_SERVERS", KafkaCosmosTestConfigurations.BOOTSTRAP_SERVER); - withEnv("CONNECT_SECURITY_PROTOCOL", "SASL_SSL"); - withEnv("CONNECT_SASL_JAAS_CONFIG", KafkaCosmosTestConfigurations.SASL_JAAS); - withEnv("CONNECT_SASL_MECHANISM", "PLAIN"); - - withEnv("CONNECT_PRODUCER_SECURITY_PROTOCOL", "SASL_SSL"); - withEnv("CONNECT_PRODUCER_SASL_JAAS_CONFIG", KafkaCosmosTestConfigurations.SASL_JAAS); - withEnv("CONNECT_PRODUCER_SASL_MECHANISM", "PLAIN"); - - withEnv("CONNECT_CONSUMER_SECURITY_PROTOCOL", "SASL_SSL"); - withEnv("CONNECT_CONSUMER_SASL_JAAS_CONFIG", KafkaCosmosTestConfigurations.SASL_JAAS); - withEnv("CONNECT_CONSUMER_SASL_MECHANISM", "PLAIN"); - return self(); - } - - public KafkaCosmosConnectContainer withLocalBootstrapServer(String localBootstrapServer) { - Properties consumerProperties = defaultConsumerConfig(); - consumerProperties.put("bootstrap.servers", localBootstrapServer); - this.kafkaConsumer = new KafkaConsumer<>(consumerProperties); - - Properties producerProperties = defaultProducerConfig(); - producerProperties.put("bootstrap.servers", localBootstrapServer); - this.kafkaProducer = new KafkaProducer<>(producerProperties); - - this.adminClient = this.getAdminClient(localBootstrapServer); - return self(); - } - - public KafkaCosmosConnectContainer withCloudBootstrapServer() { - Properties consumerProperties = defaultConsumerConfig(); - consumerProperties.put("bootstrap.servers", KafkaCosmosTestConfigurations.BOOTSTRAP_SERVER); - consumerProperties.put("sasl.jaas.config", KafkaCosmosTestConfigurations.SASL_JAAS); - consumerProperties.put("security.protocol", "SASL_SSL"); - consumerProperties.put("sasl.mechanism", "PLAIN"); - - this.kafkaConsumer = new KafkaConsumer<>(consumerProperties); - - Properties producerProperties = defaultProducerConfig(); - producerProperties.put("bootstrap.servers", KafkaCosmosTestConfigurations.BOOTSTRAP_SERVER); - producerProperties.put("sasl.jaas.config", KafkaCosmosTestConfigurations.SASL_JAAS); - producerProperties.put("security.protocol", "SASL_SSL"); - producerProperties.put("sasl.mechanism", "PLAIN"); - this.kafkaProducer = new KafkaProducer<>(producerProperties); - - this.adminClient = this.getAdminClient(KafkaCosmosTestConfigurations.BOOTSTRAP_SERVER); - this.replicationFactor = 3; - return self(); - } - - public void registerConnector(String name, Map<String, String> config) { - NewConnectorDefinition newConnectorDefinition = new NewConnectorDefinition(name, config); - KafkaConnectClient kafkaConnectClient = new KafkaConnectClient(new Configuration(getTarget())); - - logger.info("adding kafka connector {}", name); - - try { - Thread.sleep(500); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - ConnectorDefinition connectorDefinition = kafkaConnectClient.addConnector(newConnectorDefinition); - logger.info("adding kafka connector completed with " + connectorDefinition); - } - - public void deleteConnector(String name) { - KafkaConnectClient kafkaConnectClient = new KafkaConnectClient(new Configuration(getTarget())); - try { - kafkaConnectClient.deleteConnector(name); - logger.info("Deleting container {} succeeded.", name); - } catch (Exception exception) { - if (exception instanceof ResourceNotFoundException) { - logger.info("Connector {} not found"); - } - - logger.warn("Failed to delete connector {}", name); - } - } - - public KafkaConsumer<String, JsonNode> getConsumer() { - return this.kafkaConsumer; - } - - public KafkaProducer<String, JsonNode> getProducer() { - return this.kafkaProducer; - } - - public String getTarget() { - return "http://" + getContainerIpAddress() + ":" + getMappedPort(KAFKA_CONNECT_PORT); - } - - public void createTopic(String topicName, int numPartitions) { - this.adminClient.createTopics( - Arrays.asList(new NewTopic(topicName, numPartitions, (short) replicationFactor))); - } - - private AdminClient getAdminClient(String bootstrapServer) { - Properties properties = new Properties(); - properties.put("bootstrap.servers", bootstrapServer); - return AdminClient.create(properties); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosIntegrationTestSuiteBase.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosIntegrationTestSuiteBase.java deleted file mode 100644 index 6a113f45b037..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosIntegrationTestSuiteBase.java +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.KafkaContainer; -import org.testcontainers.containers.Network; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.lifecycle.Startables; -import org.testcontainers.utility.DockerImageName; -import org.testng.annotations.AfterSuite; -import org.testng.annotations.BeforeSuite; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.Arrays; -import java.util.stream.Stream; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.testng.AssertJUnit.fail; - -public class KafkaCosmosIntegrationTestSuiteBase extends KafkaCosmosTestSuiteBase { - private static final Logger logger = LoggerFactory.getLogger(KafkaCosmosIntegrationTestSuiteBase.class); - private static final Duration DEFAULT_CONTAINER_START_UP_TIMEOUT = Duration.ofMinutes(5); - - protected static Network network; - protected static KafkaContainer kafkaContainer; - protected static KafkaCosmosConnectContainer kafkaCosmosConnectContainer; - - @BeforeSuite(groups = { "kafka-integration" }, timeOut = 10 * SUITE_SETUP_TIMEOUT) - public static void beforeIntegrationSuite() throws IOException, InterruptedException { - - logger.info("beforeIntegrationSuite Started"); - // initialize the kafka, kafka-connect containers - setupDockerContainers(); - } - - @AfterSuite(groups = { "kafka-integration" }, timeOut = 10 * SUITE_SETUP_TIMEOUT) - public static void afterIntegrationSuite() { - - logger.info("afterIntegrationSuite Started"); - // The TestContainers library will automatically clean up resources by using Ryuk sidecar container - } - - private static void setupDockerContainers() throws IOException, InterruptedException { - createConnectorJar(); - - logger.info("Setting up docker containers..."); - - network = Network.newNetwork(); - if (StringUtils.isEmpty(KafkaCosmosTestConfigurations.BOOTSTRAP_SERVER)) { - setupDockerContainersForLocal(); - } else { - setupDockerContainersForCloud(); - } - } - - private static void setupDockerContainersForLocal() { - logger.info("Setting up local docker containers..."); - network = Network.newNetwork(); - kafkaContainer = new KafkaContainer(getDockerImageName("confluentinc/cp-kafka:")) - .withNetwork(network) - .withNetworkAliases("broker") - .withStartupTimeout(DEFAULT_CONTAINER_START_UP_TIMEOUT); - - kafkaCosmosConnectContainer = new KafkaCosmosConnectContainer(getDockerImageName("confluentinc/cp-kafka-connect:")) - .withNetwork(network) - .dependsOn(kafkaContainer) - .withLocalKafkaContainer(kafkaContainer) - .withStartupTimeout(DEFAULT_CONTAINER_START_UP_TIMEOUT) - .withFileSystemBind("src/test/connectorPlugins", "/kafka/connect/cosmos-connector") - .withLogConsumer(new Slf4jLogConsumer(logger)); - - Startables.deepStart(Stream.of(kafkaContainer, kafkaCosmosConnectContainer)).join(); - - // the mapped bootstrap server port can only be obtained after the container started - kafkaCosmosConnectContainer.withLocalBootstrapServer(kafkaContainer.getBootstrapServers()); - } - - private static void setupDockerContainersForCloud() { - logger.info("Setting up docker containers with self-managed cloud clusters..."); - kafkaCosmosConnectContainer = new KafkaCosmosConnectContainer(getDockerImageName("confluentinc/cp-kafka-connect:")) - .withCloudKafkaContainer() - .withStartupTimeout(DEFAULT_CONTAINER_START_UP_TIMEOUT) - .withFileSystemBind("src/test/connectorPlugins", "/kafka/connect/cosmos-connector") - .withLogConsumer(new Slf4jLogConsumer(logger)); - - Startables.deepStart(Stream.of(kafkaCosmosConnectContainer)).join(); - - kafkaCosmosConnectContainer.withCloudBootstrapServer(); - } - - private static void createConnectorJar() throws IOException, InterruptedException { - logger.info("Start creating connector jars..."); - - boolean isWindows = System.getProperty("os.name").startsWith("windows"); - Path connectorPluginsPath = Paths.get("src/test/connectorPlugins"); - - ProcessBuilder processBuilder; - if (isWindows) { - String buildScriptPath = connectorPluginsPath + "/build.ps1"; - processBuilder = new ProcessBuilder("powershell.exe", buildScriptPath); - } else { - String buildScriptPath = connectorPluginsPath + "/build.sh"; - processBuilder = new ProcessBuilder("/bin/bash", buildScriptPath); - } - - processBuilder.redirectErrorStream(true); - Process process = processBuilder.start(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line; - while ((line = reader.readLine()) != null) { - logger.info(line); - System.out.println(line); - } - - // Wait for the script to complete - int exitCode = process.waitFor(); - if (exitCode == 0) { - logger.info("Build script completed successfully"); - //validate the jar exists - File jarFile = findFile("src/test/connectorPlugins/connectors", "jar-with-dependencies.jar"); - - assertThat(jarFile).isNotNull(); - assertThat(jarFile.exists()).isTrue(); - - } else { - fail("Build script failed with error code " + exitCode); - } - } - - private static File findFile(String folder, String filenameFilterEndsWith) { - File file = new File(folder); - if (!file.exists() || !file.isDirectory()) { - return null; - } - return Arrays.stream(file.listFiles()) - .filter(f -> f.getName().endsWith(filenameFilterEndsWith)) - .findFirst().orElse(null); - } - - private static DockerImageName getDockerImageName(String prefix) { - return DockerImageName.parse(prefix + KafkaCosmosTestConfigurations.CONFLUENT_VERSION); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosReflectionUtils.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosReflectionUtils.java deleted file mode 100644 index cce96e741deb..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosReflectionUtils.java +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.kafka.connect.implementation.sink.CosmosSinkTask; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceConfig; -import com.azure.cosmos.kafka.connect.implementation.source.CosmosSourceOffsetStorageReader; -import com.azure.cosmos.kafka.connect.implementation.source.MetadataMonitorThread; -import org.apache.commons.lang3.reflect.FieldUtils; -import org.apache.kafka.connect.sink.SinkTaskContext; -import org.apache.kafka.connect.storage.OffsetStorageReader; - -public class KafkaCosmosReflectionUtils { - private static <T> void set(Object object, T newValue, String fieldName) { - try { - FieldUtils.writeField(object, fieldName, newValue, true); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - - @SuppressWarnings("unchecked") - private static <T> T get(Object object, String fieldName) { - try { - return (T) FieldUtils.readField(object, fieldName, true); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - - public static void setCosmosClient(CosmosDBSourceConnector sourceConnector, CosmosAsyncClient cosmosAsyncClient) { - set(sourceConnector, cosmosAsyncClient,"cosmosClient"); - } - - public static void setCosmosSourceConfig(CosmosDBSourceConnector sourceConnector, CosmosSourceConfig sourceConfig) { - set(sourceConnector, sourceConfig,"config"); - } - - public static void setOffsetStorageReader( - CosmosDBSourceConnector sourceConnector, - CosmosSourceOffsetStorageReader storageReader) { - set(sourceConnector, storageReader,"offsetStorageReader"); - } - - public static void setMetadataMonitorThread( - CosmosDBSourceConnector sourceConnector, - MetadataMonitorThread metadataMonitorThread) { - set(sourceConnector, metadataMonitorThread,"monitorThread"); - } - - public static CosmosAsyncClient getCosmosClient(CosmosDBSourceConnector sourceConnector) { - return get(sourceConnector,"cosmosClient"); - } - - public static CosmosSourceOffsetStorageReader getSourceOffsetStorageReader(CosmosDBSourceConnector sourceConnector) { - return get(sourceConnector,"offsetStorageReader"); - } - - public static OffsetStorageReader getOffsetStorageReader(CosmosSourceOffsetStorageReader sourceOffsetStorageReader) { - return get(sourceOffsetStorageReader,"offsetStorageReader"); - } - - public static void setSinkTaskContext(CosmosSinkTask sinkTask, SinkTaskContext sinkTaskContext) { - set(sinkTask, sinkTaskContext, "context"); - } - - public static CosmosAsyncClient getSinkTaskCosmosClient(CosmosSinkTask sinkTask) { - return get(sinkTask,"cosmosClient"); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestConfigurations.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestConfigurations.java deleted file mode 100644 index f26d86ce1728..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestConfigurations.java +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.cosmos.implementation.apachecommons.lang.StringUtils; -import com.azure.cosmos.implementation.guava25.base.Strings; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.InputStream; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Properties; - -public class KafkaCosmosTestConfigurations { - private static final Logger logger = LoggerFactory.getLogger(KafkaCosmosTestConfigurations.class); - private static Properties properties = loadProperties(); - - private static final String COSMOS_EMULATOR_KEY = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="; - private static final String COSMOS_EMULATOR_HOST = "https://localhost:8081/"; - public static final String DEFAULT_CONFLUENT_VERSION = "7.6.0"; //https://docs.confluent.io/platform/current/installation/versions-interoperability.html - public static final String DEFAULT_CONNECT_GROUP_ID = "1"; - public static final String DEFAULT_CONNECT_CONFIG_STORAGE_TOPIC = "docker-connect-configs"; - public static final String DEFAULT_CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR = "1"; - public static final String DEFAULT_CONNECT_OFFSET_STORAGE_TOPIC = "docker-connect-offsets"; - public static final String DEFAULT_CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR = "1"; - public static final String DEFAULT_CONNECT_STATUS_STORAGE_TOPIC = "docker-connect-status"; - public static final String DEFAULT_CONNECT_STATUS_STORAGE_REPLICATION_FACTOR = "1"; - public static final String DEFAULT_CONNECT_KEY_CONVERTER = "org.apache.kafka.connect.storage.StringConverter"; - public static final String DEFAULT_CONNECT_VALUE_CONVERTER = "org.apache.kafka.connect.json.JsonConverter"; - public static final String DEFAULT_CONNECT_PLUGIN_PATH = "/kafka/connect/cosmos-connector"; - public static final String DEFAULT_CONNECT_REST_ADVERTISED_HOST_NAME = "connect"; - - public final static String MASTER_KEY = - properties - .getProperty( - "ACCOUNT_KEY", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("ACCOUNT_KEY")), COSMOS_EMULATOR_KEY)); - - public final static String SECONDARY_MASTER_KEY = - properties - .getProperty( - "SECONDARY_ACCOUNT_KEY", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("SECONDARY_ACCOUNT_KEY")), COSMOS_EMULATOR_KEY)); - - public final static String HOST = - properties - .getProperty( - "ACCOUNT_HOST", - StringUtils.defaultString( - Strings.emptyToNull(System.getenv().get("ACCOUNT_HOST")), COSMOS_EMULATOR_HOST)); - - public final static String KAFKA_CLUSTER_KEY = - properties - .getProperty( - "KAFKA_CLUSTER_KEY", - StringUtils.defaultString( - Strings.emptyToNull(System.getenv().get("KAFKA_CLUSTER_KEY")), "")); - - public final static String KAFKA_CLUSTER_SECRET = - properties - .getProperty( - "KAFKA_CLUSTER_SECRET", - StringUtils.defaultString( - Strings.emptyToNull(System.getenv().get("KAFKA_CLUSTER_SECRET")), "")); - - public final static String SCHEMA_REGISTRY_KEY = - properties - .getProperty( - "SCHEMA_REGISTRY_KEY", - StringUtils.defaultString( - Strings.emptyToNull(System.getenv().get("SCHEMA_REGISTRY_KEY")), "")); - - public final static String SCHEMA_REGISTRY_SECRET = - properties - .getProperty( - "SCHEMA_REGISTRY_SECRET", - StringUtils.defaultString( - Strings.emptyToNull(System.getenv().get("SCHEMA_REGISTRY_SECRET")), "")); - - public final static String SCHEMA_REGISTRY_URL = - properties - .getProperty( - "SCHEMA_REGISTRY_URL", - StringUtils.defaultString( - Strings.emptyToNull(System.getenv().get("SCHEMA_REGISTRY_URL")), "")); - - public final static String BOOTSTRAP_SERVER = - properties - .getProperty( - "BOOTSTRAP_SERVER", - StringUtils.defaultString( - Strings.emptyToNull(System.getenv().get("BOOTSTRAP_SERVER")), "")); - - public final static String SASL_JAAS = - properties - .getProperty( - "SASL_JAAS", - StringUtils.defaultString( - Strings.emptyToNull(System.getenv().get("SASL_JAAS")), "")); - - public final static String CONFLUENT_VERSION = - properties - .getProperty( - "CONFLUENT_VERSION", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONFLUENT_VERSION")), DEFAULT_CONFLUENT_VERSION)); - - public final static String CONNECT_GROUP_ID = - properties - .getProperty( - "CONNECT_GROUP_ID", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_GROUP_ID")), DEFAULT_CONNECT_GROUP_ID)); - - public final static String CONNECT_CONFIG_STORAGE_TOPIC = - properties - .getProperty( - "CONNECT_CONFIG_STORAGE_TOPIC", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_CONFIG_STORAGE_TOPIC")), DEFAULT_CONNECT_CONFIG_STORAGE_TOPIC)); - - public final static String CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR = - properties - .getProperty( - "CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR")), DEFAULT_CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR)); - - public final static String CONNECT_OFFSET_STORAGE_TOPIC = - properties - .getProperty( - "CONNECT_OFFSET_STORAGE_TOPIC", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_OFFSET_STORAGE_TOPIC")), DEFAULT_CONNECT_OFFSET_STORAGE_TOPIC)); - - public final static String CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR = - properties - .getProperty( - "CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR")), DEFAULT_CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR)); - - public final static String CONNECT_STATUS_STORAGE_TOPIC = - properties - .getProperty( - "CONNECT_STATUS_STORAGE_TOPIC", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_STATUS_STORAGE_TOPIC")), DEFAULT_CONNECT_STATUS_STORAGE_TOPIC)); - - public final static String CONNECT_STATUS_STORAGE_REPLICATION_FACTOR = - properties - .getProperty( - "CONNECT_STATUS_STORAGE_REPLICATION_FACTOR", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_STATUS_STORAGE_REPLICATION_FACTOR")), DEFAULT_CONNECT_STATUS_STORAGE_REPLICATION_FACTOR)); - - public final static String CONNECT_KEY_CONVERTER = - properties - .getProperty( - "CONNECT_KEY_CONVERTER", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_KEY_CONVERTER")), DEFAULT_CONNECT_KEY_CONVERTER)); - - public final static String CONNECT_VALUE_CONVERTER = - properties - .getProperty( - "CONNECT_VALUE_CONVERTER", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_VALUE_CONVERTER")), DEFAULT_CONNECT_VALUE_CONVERTER)); - - public final static String CONNECT_PLUGIN_PATH = - properties - .getProperty( - "CONNECT_PLUGIN_PATH", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_PLUGIN_PATH")), DEFAULT_CONNECT_PLUGIN_PATH)); - - public final static String CONNECT_REST_ADVERTISED_HOST_NAME = - properties - .getProperty( - "CONNECT_REST_ADVERTISED_HOST_NAME", - StringUtils.defaultString(Strings.emptyToNull(System.getenv().get("CONNECT_REST_ADVERTISED_HOST_NAME")), DEFAULT_CONNECT_REST_ADVERTISED_HOST_NAME)); - - private static Properties loadProperties() { - Path root = FileSystems.getDefault().getPath("").toAbsolutePath(); - Path propertiesInProject = Paths.get(root.toString(),"../kafka-cosmos-local.properties"); - - Properties props = loadFromPathIfExists(propertiesInProject); - if (props != null) { - return props; - } - - Path propertiesInUserHome = Paths.get(System.getProperty("user.home"), "kafka-cosmos-local.properties"); - props = loadFromPathIfExists(propertiesInUserHome); - if (props != null) { - return props; - } - - return System.getProperties(); - } - - private static Properties loadFromPathIfExists(Path propertiesFilePath) { - if (Files.exists(propertiesFilePath)) { - try (InputStream in = Files.newInputStream(propertiesFilePath)) { - Properties props = new Properties(); - props.load(in); - logger.info("properties loaded from {}", propertiesFilePath); - return props; - } catch (Exception e) { - logger.error("Loading properties {} failed", propertiesFilePath, e); - } - } - return null; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestNGLogListener.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestNGLogListener.java deleted file mode 100644 index 0d4366c9d729..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestNGLogListener.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.IInvokedMethod; -import org.testng.IInvokedMethodListener; -import org.testng.ITestResult; -import org.testng.SkipException; - -public class KafkaCosmosTestNGLogListener implements IInvokedMethodListener { - private final Logger logger = LoggerFactory.getLogger(KafkaCosmosTestNGLogListener.class); - - @Override - public void beforeInvocation(IInvokedMethod iInvokedMethod, ITestResult iTestResult) { - logger.info("beforeInvocation: {}", methodName(iInvokedMethod)); - } - - @Override - public void afterInvocation(IInvokedMethod iInvokedMethod, ITestResult iTestResult) { - logger.info("afterInvocation: {}, total time {}ms, result {}", - methodName(iInvokedMethod), - iTestResult.getEndMillis() - iTestResult.getStartMillis(), - resultDetails(iTestResult) - ); - } - - private String resultDetails(ITestResult iTestResult) { - if (iTestResult.isSuccess()) { - return "success"; - } - - if (iTestResult.getThrowable() instanceof SkipException) { - return "skipped. reason: " + failureDetails(iTestResult); - } - - return "failed. reason: " + failureDetails(iTestResult); - } - - private String failureDetails(ITestResult iTestResult) { - if (iTestResult.isSuccess()) { - return null; - } - - if (iTestResult.getThrowable() == null) { - logger.error("throwable is null"); - return null; - } - - return iTestResult.getThrowable().getClass().getName() + ": " + iTestResult.getThrowable().getMessage(); - } - - private String methodName(IInvokedMethod iInvokedMethod) { - return iInvokedMethod.getTestMethod().getRealClass().getSimpleName() + "#" + iInvokedMethod.getTestMethod().getMethodName(); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java deleted file mode 100644 index 8e601948a6c3..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/KafkaCosmosTestSuiteBase.java +++ /dev/null @@ -1,262 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import com.azure.core.credential.AzureKeyCredential; -import com.azure.cosmos.ConsistencyLevel; -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.CosmosAsyncDatabase; -import com.azure.cosmos.CosmosClientBuilder; -import com.azure.cosmos.GatewayConnectionConfig; -import com.azure.cosmos.ThrottlingRetryOptions; -import com.azure.cosmos.implementation.ImplementationBridgeHelpers; -import com.azure.cosmos.implementation.guava27.Strings; -import com.azure.cosmos.models.CosmosContainerProperties; -import com.azure.cosmos.models.CosmosContainerRequestOptions; -import com.azure.cosmos.models.CosmosItemRequestOptions; -import com.azure.cosmos.models.IncludedPath; -import com.azure.cosmos.models.IndexingPolicy; -import com.azure.cosmos.models.PartitionKeyDefinition; -import com.azure.cosmos.models.ThroughputProperties; -import com.fasterxml.jackson.databind.JsonNode; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.ITest; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.AfterSuite; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.BeforeSuite; -import org.testng.annotations.Listeners; - -import java.lang.reflect.Method; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.UUID; - -@Listeners({KafkaCosmosTestNGLogListener.class}) -public class KafkaCosmosTestSuiteBase implements ITest { - protected static Logger logger = LoggerFactory.getLogger(KafkaCosmosTestSuiteBase.class.getSimpleName()); - protected static final int TIMEOUT = 60000; - - protected static final int SUITE_SETUP_TIMEOUT = 120000; - protected static final int SUITE_SHUTDOWN_TIMEOUT = 60000; - - protected static final AzureKeyCredential credential; - protected static String databaseName; - protected static String multiPartitionContainerWithIdAsPartitionKeyName; - protected static String multiPartitionContainerName; - protected static String singlePartitionContainerName; - private String testName; - - protected static CosmosAsyncDatabase getDatabase(CosmosAsyncClient client) { - return client.getDatabase(databaseName); - } - - protected static CosmosContainerProperties getMultiPartitionContainerWithIdAsPartitionKey(CosmosAsyncClient client) { - return client - .getDatabase(databaseName) - .getContainer(multiPartitionContainerWithIdAsPartitionKeyName) - .read() - .block() - .getProperties(); - } - - protected static CosmosContainerProperties getMultiPartitionContainer(CosmosAsyncClient client) { - return client - .getDatabase(databaseName) - .getContainer(multiPartitionContainerName) - .read() - .block() - .getProperties(); - } - - protected static CosmosContainerProperties getSinglePartitionContainer(CosmosAsyncClient client) { - return client - .getDatabase(databaseName) - .getContainer(singlePartitionContainerName) - .read() - .block() - .getProperties(); - } - - static { - credential = new AzureKeyCredential(KafkaCosmosTestConfigurations.MASTER_KEY); - } - - @BeforeSuite(groups = { "kafka", "kafka-integration" }, timeOut = SUITE_SETUP_TIMEOUT) - public static void beforeSuite() { - - logger.info("beforeSuite Started"); - try (CosmosAsyncClient houseKeepingClient = createGatewayHouseKeepingDocumentClient(true).buildAsyncClient()) { - databaseName = createDatabase(houseKeepingClient); - - CosmosContainerRequestOptions options = new CosmosContainerRequestOptions(); - multiPartitionContainerName = - createCollection( - houseKeepingClient, - databaseName, - getCollectionDefinitionWithRangeRangeIndex(), - options, - 10100); - multiPartitionContainerWithIdAsPartitionKeyName = - createCollection( - houseKeepingClient, - databaseName, - getCollectionDefinitionWithRangeRangeIndexWithIdAsPartitionKey(), - options, - 10100); - singlePartitionContainerName = - createCollection( - houseKeepingClient, - databaseName, - getCollectionDefinitionWithRangeRangeIndex(), - options, - 6000); - } - } - - @BeforeSuite(groups = { "unit" }, timeOut = SUITE_SETUP_TIMEOUT) - public static void beforeSuiteUnit() { - logger.info("beforeSuite for unit tests started"); - - databaseName = - StringUtils.isEmpty(databaseName) ? "KafkaCosmosTest-" + UUID.randomUUID() : databaseName; - multiPartitionContainerName = - StringUtils.isEmpty(multiPartitionContainerName) ? UUID.randomUUID().toString() : multiPartitionContainerName; - singlePartitionContainerName = - StringUtils.isEmpty(singlePartitionContainerName) ? UUID.randomUUID().toString() : singlePartitionContainerName; - } - - @AfterSuite(groups = { "kafka", "kafka-integration" }, timeOut = SUITE_SHUTDOWN_TIMEOUT) - public static void afterSuite() { - - logger.info("afterSuite Started"); - - try (CosmosAsyncClient houseKeepingClient = createGatewayHouseKeepingDocumentClient(true).buildAsyncClient()) { - safeDeleteDatabase(houseKeepingClient, databaseName); - } - } - - - static protected CosmosClientBuilder createGatewayHouseKeepingDocumentClient(boolean contentResponseOnWriteEnabled) { - ThrottlingRetryOptions options = new ThrottlingRetryOptions(); - options.setMaxRetryWaitTime(Duration.ofSeconds(SUITE_SETUP_TIMEOUT)); - GatewayConnectionConfig gatewayConnectionConfig = new GatewayConnectionConfig(); - return new CosmosClientBuilder().endpoint(KafkaCosmosTestConfigurations.HOST) - .credential(credential) - .gatewayMode(gatewayConnectionConfig) - .throttlingRetryOptions(options) - .contentResponseOnWriteEnabled(contentResponseOnWriteEnabled) - .consistencyLevel(ConsistencyLevel.SESSION); - } - - private static String createDatabase(CosmosAsyncClient cosmosAsyncClient) { - String databaseName = "KafkaCosmosTest-" + UUID.randomUUID(); - cosmosAsyncClient.createDatabase(databaseName).block(); - - return databaseName; - } - - private static String createCollection( - CosmosAsyncClient cosmosAsyncClient, - String database, - CosmosContainerProperties cosmosContainerProperties, - CosmosContainerRequestOptions options, - int throughput) { - - cosmosAsyncClient - .getDatabase(database) - .createContainer( - cosmosContainerProperties, - ThroughputProperties.createManualThroughput(throughput), - options) - .block(); - - // Creating a container is async - especially on multi-partition or multi-region accounts - boolean isMultiRegional = ImplementationBridgeHelpers - .CosmosAsyncClientHelper - .getCosmosAsyncClientAccessor() - .getPreferredRegions(cosmosAsyncClient).size() > 1; - - if (throughput > 6000 || isMultiRegional) { - try { - Thread.sleep(3000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - - return cosmosContainerProperties.getId(); - } - - static protected CosmosContainerProperties getCollectionDefinitionWithRangeRangeIndex() { - return getCollectionDefinitionWithRangeRangeIndex(Collections.singletonList("/mypk")); - } - - static protected CosmosContainerProperties getCollectionDefinitionWithRangeRangeIndexWithIdAsPartitionKey() { - return getCollectionDefinitionWithRangeRangeIndex(Collections.singletonList("/id")); - } - - public static void cleanUpContainer(CosmosAsyncClient client, String databaseName, String containerName) { - CosmosAsyncContainer container = client.getDatabase(databaseName).getContainer(containerName); - List<JsonNode> allItems = - container.queryItems("select * from c", JsonNode.class) - .byPage() - .flatMapIterable(feedResponse -> feedResponse.getResults()) - .collectList() - .block(); - - // do a batch delete - for (JsonNode item : allItems) { - container.deleteItem(item, new CosmosItemRequestOptions()).block(); - } - } - - static protected CosmosContainerProperties getCollectionDefinitionWithRangeRangeIndex(List<String> partitionKeyPath) { - PartitionKeyDefinition partitionKeyDef = new PartitionKeyDefinition(); - - partitionKeyDef.setPaths(partitionKeyPath); - IndexingPolicy indexingPolicy = new IndexingPolicy(); - List<IncludedPath> includedPaths = new ArrayList<>(); - IncludedPath includedPath = new IncludedPath("/*"); - includedPaths.add(includedPath); - indexingPolicy.setIncludedPaths(includedPaths); - - CosmosContainerProperties cosmosContainerProperties = new CosmosContainerProperties(UUID.randomUUID().toString(), partitionKeyDef); - cosmosContainerProperties.setIndexingPolicy(indexingPolicy); - - return cosmosContainerProperties; - } - - private static void safeDeleteDatabase(CosmosAsyncClient client, String database) { - if (StringUtils.isNotEmpty(database)) { - try { - client.getDatabase(database).delete().block(); - } catch (Exception e) { - logger.error("Failed to delete database {}", database, e); - } - } - } - - @BeforeMethod(alwaysRun = true) - public final void setTestName(Method method, Object[] row) { - this.testName = Strings.lenientFormat("%s::%s", - method.getDeclaringClass().getSimpleName(), - method.getName()); - } - - @AfterMethod(alwaysRun = true) - public final void unsetTestName() { - this.testName = null; - } - - @Override - public String getTestName() { - return this.testName; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/TestItem.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/TestItem.java deleted file mode 100644 index 669dd67d9349..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/TestItem.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect; - -import java.util.UUID; - -public class TestItem { - private String id; - private String mypk; - private String prop; - - public TestItem() { - } - - public TestItem(String id, String mypk, String prop) { - this.id = id; - this.mypk = mypk; - this.prop = prop; - } - - public static TestItem createNewItem() { - return new TestItem(UUID.randomUUID().toString(), UUID.randomUUID().toString(), UUID.randomUUID().toString()); - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getMypk() { - return mypk; - } - - public void setMypk(String mypk) { - this.mypk = mypk; - } - - public String getProp() { - return prop; - } - - public void setProp(String prop) { - this.prop = prop; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskTest.java deleted file mode 100644 index 89cc8322b050..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/CosmosSinkTaskTest.java +++ /dev/null @@ -1,571 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.implementation.TestConfigurations; -import com.azure.cosmos.implementation.Utils; -import com.azure.cosmos.kafka.connect.KafkaCosmosReflectionUtils; -import com.azure.cosmos.kafka.connect.KafkaCosmosTestSuiteBase; -import com.azure.cosmos.kafka.connect.TestItem; -import com.azure.cosmos.kafka.connect.implementation.source.JsonToStruct; -import com.azure.cosmos.models.CosmosContainerProperties; -import com.azure.cosmos.models.PartitionKey; -import com.azure.cosmos.test.faultinjection.CosmosFaultInjectionHelper; -import com.azure.cosmos.test.faultinjection.FaultInjectionConditionBuilder; -import com.azure.cosmos.test.faultinjection.FaultInjectionResultBuilders; -import com.azure.cosmos.test.faultinjection.FaultInjectionRule; -import com.azure.cosmos.test.faultinjection.FaultInjectionRuleBuilder; -import com.azure.cosmos.test.faultinjection.FaultInjectionServerErrorType; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.kafka.connect.data.ConnectSchema; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaAndValue; -import org.apache.kafka.connect.sink.SinkRecord; -import org.apache.kafka.connect.sink.SinkTaskContext; -import org.mockito.Mockito; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; -import reactor.core.publisher.Mono; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - - -public class CosmosSinkTaskTest extends KafkaCosmosTestSuiteBase { - @DataProvider(name = "sinkTaskParameterProvider") - public Object[][] sinkTaskParameterProvider() { - return new Object[][]{ - // flag to indicate whether bulk enabled or not, sink record value schema - { true, Schema.Type.MAP }, - { false, Schema.Type.MAP }, - { true, Schema.Type.STRUCT }, - { false, Schema.Type.STRUCT } - }; - } - - @DataProvider(name = "bulkEnableParameterProvider") - public Object[][] bulkEnableParameterProvider() { - return new Object[][]{ - // flag to indicate whether bulk enabled or not - { true }, - { false } - }; - } - - @Test(groups = { "kafka" }, dataProvider = "sinkTaskParameterProvider", timeOut = TIMEOUT) - public void sinkWithValidRecords(boolean bulkEnabled, Schema.Type valueSchemaType) { - String topicName = singlePartitionContainerName; - - Map<String, String> sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - try { - List<SinkRecord> sinkRecordList = new ArrayList<>(); - List<TestItem> toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - valueSchemaType, - toBeCreateItems, - sinkRecordList); - - sinkTask.put(sinkRecordList); - - // get all the items - List<String> writtenItemIds = new ArrayList<>(); - String query = "select * from c"; - container.queryItems(query, TestItem.class) - .byPage() - .flatMap(response -> { - writtenItemIds.addAll( - response.getResults().stream().map(TestItem::getId).collect(Collectors.toList())); - return Mono.empty(); - }) - .blockLast(); - - assertThat(writtenItemIds.size()).isEqualTo(toBeCreateItems.size()); - List<String> toBeCreateItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(writtenItemIds.containsAll(toBeCreateItemIds)).isTrue(); - - } finally { - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - sinkTask.stop(); - } - } - } - - @Test(groups = { "kafka" }, dataProvider = "bulkEnableParameterProvider", timeOut = 10 * TIMEOUT) - public void retryOnServiceUnavailable(boolean bulkEnabled) { - String topicName = singlePartitionContainerName; - - Map<String, String> sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - // configure fault injection rule - FaultInjectionRule goneExceptionRule = - new FaultInjectionRuleBuilder("goneExceptionRule-" + UUID.randomUUID()) - .condition(new FaultInjectionConditionBuilder().build()) - .result( - FaultInjectionResultBuilders - .getResultBuilder(FaultInjectionServerErrorType.GONE) - .build()) - // high enough so the batch requests will fail with 503 in the first time but low enough so the second retry from kafka connector can succeed - .hitLimit(10) - .build(); - - try { - List<SinkRecord> sinkRecordList = new ArrayList<>(); - List<TestItem> toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - Schema.Type.STRUCT, - toBeCreateItems, - sinkRecordList); - - CosmosFaultInjectionHelper.configureFaultInjectionRules(container, Arrays.asList(goneExceptionRule)).block(); - sinkTask.put(sinkRecordList); - - // get all the items - List<String> writtenItemIds = new ArrayList<>(); - String query = "select * from c"; - container.queryItems(query, TestItem.class) - .byPage() - .flatMap(response -> { - writtenItemIds.addAll( - response.getResults().stream().map(TestItem::getId).collect(Collectors.toList())); - return Mono.empty(); - }) - .blockLast(); - - assertThat(writtenItemIds.size()).isEqualTo(toBeCreateItems.size()); - List<String> toBeCreateItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(toBeCreateItemIds.containsAll(writtenItemIds)).isTrue(); - - } finally { - goneExceptionRule.disable(); - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - cosmosClient.close(); - } - } - } - - @Test(groups = { "kafka" }, dataProvider = "bulkEnableParameterProvider", timeOut = TIMEOUT) - public void sinkWithItemAppend(boolean bulkEnabled) { - String topicName = singlePartitionContainerName; - - Map<String, String> sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - sinkConfigMap.put("kafka.connect.cosmos.sink.write.strategy", ItemWriteStrategy.ITEM_APPEND.getName()); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - try { - List<SinkRecord> sinkRecordList = new ArrayList<>(); - List<TestItem> toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - Schema.Type.MAP, - toBeCreateItems, - sinkRecordList); - - sinkTask.put(sinkRecordList); - - // get all the items - List<String> writtenItemIds = this.getAllItemIds(container); - - assertThat(toBeCreateItems.size()).isEqualTo(writtenItemIds.size()); - List<String> toBeCreateItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(toBeCreateItemIds.containsAll(writtenItemIds)).isTrue(); - - // add the same batch sink records, 409 should be ignored - sinkTask.put(sinkRecordList); - } finally { - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - sinkTask.stop(); - } - } - } - - @Test(groups = { "kafka" }, dataProvider = "bulkEnableParameterProvider", timeOut = 3 * TIMEOUT) - public void sinkWithItemOverwriteIfNotModified(boolean bulkEnabled) { - String topicName = singlePartitionContainerName; - - Map<String, String> sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - sinkConfigMap.put("kafka.connect.cosmos.sink.write.strategy", ItemWriteStrategy.ITEM_OVERWRITE_IF_NOT_MODIFIED.getName()); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - try { - List<SinkRecord> sinkRecordList = new ArrayList<>(); - List<TestItem> toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - Schema.Type.MAP, - toBeCreateItems, - sinkRecordList); - - sinkTask.put(sinkRecordList); - - // get all the items - List<String> writtenItemIds = this.getAllItemIds(container); - - assertThat(toBeCreateItems.size()).isEqualTo(writtenItemIds.size()); - List<String> toBeCreateItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(toBeCreateItemIds.containsAll(writtenItemIds)).isTrue(); - - ObjectNode existedItem = - container - .readItem(toBeCreateItems.get(0).getId(), new PartitionKey(toBeCreateItems.get(0).getMypk()), ObjectNode.class) - .block() - .getItem(); - - // test precondition-failed exception will be ignored - logger.info( - "Testing precondition-failed exception will be ignored for ItemWriteStrategy " - + ItemWriteStrategy.ITEM_OVERWRITE_IF_NOT_MODIFIED.getName()); - - ObjectNode itemWithWrongEtag = Utils.getSimpleObjectMapper().createObjectNode(); - itemWithWrongEtag.setAll(existedItem); - itemWithWrongEtag.put("_etag", UUID.randomUUID().toString()); - SinkRecord sinkRecordWithWrongEtag = - this.getSinkRecord( - topicName, - itemWithWrongEtag, - new ConnectSchema(Schema.Type.STRING), - itemWithWrongEtag.get("id").asText(), - Schema.Type.MAP); - - sinkTask.put(Arrays.asList(sinkRecordWithWrongEtag)); - - // test with correct etag, the item can be modified - logger.info( - "Testing item can be modified with correct etag for ItemWriteStrategy " - + ItemWriteStrategy.ITEM_OVERWRITE_IF_NOT_MODIFIED.getName()); - ObjectNode modifiedItem = Utils.getSimpleObjectMapper().createObjectNode(); - modifiedItem.setAll(existedItem); - modifiedItem.put("prop", UUID.randomUUID().toString()); - SinkRecord sinkRecordWithModifiedItem = - this.getSinkRecord( - topicName, - modifiedItem, - new ConnectSchema(Schema.Type.STRING), - modifiedItem.get("id").asText(), - Schema.Type.MAP); - sinkTask.put(Arrays.asList(sinkRecordWithModifiedItem)); - - existedItem = - container - .readItem(toBeCreateItems.get(0).getId(), new PartitionKey(toBeCreateItems.get(0).getMypk()), ObjectNode.class) - .block() - .getItem(); - assertThat(existedItem.get("prop").asText()).isEqualTo(modifiedItem.get("prop").asText()); - - } finally { - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - sinkTask.stop(); - } - } - } - - @Test(groups = { "kafka" }, dataProvider = "bulkEnableParameterProvider", timeOut = 3 * TIMEOUT) - public void sinkWithItemDelete(boolean bulkEnabled) { - String topicName = singlePartitionContainerName; - - Map<String, String> sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - sinkConfigMap.put("kafka.connect.cosmos.sink.write.strategy", ItemWriteStrategy.ITEM_DELETE.getName()); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - try { - List<SinkRecord> sinkRecordList = new ArrayList<>(); - List<TestItem> toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - Schema.Type.MAP, - toBeCreateItems, - sinkRecordList); - - // first time delete, ignore 404 exceptions - sinkTask.put(sinkRecordList); - - // creating the items in the container - for (TestItem testItem : toBeCreateItems) { - container.createItem(testItem).block(); - } - - // get all the items - List<String> createdItemIds = this.getAllItemIds(container); - - assertThat(toBeCreateItems.size()).isEqualTo(createdItemIds.size()); - List<String> toBeCreateItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(toBeCreateItemIds.containsAll(createdItemIds)).isTrue(); - - // now using the connector to delete the items - sinkTask.put(sinkRecordList); - - // verify all the items have deleted - List<String> existingItemIds = this.getAllItemIds(container); - - assertThat(existingItemIds.isEmpty()).isTrue(); - - } finally { - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - sinkTask.stop(); - } - } - } - - @Test(groups = { "kafka" }, dataProvider = "bulkEnableParameterProvider", timeOut = 3 * TIMEOUT) - public void sinkWithItemDeleteIfNotModified(boolean bulkEnabled) throws InterruptedException { - String topicName = singlePartitionContainerName; - - Map<String, String> sinkConfigMap = new HashMap<>(); - sinkConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sinkConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sinkConfigMap.put("kafka.connect.cosmos.sink.database.name", databaseName); - sinkConfigMap.put("kafka.connect.cosmos.sink.containers.topicMap", topicName + "#" + singlePartitionContainerName); - sinkConfigMap.put("kafka.connect.cosmos.sink.bulk.enabled", String.valueOf(bulkEnabled)); - sinkConfigMap.put("kafka.connect.cosmos.sink.write.strategy", ItemWriteStrategy.ITEM_DELETE_IF_NOT_MODIFIED.getName()); - - CosmosSinkTask sinkTask = new CosmosSinkTask(); - SinkTaskContext sinkTaskContext = Mockito.mock(SinkTaskContext.class); - Mockito.when(sinkTaskContext.errantRecordReporter()).thenReturn(null); - KafkaCosmosReflectionUtils.setSinkTaskContext(sinkTask, sinkTaskContext); - sinkTask.start(sinkConfigMap); - - CosmosAsyncClient cosmosClient = KafkaCosmosReflectionUtils.getSinkTaskCosmosClient(sinkTask); - CosmosContainerProperties singlePartitionContainerProperties = getSinglePartitionContainer(cosmosClient); - CosmosAsyncContainer container = cosmosClient.getDatabase(databaseName).getContainer(singlePartitionContainerProperties.getId()); - - try { - List<SinkRecord> sinkRecordList = new ArrayList<>(); - List<TestItem> toBeCreateItems = new ArrayList<>(); - this.createSinkRecords( - 10, - topicName, - Schema.Type.MAP, - toBeCreateItems, - sinkRecordList); - - // first time delete, ignore 404 exceptions - sinkTask.put(sinkRecordList); - - // creating the items in the container - for (TestItem testItem : toBeCreateItems) { - container.createItem(testItem).block(); - } - - // get all the items - List<ObjectNode> createdItems = this.getAllItems(container); - List<String> createdItemIds = - createdItems - .stream() - .map(objectNode -> objectNode.get("id").asText()) - .collect(Collectors.toList()); - List<String> expectItemIds = toBeCreateItems.stream().map(TestItem::getId).collect(Collectors.toList()); - assertThat(toBeCreateItems.size()).isEqualTo(createdItemIds.size()); - assertThat(expectItemIds.containsAll(createdItemIds)).isTrue(); - - // using wrong etag to delete the items, verify no item will be deleted - List<SinkRecord> sinkRecordsWithWrongEtag = new ArrayList<>(); - for (ObjectNode createdItem : createdItems) { - ObjectNode testItemWithWrongEtag = Utils.getSimpleObjectMapper().createObjectNode(); - testItemWithWrongEtag.setAll(createdItem); - testItemWithWrongEtag.put("_etag", UUID.randomUUID().toString()); - sinkRecordsWithWrongEtag.add( - this.getSinkRecord( - topicName, - testItemWithWrongEtag, - new ConnectSchema(Schema.Type.STRING), - createdItem.get("id").asText(), - Schema.Type.STRUCT) - ); - } - sinkTask.put(sinkRecordsWithWrongEtag); - Thread.sleep(500); // delete happens in the background - List<String> existingItemIds = this.getAllItemIds(container); - assertThat(existingItemIds.size()).isEqualTo(createdItemIds.size()); - assertThat(existingItemIds.containsAll(createdItemIds)).isTrue(); - - // verify all the items have deleted - List<SinkRecord> sinkRecordsWithCorrectEtag = new ArrayList<>(); - for (ObjectNode createdItem : createdItems) { - sinkRecordsWithCorrectEtag.add( - this.getSinkRecord( - topicName, - createdItem, - new ConnectSchema(Schema.Type.STRING), - createdItem.get("id").asText(), - Schema.Type.STRUCT) - ); - } - - sinkTask.put(sinkRecordsWithCorrectEtag); - Thread.sleep(500); // delete happens in the background - existingItemIds = this.getAllItemIds(container); - assertThat(existingItemIds.isEmpty()).isTrue(); - - } finally { - if (cosmosClient != null) { - cleanUpContainer(cosmosClient, databaseName, singlePartitionContainerProperties.getId()); - sinkTask.stop(); - } - } - } - - private SinkRecord getSinkRecord( - String topicName, - ObjectNode objectNode, - Schema keySchema, - String keyValue, - Schema.Type valueSchemaType) { - if (valueSchemaType == Schema.Type.STRUCT) { - SchemaAndValue schemaAndValue = - JsonToStruct.recordToSchemaAndValue(objectNode); - - return new SinkRecord( - topicName, - 1, - keySchema, - keyValue, - schemaAndValue.schema(), - schemaAndValue.value(), - 0L); - } else { - return new SinkRecord( - topicName, - 1, - keySchema, - keyValue, - new ConnectSchema(Schema.Type.MAP), - Utils.getSimpleObjectMapper().convertValue(objectNode, new TypeReference<Map<String, Object>>() {}), - 0L); - } - } - - private void createSinkRecords( - int numberOfItems, - String topicName, - Schema.Type valueSchemaType, - List<TestItem> createdItems, - List<SinkRecord> sinkRecordList) { - - Schema keySchema = new ConnectSchema(Schema.Type.STRING); - - for (int i = 0; i < numberOfItems; i++) { - TestItem testItem = TestItem.createNewItem(); - createdItems.add(testItem); - - SinkRecord sinkRecord = - this.getSinkRecord( - topicName, - Utils.getSimpleObjectMapper().convertValue(testItem, ObjectNode.class), - keySchema, - testItem.getId(), - valueSchemaType); - sinkRecordList.add(sinkRecord); - } - } - - private List<String> getAllItemIds(CosmosAsyncContainer container) { - return getAllItems(container) - .stream() - .map(objectNode -> objectNode.get("id").asText()) - .collect(Collectors.toList()); - } - - private List<ObjectNode> getAllItems(CosmosAsyncContainer container) { - String query = "select * from c"; - return container.queryItems(query, ObjectNode.class) - .byPage() - .flatMapIterable(response -> response.getResults()) - .collectList() - .block(); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/idStrategy/ProvidedInStrategyTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/idStrategy/ProvidedInStrategyTest.java deleted file mode 100644 index 19c8fb92dd37..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/sink/idStrategy/ProvidedInStrategyTest.java +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.sink.idStrategy; - -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.IdStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.ProvidedInConfig; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.ProvidedInKeyStrategy; -import com.azure.cosmos.kafka.connect.implementation.sink.idstrategy.ProvidedInValueStrategy; -import org.apache.kafka.connect.data.Schema; -import org.apache.kafka.connect.data.SchemaBuilder; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.errors.ConnectException; -import org.apache.kafka.connect.sink.SinkRecord; -import org.mockito.Mockito; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.mockito.Mockito.when; - -public class ProvidedInStrategyTest { - protected static final int TIMEOUT = 60000; - - @DataProvider(name = "idStrategyParameterProvider") - public static Object[][] idStrategyParameterProvider() { - return new Object[][]{ - { new ProvidedInValueStrategy() }, - { new ProvidedInKeyStrategy() }, - }; - } - - private void returnOnKeyOrValue( - Schema schema, - Object ret, - IdStrategy idStrategy, - SinkRecord sinkRecord) { - if (idStrategy.getClass() == ProvidedInKeyStrategy.class) { - when(sinkRecord.keySchema()).thenReturn(schema); - when(sinkRecord.key()).thenReturn(ret); - } else { - when(sinkRecord.valueSchema()).thenReturn(schema); - when(sinkRecord.value()).thenReturn(ret); - } - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", expectedExceptions = ConnectException.class, timeOut = TIMEOUT) - public void valueNotStructOrMapShouldFail(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue(Schema.STRING_SCHEMA, "a string", idStrategy, sinkRecord); - idStrategy.generateId(sinkRecord); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", expectedExceptions = ConnectException.class, timeOut = TIMEOUT) - public void noIdInValueShouldFail(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue(null, new HashMap<>(), idStrategy, sinkRecord); - idStrategy.generateId(sinkRecord); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void stringIdOnMapShouldReturn(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap<String, Object>(){{ put("id", "1234567"); }}, - idStrategy, - sinkRecord); - assertThat("1234567").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void nonStringIdOnMapShouldReturn(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap<String, Object>(){{ put("id", 1234567); }}, - idStrategy, - sinkRecord); - assertThat("1234567").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void stringIdOnStructShouldReturn(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - Schema schema = SchemaBuilder.struct() - .field("id", Schema.STRING_SCHEMA) - .build(); - - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - Struct struct = new Struct(schema).put("id", "1234567"); - returnOnKeyOrValue(struct.schema(), struct, idStrategy, sinkRecord); - - assertThat("1234567").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void structIdOnStructShouldReturn(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - Schema idSchema = SchemaBuilder.struct() - .field("name", Schema.STRING_SCHEMA) - .build(); - Schema schema = SchemaBuilder.struct() - .field("id", idSchema) - .build(); - Struct struct = new Struct(schema) - .put("id", new Struct(idSchema).put("name", "cosmos kramer")); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue(struct.schema(), struct, idStrategy, sinkRecord); - - assertThat("{\"name\":\"cosmos kramer\"}").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void jsonPathOnStruct(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<String, Object>(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id.name"); }}); - - Schema idSchema = SchemaBuilder.struct() - .field("name", Schema.STRING_SCHEMA) - .build(); - Schema schema = SchemaBuilder.struct() - .field("id", idSchema) - .build(); - Struct struct = new Struct(schema) - .put("id", new Struct(idSchema).put("name", "franz kafka")); - - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue(struct.schema(), struct, idStrategy, sinkRecord); - assertThat("franz kafka").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void jsonPathOnMap(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<String, Object>(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id.name"); }}); - - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap<String, Object>(){{ put("id", new HashMap<String, Object>(){{ put("name", "franz kafka"); }}); }}, - idStrategy, - sinkRecord); - assertThat("franz kafka").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", expectedExceptions = ConnectException.class, timeOut = TIMEOUT) - public void invalidJsonPathThrows(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<String, Object>(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "invalid.path"); }}); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap<String, Object>(){{ put("id", new HashMap<String, Object>(){{ put("name", "franz kafka"); }}); }}, - idStrategy, - sinkRecord); - - idStrategy.generateId(sinkRecord); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", expectedExceptions = ConnectException.class, timeOut = TIMEOUT) - public void jsonPathNotExistThrows(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<String, Object>(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id.not.exist"); }}); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap<String, Object>(){{ put("id", new HashMap<String, Object>(){{ put("name", "franz kafka"); }}); }}, - idStrategy, - sinkRecord); - - idStrategy.generateId(sinkRecord); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void complexJsonPath(IdStrategy idStrategy) { - Map<String, Object> map1 = new LinkedHashMap<>(); - map1.put("id", 0); - map1.put("name", "cosmos kramer"); - map1.put("occupation", "unknown"); - Map<String, Object> map2 = new LinkedHashMap<>(); - map2.put("id", 1); - map2.put("name", "franz kafka"); - map2.put("occupation", "writer"); - - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap<String, Object>() {{ put("id", Arrays.asList(map1, map2)); }}, - idStrategy, - sinkRecord); - - idStrategy.configure(new HashMap<String, Object>(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id[0].name"); }}); - assertThat("cosmos kramer").isEqualTo(idStrategy.generateId(sinkRecord)); - - idStrategy.configure(new HashMap<String, Object>(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id[1].name"); }}); - assertThat("franz kafka").isEqualTo(idStrategy.generateId(sinkRecord)); - - idStrategy.configure(new HashMap<String, Object>(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id[*].id"); }}); - assertThat("[0,1]").isEqualTo(idStrategy.generateId(sinkRecord)); - - idStrategy.configure(new HashMap<String, Object>(){{ put(ProvidedInConfig.JSON_PATH_CONFIG, "$.id"); }}); - assertThat("[{\"id\":0,\"name\":\"cosmos kramer\",\"occupation\":\"unknown\"},{\"id\":1,\"name\":\"franz kafka\",\"occupation\":\"writer\"}]").isEqualTo(idStrategy.generateId(sinkRecord)); - } - - @Test(groups = { "unit" }, dataProvider = "idStrategyParameterProvider", timeOut = TIMEOUT) - public void generatedIdSanitized(IdStrategy idStrategy) { - idStrategy.configure(new HashMap<>()); - SinkRecord sinkRecord = Mockito.mock(SinkRecord.class); - returnOnKeyOrValue( - null, - new HashMap<String, Object>() {{put("id", "#my/special\\id?");}}, - idStrategy, - sinkRecord); - - assertThat("_my_special_id_").isEqualTo(idStrategy.generateId(sinkRecord)); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTaskTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTaskTest.java deleted file mode 100644 index e137624fe22d..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/CosmosSourceTaskTest.java +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.CosmosAsyncContainer; -import com.azure.cosmos.implementation.TestConfigurations; -import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; -import com.azure.cosmos.implementation.routing.Range; -import com.azure.cosmos.kafka.connect.KafkaCosmosTestSuiteBase; -import com.azure.cosmos.kafka.connect.TestItem; -import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.models.CosmosContainerProperties; -import com.azure.cosmos.models.CosmosQueryRequestOptions; -import com.azure.cosmos.models.FeedRange; -import com.azure.cosmos.models.ThroughputProperties; -import com.azure.cosmos.models.ThroughputResponse; -import org.apache.kafka.connect.data.Struct; -import org.apache.kafka.connect.source.SourceRecord; -import org.testng.annotations.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -public class CosmosSourceTaskTest extends KafkaCosmosTestSuiteBase { - private final int CONTAINER_THROUGHPUT_FOR_SPLIT = 10100; - - @Test(groups = {"kafka"}, timeOut = 10 * TIMEOUT) - public void poll() throws InterruptedException { - String testContainerName = "KafkaCosmosTestPoll-" + UUID.randomUUID(); - Map<String, String> sourceConfigMap = new HashMap<>(); - sourceConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sourceConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sourceConfigMap.put("kafka.connect.cosmos.source.database.name", databaseName); - List<String> containersIncludedList = Arrays.asList(testContainerName); - sourceConfigMap.put("kafka.connect.cosmos.source.containers.includedList", containersIncludedList.toString()); - - CosmosSourceConfig sourceConfig = new CosmosSourceConfig(sourceConfigMap); - CosmosAsyncClient client = CosmosClientStore.getCosmosClient(sourceConfig.getAccountConfig()); - - // create a new container as we are going to trigger split as well, isolate the possible impact for other tests - CosmosContainerProperties testContainer = - client - .getDatabase(databaseName) - .createContainer(testContainerName, "/id") - .block() - .getProperties(); - - try { - Map<String, String> taskConfigMap = sourceConfig.originalsStrings(); - - // define metadata task - List<FeedRange> feedRanges = - client.getDatabase(databaseName).getContainer(testContainerName).getFeedRanges().block(); - assertThat(feedRanges.size()).isEqualTo(1); - - Map<String, List<Range<String>>> containersEffectiveRangesMap = new HashMap<>(); - containersEffectiveRangesMap.put(testContainer.getResourceId(), Arrays.asList(FeedRangeEpkImpl.forFullRange().getRange())); - MetadataTaskUnit metadataTaskUnit = new MetadataTaskUnit( - databaseName, - Arrays.asList(testContainer.getResourceId()), - containersEffectiveRangesMap, - testContainerName); - taskConfigMap.putAll(CosmosSourceTaskConfig.getMetadataTaskUnitConfigMap(metadataTaskUnit)); - - // define feedRanges task - FeedRangeTaskUnit feedRangeTaskUnit = new FeedRangeTaskUnit( - databaseName, - testContainerName, - testContainer.getResourceId(), - FeedRangeEpkImpl.forFullRange().getRange(), - null, - testContainerName); - taskConfigMap.putAll(CosmosSourceTaskConfig.getFeedRangeTaskUnitsConfigMap(Arrays.asList(feedRangeTaskUnit))); - - CosmosSourceTask sourceTask = new CosmosSourceTask(); - sourceTask.start(taskConfigMap); - - // first creating few items in the container - List<TestItem> createdItems = this.createItems(client, databaseName, testContainerName, 10); - - List<SourceRecord> sourceRecords = sourceTask.poll(); - // Since there are metadata task unit being defined, we expected to get the metadata records first. - validateMetadataRecords(sourceRecords, metadataTaskUnit); - - sourceRecords = sourceTask.poll(); - validateFeedRangeRecords(sourceRecords, createdItems); - - logger.info("Testing split..."); - // trigger split - ThroughputResponse throughputResponse = - client - .getDatabase(databaseName) - .getContainer(testContainerName) - .replaceThroughput(ThroughputProperties.createManualThroughput(CONTAINER_THROUGHPUT_FOR_SPLIT)) - .block(); - - // Wait for the throughput update to complete so that we get the partition split - while (true) { - assert throughputResponse != null; - if (!throughputResponse.isReplacePending()) { - break; - } - logger.info("Waiting for split to complete"); - Thread.sleep(10 * 1000); - throughputResponse = client.getDatabase(databaseName).getContainer(testContainerName).readThroughput().block(); - } - - createdItems = this.createItems(client, databaseName, testContainerName, 10); - sourceRecords = new ArrayList<>(); - // the first poll will return 0 records as it will be the first time the task detect split happened - // internally it will create two new feedRange task units - // so here we will need to poll 3 times to get all newly created items - for (int i = 0; i < 3; i++) { - sourceRecords.addAll(sourceTask.poll()); - } - validateFeedRangeRecords(sourceRecords, createdItems); - } finally { - if (client != null) { - client.getDatabase(databaseName).getContainer(testContainerName).delete().block(); - client.close(); - } - } - } - - @Test(groups = { "kafka" }, timeOut = TIMEOUT) - public void pollWithSpecificFeedRange() { - // Test only items belong to the feedRange defined in the feedRangeTaskUnit will be returned - Map<String, String> sourceConfigMap = new HashMap<>(); - sourceConfigMap.put("kafka.connect.cosmos.accountEndpoint", TestConfigurations.HOST); - sourceConfigMap.put("kafka.connect.cosmos.accountKey", TestConfigurations.MASTER_KEY); - sourceConfigMap.put("kafka.connect.cosmos.source.database.name", databaseName); - List<String> containersIncludedList = Arrays.asList(multiPartitionContainerName); - sourceConfigMap.put("kafka.connect.cosmos.source.containers.includedList", containersIncludedList.toString()); - - CosmosSourceConfig sourceConfig = new CosmosSourceConfig(sourceConfigMap); - CosmosAsyncClient client = CosmosClientStore.getCosmosClient(sourceConfig.getAccountConfig()); - - try { - Map<String, String> taskConfigMap = sourceConfig.originalsStrings(); - - // define metadata task - List<FeedRange> feedRanges = - client.getDatabase(databaseName).getContainer(multiPartitionContainerName).getFeedRanges().block(); - CosmosContainerProperties multiPartitionContainer = getMultiPartitionContainer(client); - assertThat(feedRanges.size()).isGreaterThan(1); - - // define feedRanges task - FeedRangeTaskUnit feedRangeTaskUnit = new FeedRangeTaskUnit( - databaseName, - multiPartitionContainer.getId(), - multiPartitionContainer.getResourceId(), - ((FeedRangeEpkImpl)feedRanges.get(0)).getRange(), - null, - multiPartitionContainer.getId()); - taskConfigMap.putAll(CosmosSourceTaskConfig.getFeedRangeTaskUnitsConfigMap(Arrays.asList(feedRangeTaskUnit))); - - CosmosSourceTask sourceTask = new CosmosSourceTask(); - sourceTask.start(taskConfigMap); - - // first creating few items in the container - this.createItems(client, databaseName, multiPartitionContainer.getId(), 10); - - List<SourceRecord> sourceRecords = new ArrayList<>(); - for (int i = 0; i < 3; i++) { // poll few times - sourceRecords.addAll(sourceTask.poll()); - } - - // get all items belong to feed range 0 - CosmosQueryRequestOptions queryRequestOptions = new CosmosQueryRequestOptions(); - queryRequestOptions.setFeedRange(feedRanges.get(0)); - List<TestItem> expectedItems = client - .getDatabase(databaseName) - .getContainer(multiPartitionContainer.getId()) - .queryItems("select * from c", queryRequestOptions, TestItem.class) - .byPage() - .flatMapIterable(feedResponse -> feedResponse.getResults()) - .collectList() - .block(); - - validateFeedRangeRecords(sourceRecords, expectedItems); - } finally { - if (client != null) { - // clean up containers - cleanUpContainer(client, databaseName, multiPartitionContainerName); - client.close(); - } - } - } - - private void validateMetadataRecords(List<SourceRecord> sourceRecords, MetadataTaskUnit metadataTaskUnit) { - // one containers metadata - // one feedRanges metadata record for each container - assertThat(sourceRecords.size()).isEqualTo(metadataTaskUnit.getContainerRids().size() + 1); - - ContainersMetadataTopicPartition containersMetadataTopicPartition = - new ContainersMetadataTopicPartition(metadataTaskUnit.getDatabaseName()); - ContainersMetadataTopicOffset containersMetadataTopicOffset = - new ContainersMetadataTopicOffset(metadataTaskUnit.getContainerRids()); - assertThat(sourceRecords.get(0).sourcePartition()).isEqualTo(ContainersMetadataTopicPartition.toMap(containersMetadataTopicPartition)); - assertThat(sourceRecords.get(0).sourceOffset()).isEqualTo(ContainersMetadataTopicOffset.toMap(containersMetadataTopicOffset)); - - for (int i = 0; i < metadataTaskUnit.getContainerRids().size(); i++) { - String containerRid = metadataTaskUnit.getContainerRids().get(i); - SourceRecord sourceRecord = sourceRecords.get(i + 1); - List<Range<String>> containerFeedRanges = - metadataTaskUnit.getContainersEffectiveRangesMap().get(containerRid); - assertThat(containerFeedRanges).isNotNull(); - - FeedRangesMetadataTopicPartition feedRangesMetadataTopicPartition = - new FeedRangesMetadataTopicPartition(metadataTaskUnit.getDatabaseName(), containerRid); - FeedRangesMetadataTopicOffset feedRangesMetadataTopicOffset = - new FeedRangesMetadataTopicOffset(containerFeedRanges); - assertThat(sourceRecord.sourcePartition()).isEqualTo(FeedRangesMetadataTopicPartition.toMap(feedRangesMetadataTopicPartition)); - assertThat(sourceRecord.sourceOffset()).isEqualTo(FeedRangesMetadataTopicOffset.toMap(feedRangesMetadataTopicOffset)); - } - } - - private void validateFeedRangeRecords(List<SourceRecord> sourceRecords, List<TestItem> expectedItems) { - List<String> idsReceived = - sourceRecords - .stream() - .map(sourceRecord -> ((Struct)sourceRecord.value()).get("id").toString()) - .collect(Collectors.toList()); - List<String> expectedIds = - expectedItems - .stream() - .map(testItem -> testItem.getId()) - .collect(Collectors.toList()); - assertThat(idsReceived.size()).isEqualTo(expectedItems.size()); - assertThat(idsReceived.containsAll(expectedIds)); - } - - private List<TestItem> createItems( - CosmosAsyncClient client, - String databaseName, - String containerName, - int numberOfItems) { - - List<TestItem> testItems = new ArrayList<>(); - CosmosAsyncContainer container = client.getDatabase(databaseName).getContainer(containerName); - for (int i = 0; i < numberOfItems; i++) { - TestItem testItem = TestItem.createNewItem(); - container.createItem(testItem).block(); - testItems.add(testItem); - } - - return testItems; - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThreadTest.java b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThreadTest.java deleted file mode 100644 index 76479de10981..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/java/com/azure/cosmos/kafka/connect/implementation/source/MetadataMonitorThreadTest.java +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package com.azure.cosmos.kafka.connect.implementation.source; - -import com.azure.cosmos.CosmosAsyncClient; -import com.azure.cosmos.implementation.ImplementationBridgeHelpers; -import com.azure.cosmos.implementation.TestConfigurations; -import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; -import com.azure.cosmos.kafka.connect.InMemoryStorageReader; -import com.azure.cosmos.kafka.connect.KafkaCosmosTestSuiteBase; -import com.azure.cosmos.kafka.connect.implementation.CosmosAccountConfig; -import com.azure.cosmos.kafka.connect.implementation.CosmosClientStore; -import com.azure.cosmos.models.CosmosContainerProperties; -import com.azure.cosmos.models.FeedRange; -import org.apache.kafka.connect.source.SourceConnectorContext; -import org.mockito.Mockito; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.assertj.core.api.AssertionsForClassTypes.assertThat; - -public class MetadataMonitorThreadTest extends KafkaCosmosTestSuiteBase { - private CosmosAsyncClient client; - @BeforeClass(groups = {"kafka"}, timeOut = TIMEOUT) - public void before_MetadataMonitorThreadTest() { - CosmosAccountConfig accountConfig = new CosmosAccountConfig( - TestConfigurations.HOST, - TestConfigurations.MASTER_KEY, - "requestTaskReconfigurationTest", - false, - new ArrayList<String>()); - this.client = CosmosClientStore.getCosmosClient(accountConfig); - } - - @AfterClass(groups = {"kafka"}, timeOut = TIMEOUT) - public void after_MetadataMonitorThreadTest() { - if (this.client != null) { - this.client.close(); - } - } - - @Test(groups = "{ kafka }", timeOut = TIMEOUT) - public void requestTaskReconfigurationOnContainersChange() throws InterruptedException { - - CosmosSourceContainersConfig cosmosSourceContainersConfig = - new CosmosSourceContainersConfig( - databaseName, - true, - new ArrayList<String>(), - new HashMap<String, String>()); - CosmosMetadataConfig metadataConfig = - new CosmosMetadataConfig(500, "_cosmos.metadata.topic"); - SourceConnectorContext sourceConnectorContext = Mockito.mock(SourceConnectorContext.class); - InMemoryStorageReader inMemoryStorageReader = new InMemoryStorageReader(); - CosmosSourceOffsetStorageReader sourceOffsetStorageReader = new CosmosSourceOffsetStorageReader(inMemoryStorageReader); - - MetadataMonitorThread monitorThread = - new MetadataMonitorThread( - cosmosSourceContainersConfig, - metadataConfig, - sourceConnectorContext, - sourceOffsetStorageReader, - this.client); - - monitorThread.run(); - - Thread.sleep(1000); - // Since there is no offset yet, no requestTaskReconfiguration will happen - Mockito.verify(sourceConnectorContext, Mockito.never()).requestTaskReconfiguration(); - - // now populate containers metadata offset - CosmosContainerProperties singlePartitionContainer = getSinglePartitionContainer(this.client); - ContainersMetadataTopicPartition containersMetadataTopicPartition = - new ContainersMetadataTopicPartition(databaseName); - ContainersMetadataTopicOffset containersMetadataTopicOffset = - new ContainersMetadataTopicOffset(Arrays.asList(singlePartitionContainer.getResourceId())); - - Map<Map<String, Object>, Map<String, Object>> offsetMap = new HashMap<>(); - offsetMap.put( - ContainersMetadataTopicPartition.toMap(containersMetadataTopicPartition), - ContainersMetadataTopicOffset.toMap(containersMetadataTopicOffset)); - - inMemoryStorageReader.populateOffset(offsetMap); - - Thread.sleep(5000); // give enough time to do the containers query request - monitorThread.close(); - - Mockito.verify(sourceConnectorContext, Mockito.atLeastOnce()).requestTaskReconfiguration(); - } - - @Test(groups = "{ kafka }", timeOut = TIMEOUT) - public void requestTaskReconfigurationOnSplit() throws InterruptedException { - - CosmosSourceContainersConfig cosmosSourceContainersConfig = - new CosmosSourceContainersConfig( - databaseName, - false, - Arrays.asList(multiPartitionContainerName), - new HashMap<String, String>()); - CosmosMetadataConfig metadataConfig = - new CosmosMetadataConfig(500, "_cosmos.metadata.topic"); - SourceConnectorContext sourceConnectorContext = Mockito.mock(SourceConnectorContext.class); - - InMemoryStorageReader inMemoryStorageReader = new InMemoryStorageReader(); - CosmosSourceOffsetStorageReader sourceOffsetStorageReader = new CosmosSourceOffsetStorageReader(inMemoryStorageReader); - - //populate containers metadata offset - CosmosContainerProperties multiPartitionContainer = getMultiPartitionContainer(this.client); - ContainersMetadataTopicPartition containersMetadataTopicPartition = - new ContainersMetadataTopicPartition(databaseName); - ContainersMetadataTopicOffset containersMetadataTopicOffset = - new ContainersMetadataTopicOffset(Arrays.asList(multiPartitionContainer.getResourceId())); - Map<Map<String, Object>, Map<String, Object>> offsetMap = new HashMap<>(); - offsetMap.put( - ContainersMetadataTopicPartition.toMap(containersMetadataTopicPartition), - ContainersMetadataTopicOffset.toMap(containersMetadataTopicOffset)); - - inMemoryStorageReader.populateOffset(offsetMap); - - MetadataMonitorThread monitorThread = - new MetadataMonitorThread( - cosmosSourceContainersConfig, - metadataConfig, - sourceConnectorContext, - sourceOffsetStorageReader, - this.client); - - monitorThread.run(); - - Thread.sleep(2000); // give some time for the query containers requests - // Since there is no offset yet, no requestTaskReconfiguration will happen - Mockito.verify(sourceConnectorContext, Mockito.never()).requestTaskReconfiguration(); - - // now populate container feedRanges metadata - List<FeedRange> feedRanges = - this.client - .getDatabase(databaseName) - .getContainer(multiPartitionContainer.getId()) - .getFeedRanges() - .block(); - assertThat(feedRanges.size()).isGreaterThan(1); - - FeedRangesMetadataTopicPartition feedRangesMetadataTopicPartition = - new FeedRangesMetadataTopicPartition(databaseName, multiPartitionContainer.getResourceId()); - FeedRangesMetadataTopicOffset feedRangesMetadataTopicOffset = - new FeedRangesMetadataTopicOffset(Arrays.asList(FeedRangeEpkImpl.forFullRange().getRange())); - - Map<Map<String, Object>, Map<String, Object>> feedRangesOffSetMap = new HashMap<>(); - feedRangesOffSetMap.put( - FeedRangesMetadataTopicPartition.toMap(feedRangesMetadataTopicPartition), - FeedRangesMetadataTopicOffset.toMap(feedRangesMetadataTopicOffset)); - - inMemoryStorageReader.populateOffset(feedRangesOffSetMap); - - Thread.sleep(5000); // give enough time for the containers query and feedRanges request - monitorThread.close(); - - // for merge, no task reconfiguration is needed - Mockito.verify(sourceConnectorContext, Mockito.atLeastOnce()).requestTaskReconfiguration(); - } - - @Test(groups = "{ kafka }", timeOut = TIMEOUT) - public void requestTaskReconfigurationOnMerge() throws InterruptedException { - - CosmosSourceContainersConfig cosmosSourceContainersConfig = - new CosmosSourceContainersConfig( - databaseName, - false, - Arrays.asList(singlePartitionContainerName), - new HashMap<String, String>()); - CosmosMetadataConfig metadataConfig = - new CosmosMetadataConfig(500, "_cosmos.metadata.topic"); - SourceConnectorContext sourceConnectorContext = Mockito.mock(SourceConnectorContext.class); - - InMemoryStorageReader inMemoryStorageReader = new InMemoryStorageReader(); - CosmosSourceOffsetStorageReader sourceOffsetStorageReader = new CosmosSourceOffsetStorageReader(inMemoryStorageReader); - - //populate containers metadata offset - CosmosContainerProperties singlePartitionContainer = getSinglePartitionContainer(this.client); - ContainersMetadataTopicPartition containersMetadataTopicPartition = - new ContainersMetadataTopicPartition(databaseName); - ContainersMetadataTopicOffset containersMetadataTopicOffset = - new ContainersMetadataTopicOffset(Arrays.asList(singlePartitionContainer.getResourceId())); - Map<Map<String, Object>, Map<String, Object>> offsetMap = new HashMap<>(); - offsetMap.put( - ContainersMetadataTopicPartition.toMap(containersMetadataTopicPartition), - ContainersMetadataTopicOffset.toMap(containersMetadataTopicOffset)); - - inMemoryStorageReader.populateOffset(offsetMap); - - MetadataMonitorThread monitorThread = - new MetadataMonitorThread( - cosmosSourceContainersConfig, - metadataConfig, - sourceConnectorContext, - sourceOffsetStorageReader, - this.client); - - monitorThread.run(); - - Thread.sleep(2000); // give some time for the query containers requests - // Since there is no offset yet, no requestTaskReconfiguration will happen - Mockito.verify(sourceConnectorContext, Mockito.never()).requestTaskReconfiguration(); - - // now populate container feedRanges metadata - List<FeedRange> feedRanges = - this.client - .getDatabase(databaseName) - .getContainer(singlePartitionContainer.getId()) - .getFeedRanges() - .block(); - assertThat(feedRanges.size()).isEqualTo(1); - - List<FeedRangeEpkImpl> childRanges = - ImplementationBridgeHelpers - .CosmosAsyncContainerHelper - .getCosmosAsyncContainerAccessor() - .trySplitFeedRange( - this.client.getDatabase(databaseName).getContainer(singlePartitionContainer.getId()), - FeedRange.forFullRange(), - 2) - .block(); - - FeedRangesMetadataTopicPartition feedRangesMetadataTopicPartition = - new FeedRangesMetadataTopicPartition(databaseName, singlePartitionContainer.getResourceId()); - FeedRangesMetadataTopicOffset feedRangesMetadataTopicOffset = - new FeedRangesMetadataTopicOffset( - childRanges - .stream() - .map(FeedRangeEpkImpl::getRange) - .collect(Collectors.toList())); - - Map<Map<String, Object>, Map<String, Object>> feedRangesOffSetMap = new HashMap<>(); - feedRangesOffSetMap.put( - FeedRangesMetadataTopicPartition.toMap(feedRangesMetadataTopicPartition), - FeedRangesMetadataTopicOffset.toMap(feedRangesMetadataTopicOffset)); - - inMemoryStorageReader.populateOffset(feedRangesOffSetMap); - - Thread.sleep(5000); // give enough time for the containers query and feedRanges request - monitorThread.close(); - - // for merge, no task reconfiguration is needed - Mockito.verify(sourceConnectorContext, Mockito.never()).requestTaskReconfiguration(); - } -} diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/resources/kafka-testng.xml b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/resources/kafka-testng.xml deleted file mode 100644 index 4d30f16b4df5..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/resources/kafka-testng.xml +++ /dev/null @@ -1,35 +0,0 @@ -<!-- - ~ The MIT License (MIT) - ~ Copyright (c) 2018 Microsoft Corporation - ~ - ~ Permission is hereby granted, free of charge, to any person obtaining a copy - ~ of this software and associated documentation files (the "Software"), to deal - ~ in the Software without restriction, including without limitation the rights - ~ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - ~ copies of the Software, and to permit persons to whom the Software is - ~ furnished to do so, subject to the following conditions: - ~ - ~ The above copyright notice and this permission notice shall be included in all - ~ copies or substantial portions of the Software. - ~ - ~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - ~ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - ~ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - ~ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - ~ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - ~ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - ~ SOFTWARE. - --> -<!DOCTYPE suite SYSTEM "https://testng.org/testng-1.0.dtd"> -<suite name="kafka"> - <test name="kafka" group-by-instances="true"> - <groups> - <run> - <include name="kafka"/> - </run> - </groups> - <packages> - <package name="com.azure.cosmos.kafka.connect.*"/> - </packages> - </test> -</suite> diff --git a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/resources/log4j2.properties b/sdk/cosmos/azure-cosmos-kafka-connect/src/test/resources/log4j2.properties deleted file mode 100644 index b68efa3afd76..000000000000 --- a/sdk/cosmos/azure-cosmos-kafka-connect/src/test/resources/log4j2.properties +++ /dev/null @@ -1,24 +0,0 @@ -# Set root logger level to INFO and its default appender to be 'STDOUT'. -rootLogger.level = info -rootLogger.appenderRef.stdout.ref = STDOUT - -# Uncomment here and lines 21 - 25 to enable logging to a file as well. -# rootLogger.appenderRef.logFile.ref = FILE - -property.logDirectory = $${sys:azure.cosmos.logger.directory} -property.hostName = $${sys:azure.cosmos.hostname} - -logger.netty.name = io.netty -logger.netty.level = off - -# STDOUT is a ConsoleAppender and uses PatternLayout. -appender.console.name = STDOUT -appender.console.type = Console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d %5X{pid} [%t] %-5p %c - %m%n - -# appender.logfile.name = FILE -# appender.logfile.type = File -# appender.logfile.filename = ${logDirectory}/azure-cosmos-benchmark.log -# appender.logfile.layout.type = PatternLayout -# appender.logfile.layout.pattern = [%d][%p][${hostName}][thread:%t][logger:%c] %m%n diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java index bd7a295bc9ae..bc4de6cf7917 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/CosmosAsyncContainer.java @@ -2775,20 +2775,6 @@ public <T> Function<CosmosPagedFluxOptions, Flux<FeedResponse<T>>> queryItemsInt public Mono<List<FeedRange>> getFeedRanges(CosmosAsyncContainer cosmosAsyncContainer, boolean forceRefresh) { return cosmosAsyncContainer.getFeedRanges(forceRefresh); } - - @Override - public Mono<List<FeedRangeEpkImpl>> trySplitFeedRange( - CosmosAsyncContainer cosmosAsyncContainer, - FeedRange feedRange, - int targetedCountAfterSplit) { - - return cosmosAsyncContainer.trySplitFeedRange(feedRange, targetedCountAfterSplit); - } - - @Override - public String getLinkWithoutTrailingSlash(CosmosAsyncContainer cosmosAsyncContainer) { - return cosmosAsyncContainer.getLinkWithoutTrailingSlash(); - } }); } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java index 8b2c937e4915..1f23419c63eb 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/ImplementationBridgeHelpers.java @@ -37,7 +37,6 @@ import com.azure.cosmos.implementation.directconnectivity.Uri; import com.azure.cosmos.implementation.directconnectivity.rntbd.RntbdChannelStatistics; import com.azure.cosmos.implementation.faultinjection.IFaultInjectorProvider; -import com.azure.cosmos.implementation.feedranges.FeedRangeEpkImpl; import com.azure.cosmos.implementation.patch.PatchOperation; import com.azure.cosmos.implementation.routing.PartitionKeyInternal; import com.azure.cosmos.implementation.spark.OperationContextAndListenerTuple; @@ -941,13 +940,6 @@ <T> Function<CosmosPagedFluxOptions, Flux<FeedResponse<T>>> queryItemsInternalFu Class<T> classType); Mono<List<FeedRange>> getFeedRanges(CosmosAsyncContainer cosmosAsyncContainer, boolean forceRefresh); - - Mono<List<FeedRangeEpkImpl>> trySplitFeedRange( - CosmosAsyncContainer cosmosAsyncContainer, - FeedRange feedRange, - int targetedCountAfterSplit); - - String getLinkWithoutTrailingSlash(CosmosAsyncContainer cosmosAsyncContainer); } } diff --git a/sdk/cosmos/azure-cosmos/src/main/java/module-info.java b/sdk/cosmos/azure-cosmos/src/main/java/module-info.java index 639cf5023f5e..baeb43da9e50 100644 --- a/sdk/cosmos/azure-cosmos/src/main/java/module-info.java +++ b/sdk/cosmos/azure-cosmos/src/main/java/module-info.java @@ -34,21 +34,21 @@ exports com.azure.cosmos.util; // export packages for multiple different modules - exports com.azure.cosmos.implementation to com.azure.cosmos.encryption, com.azure.cosmos.test, com.azure.cosmos.kafka.connect; - exports com.azure.cosmos.implementation.caches to com.azure.cosmos.encryption, com.azure.cosmos.test, com.azure.cosmos.kafka.connect; - exports com.azure.cosmos.implementation.feedranges to com.azure.cosmos.encryption, com.azure.cosmos.test, com.azure.cosmos.kafka.connect; - exports com.azure.cosmos.implementation.apachecommons.lang to com.azure.cosmos.encryption, com.azure.cosmos.test, com.azure.cosmos.kafka.connect; - exports com.azure.cosmos.implementation.guava25.base to com.azure.cosmos.encryption, com.azure.cosmos.test, com.azure.cosmos.kafka.connect; - exports com.azure.cosmos.implementation.guava25.collect to com.azure.cosmos.encryption, com.azure.cosmos.test, com.azure.cosmos.kafka.connect; - exports com.azure.cosmos.implementation.guava27 to com.azure.cosmos.encryption, com.azure.cosmos.test, com.azure.cosmos.kafka.connect; + exports com.azure.cosmos.implementation to com.azure.cosmos.encryption, com.azure.cosmos.test; + exports com.azure.cosmos.implementation.caches to com.azure.cosmos.encryption, com.azure.cosmos.test; + exports com.azure.cosmos.implementation.feedranges to com.azure.cosmos.encryption, com.azure.cosmos.test; + exports com.azure.cosmos.implementation.apachecommons.lang to com.azure.cosmos.encryption, com.azure.cosmos.test; + exports com.azure.cosmos.implementation.guava25.base to com.azure.cosmos.encryption, com.azure.cosmos.test; + exports com.azure.cosmos.implementation.guava25.collect to com.azure.cosmos.encryption, com.azure.cosmos.test; + exports com.azure.cosmos.implementation.guava27 to com.azure.cosmos.encryption, com.azure.cosmos.test; exports com.azure.cosmos.implementation.directconnectivity to com.azure.cosmos.encryption, com.azure.cosmos.test; opens com.azure.cosmos.implementation to com.fasterxml.jackson.databind, java.logging, com.fasterxml.jackson.module.afterburner; // exporting implementation packages specifically for cosmos encryption exports com.azure.cosmos.implementation.batch to com.azure.cosmos.encryption; exports com.azure.cosmos.implementation.patch to com.azure.cosmos.encryption; - exports com.azure.cosmos.implementation.query to com.azure.cosmos.encryption, com.azure.cosmos.kafka.connect; - exports com.azure.cosmos.implementation.apachecommons.lang.tuple to com.azure.cosmos.encryption, com.azure.cosmos.kafka.connect; + exports com.azure.cosmos.implementation.query to com.azure.cosmos.encryption; + exports com.azure.cosmos.implementation.apachecommons.lang.tuple to com.azure.cosmos.encryption; // exporting some packages specifically for Jackson opens com.azure.cosmos.implementation.caches to com.fasterxml.jackson.databind; @@ -74,10 +74,9 @@ // exporting packages specifically for cosmos test exports com.azure.cosmos.implementation.faultinjection to com.azure.cosmos.test; exports com.azure.cosmos.implementation.directconnectivity.rntbd to com.azure.cosmos.test; - exports com.azure.cosmos.implementation.routing to com.azure.cosmos.test, com.azure.cosmos.kafka.connect; + exports com.azure.cosmos.implementation.routing to com.azure.cosmos.test; opens com.azure.cosmos to com.azure.cosmos.test, com.azure.spring.data.cosmos, com.fasterxml.jackson.databind, com.fasterxml.jackson.module.afterburner, java.logging; opens com.azure.cosmos.models to com.azure.cosmos.test, com.azure.spring.data.cosmos, com.fasterxml.jackson.databind, com.fasterxml.jackson.module.afterburner, java.logging; - exports com.azure.cosmos.implementation.changefeed.common to com.azure.cosmos.kafka.connect; uses com.azure.cosmos.implementation.guava25.base.PatternCompiler; uses com.azure.core.util.tracing.Tracer; diff --git a/sdk/cosmos/kafka-integration-matrix.json b/sdk/cosmos/kafka-integration-matrix.json deleted file mode 100644 index 8b353fe22bd8..000000000000 --- a/sdk/cosmos/kafka-integration-matrix.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "matrix": { - "Cosmos": { - "Session_Integration": { - "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableMultipleWriteLocations = $false }", - "ProfileFlag": "-P kafka-integration", - "Pool": "env:LINUXPOOL", - "OSVmImage": "env:LINUXVMIMAGE" - } - }, - "TestFromSource": true, - "JavaTestVersion": ["1.8", "1.11", "1.17", "1.21"] - } -} diff --git a/sdk/cosmos/tests.yml b/sdk/cosmos/tests.yml index 98d674a636ea..82671c67cec6 100644 --- a/sdk/cosmos/tests.yml +++ b/sdk/cosmos/tests.yml @@ -68,32 +68,4 @@ extends: AdditionalVariables: - name: AdditionalArgs value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)' - - - template: /eng/pipelines/templates/stages/archetype-sdk-tests-isolated.yml - parameters: - TestName: 'Kafka_Cosmos_Integration' - CloudConfig: - Public: - SubscriptionConfigurations: - - $(sub-config-azure-cloud-test-resources) - - $(sub-config-cosmos-azure-cloud-test-resources) - MatrixConfigs: - - Name: Kafka_Cosmos_Integration_Test - Path: sdk/cosmos/kafka-integration-matrix.json - Selection: all - GenerateVMJobs: true - ServiceDirectory: cosmos - TestResourceDirectories: - - cosmos/ - Artifacts: - - name: azure-cosmos-kafka-connect - groupId: com.azure.cosmos.kafka - safeName: azurecosmoskafkaconnect - TimeoutInMinutes: 120 - PreSteps: - - template: /eng/pipelines/templates/steps/install-reporting-tools.yml - TestGoals: 'verify' - TestOptions: '$(ProfileFlag) -DskipCompile=true -DskipTestCompile=true -DcreateSourcesJar=false' - AdditionalVariables: - - name: AdditionalArgs - value: '-DCOSMOS.CLIENT_TELEMETRY_ENDPOINT=$(cosmos-client-telemetry-endpoint) -DCOSMOS.CLIENT_TELEMETRY_COSMOS_ACCOUNT=$(cosmos-client-telemetry-cosmos-account)' + \ No newline at end of file