-
Notifications
You must be signed in to change notification settings - Fork 30
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add support for additional kafka props on bulk ingest
- Loading branch information
Showing
13 changed files
with
172 additions
and
91 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
kaldb/src/main/java/com/slack/kaldb/writer/KafkaUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| package com.slack.kaldb.writer; | ||
|
|
||
| import com.google.common.annotations.VisibleForTesting; | ||
| import java.util.Properties; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| /** Shared kafka functions for producers, consumers, and stream applications */ | ||
| public class KafkaUtils { | ||
| private static final Logger LOG = LoggerFactory.getLogger(KafkaUtils.class); | ||
|
|
||
| @VisibleForTesting | ||
| public static Properties maybeOverrideProps( | ||
| Properties inputProps, String key, String value, boolean override) { | ||
| Properties changedProps = (Properties) inputProps.clone(); | ||
| String userValue = changedProps.getProperty(key); | ||
| if (userValue != null) { | ||
| if (override) { | ||
| LOG.warn( | ||
| String.format( | ||
| "Property %s is provided but will be overridden from %s to %s", | ||
| key, userValue, value)); | ||
| changedProps.setProperty(key, value); | ||
| } else { | ||
| LOG.warn( | ||
| String.format( | ||
| "Property %s is provided but won't be overridden from %s to %s", | ||
| key, userValue, value)); | ||
| } | ||
| } else { | ||
| changedProps.setProperty(key, value); | ||
| } | ||
| return changedProps; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
48 changes: 48 additions & 0 deletions
48
kaldb/src/test/java/com/slack/kaldb/writer/KafkaUtilsTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| package com.slack.kaldb.writer; | ||
|
|
||
| import static org.assertj.core.api.Assertions.assertThat; | ||
|
|
||
| import com.slack.kaldb.proto.config.KaldbConfigs; | ||
| import com.slack.kaldb.testlib.TestKafkaServer; | ||
| import com.slack.kaldb.writer.kafka.KaldbKafkaConsumer; | ||
| import java.util.Properties; | ||
| import org.apache.kafka.clients.consumer.ConsumerConfig; | ||
| import org.junit.jupiter.api.Test; | ||
|
|
||
| class KafkaUtilsTest { | ||
| public static final String TEST_KAFKA_CLIENT_GROUP = "test_kaldb_consumer"; | ||
|
|
||
| @Test | ||
| public void testOverridingProperties() { | ||
| KaldbConfigs.KafkaConfig kafkaConfig = | ||
| KaldbConfigs.KafkaConfig.newBuilder() | ||
| .setKafkaTopic(TestKafkaServer.TEST_KAFKA_TOPIC) | ||
| .setKafkaTopicPartition("0") | ||
| .setKafkaBootStrapServers("bootstrap_server") | ||
| .setKafkaClientGroup(TEST_KAFKA_CLIENT_GROUP) | ||
| .setEnableKafkaAutoCommit("true") | ||
| .setKafkaAutoCommitInterval("5000") | ||
| .setKafkaSessionTimeout("5000") | ||
| .build(); | ||
|
|
||
| Properties properties = KaldbKafkaConsumer.makeKafkaConsumerProps(kafkaConfig); | ||
| assertThat(properties.get(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG)) | ||
| .isEqualTo("org.apache.kafka.common.serialization.StringDeserializer"); | ||
|
|
||
| kafkaConfig = | ||
| KaldbConfigs.KafkaConfig.newBuilder() | ||
| .setKafkaTopic(TestKafkaServer.TEST_KAFKA_TOPIC) | ||
| .setKafkaTopicPartition("0") | ||
| .setKafkaBootStrapServers("bootstrap_server") | ||
| .setKafkaClientGroup(TEST_KAFKA_CLIENT_GROUP) | ||
| .setEnableKafkaAutoCommit("true") | ||
| .setKafkaAutoCommitInterval("5000") | ||
| .setKafkaSessionTimeout("5000") | ||
| .putAdditionalProps(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, "test_serializer") | ||
| .build(); | ||
|
|
||
| properties = KaldbKafkaConsumer.makeKafkaConsumerProps(kafkaConfig); | ||
| assertThat(properties.get(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG)) | ||
| .isEqualTo("test_serializer"); | ||
| } | ||
| } |
Oops, something went wrong.