Merge pull request #1529 from ar3ndt/kafka_additional_defaults

additional defaults for kafka config #1329

core/src/epicli/data/common/ansible/playbooks/roles/kafka/templates/server.properties.j2

@@ -11,14 +11,35 @@ auto.create.topics.enable=true
 delete.topic.enable=true
 
 #default replication factors for automatically created topics
-default.replication.factor={{ specification.kafka_var.replicas }}
+default.replication.factor={{ specification.kafka_var.default_replication_factor }}
 
 #The replication factor for the offsets topic (set higher to ensure availability). Internal topic creation will fail until the cluster size meets this replication factor requirement.
-offsets.topic.replication.factor={{ specification.kafka_var.replicas }}
+offsets.topic.replication.factor={{ specification.kafka_var.offsets_topic_replication_factor }}
 
 #Offsets older than this retention period will be discarded
 offsets.retention.minutes={{ specification.kafka_var.offset_retention_minutes }}
 
+#The maximum number of incremental fetch sessions that we will maintain
+max.incremental.fetch.session.cache.slots={{ specification.kafka_var.max_incremental_fetch_session_cache_slots }}
+
+#Enable controlled shutdown of the server
+controlled.shutdown.enable={{ specification.kafka_var.controlled_shutdown_enable | lower }}
+
+#Number of fetcher threads used to replicate messages from a source broker. 
+#Increasing this value can increase the degree of I/O parallelism in the follower broker.
+num.replica.fetchers={{ specification.kafka_var.num_replica_fetchers }}
+
+#The number of bytes of messages to attempt to fetch for each partition. 
+#This is not an absolute maximum, if the first record batch in the first 
+#non-empty partition of the fetch is larger than this value, 
+#the record batch will still be returned to ensure that progress can be made. 
+#The maximum record batch size accepted by the broker is defined 
+#via message.max.bytes (broker config) or max.message.bytes (topic config).
+replica.fetch.max.bytes={{ specification.kafka_var.replica_fetch_max_bytes }}
+
+#The socket receive buffer for network requests
+replica.socket.receive.buffer.bytes={{ specification.kafka_var.replica_socket_receive_buffer_bytes }}
+
 ############################# Socket Server Settings #############################
   {%- endif %}
 {%- endfor %}
@@ -126,9 +147,11 @@ num.partitions={{ specification.kafka_var.partitions }}
 
 # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
 # This value is recommended to be increased for installations with data dirs located in RAID array.
-num.recovery.threads.per.data.dir=1
+num.recovery.threads.per.data.dir={{ specification.kafka_var.num_recovery_threads_per_data_dir }}
 
-min.insync.replicas={{ specification.kafka_var.replicas }}
+# When a producer sets acks to "all" (or "-1"), this configuration specifies the minimum number 
+# of replicas that must acknowledge a write for the write to be considered successful.
+min.insync.replicas={{ specification.kafka_var.min_insync_replicas }}
 
 ############################# Log Flush Policy #############################
 

core/src/epicli/data/common/defaults/configuration/kafka.yml

@@ -50,14 +50,22 @@ specification:
 
     sha: "b28e81705e30528f1abb6766e22dfe9dae50b1e1e93330c880928ff7a08e6b38ee71cbfc96ec14369b2dfd24293938702cab422173c8e01955a9d1746ae43f98"
     port: 9092
-    replicas: 1 # Default to at least 1 (1 broker)
+    min_insync_replicas: 1 # Minimum number of replicas (ack write)
+    default_replication_factor: 1 # Minimum number of automatically created topics
+    offsets_topic_replication_factor: 1 # Minimum number of offsets topic (consider higher value for HA)
+    num_recovery_threads_per_data_dir: 1 # Minimum number of recovery threads per data dir
+    num_replica_fetchers: 1 # Minimum number of replica fetchers
+    replica_fetch_max_bytes: 1048576
+    replica_socket_receive_buffer_bytes: 65536
     partitions: 8 # 100 x brokers x replicas for reasonable size cluster. Small clusters can be less
     log_retention_hours: 168 # The minimum age of a log file to be eligible for deletion due to age
     log_retention_bytes: -1 # -1 is no size limit only a time limit (log_retention_hours). This limit is enforced at the partition level, multiply it by the number of partitions to compute the topic retention in bytes.
     offset_retention_minutes: 10080 # Offsets older than this retention period will be discarded
     heap_opts: "-Xmx2G -Xms2G"
     opts: "-Djavax.net.debug=all"
     jmx_opts:
+    max_incremental_fetch_session_cache_slots: 1000
+    controlled_shutdown_enable: true
 
     group: kafka
     user: kafka