update containers ports, update README.md

README.adoc

@@ -97,7 +97,13 @@ Create `demo-topic` with 6 partitions and 3 replicas.
 
 [source,bash]
 ----
-$ ./bin/kafka-topics --create --partitions 6 --replication-factor 3 --topic demo-topic
+$ docker exec -it kafka101 \
+kafka-topics \
+--create \
+--partitions 6 \
+--replication-factor 3 \
+--topic demo-topic \
+--bootstrap-server kafka101:29092
 ----
 
 **4. Produce messages.**
@@ -106,7 +112,13 @@ Open a new terminal window, generate some message to simulate producer load.
 
 [source,bash]
 ----
-$ ./bin/kafka-producer-perf-test --throughput 500 --num-records 100000000 --topic demo-topic --record-size 100
+$ docker exec -it kafka101 \
+kafka-producer-perf-test \
+--throughput 500 \
+--num-records 100000000 \
+--topic demo-topic \
+--record-size 100 \
+--producer-props bootstrap.servers=kafka101:29092
 ----
 
 **5. Consume messages.**
@@ -115,7 +127,14 @@ Open a new terminal window, generate some message to simulate consumer load.
 
 [source,bash]
 ----
-$ ./bin/kafka-consumer-perf-test --messages 100000000 --timeout 1000000 --topic demo-topic --reporting-interval 1000 --show-detailed-stats  
+$ docker exec -it kafka101 \
+kafka-consumer-perf-test \
+--messages 100000000 \
+--timeout 1000000 \
+--topic demo-topic \
+--reporting-interval 1000 \
+--show-detailed-stats \
+--bootstrap-server kafka101:29092
 ----
 
 **6. Open Grafana.**

etc/jmx_exporter/config_kafka101.yml

@@ -17,7 +17,7 @@
 # Kafka Broker JMX Metrics : 
 # Compatible with version Apache Kafka v2.x.x / Confluent Plateform 5.x.x
 #---------------------------------------------------------------------------------------------------------
-hostPort: kafka101:9992
+hostPort: kafka101:9991
 lowercaseOutputName: true
 lowercaseOutputLabelNames: true
 

etc/jmx_exporter/config_kafka102.yml

@@ -17,7 +17,7 @@
 # Kafka Broker JMX Metrics : 
 # Compatible with version Apache Kafka v2.x.x / Confluent Plateform 5.x.x
 #---------------------------------------------------------------------------------------------------------
-hostPort: kafka102:9993
+hostPort: kafka102:9992
 lowercaseOutputName: true
 lowercaseOutputLabelNames: true
 

etc/jmx_exporter/config_kafka103.yml

@@ -17,7 +17,7 @@
 # Kafka Broker JMX Metrics : 
 # Compatible with version Apache Kafka v2.x.x / Confluent Plateform 5.x.x
 #---------------------------------------------------------------------------------------------------------
-hostPort: kafka103:9994
+hostPort: kafka103:9993
 lowercaseOutputName: true
 lowercaseOutputLabelNames: true
 

sasl-single-node-stack-start.sh

@@ -26,7 +26,6 @@ echo "Creating default kafka user for SASL/SCRAM"
 
 echo -e "\n🐳 Starting Kafka Brokers..."
 docker-compose -f $DIR/zk-kafka-single-node-secured-stack.yml up -d
-
 docker-compose -f $DIR/zk-kafka-single-node-secured-stack.yml ps
 
 echo -e "\n------------------------------------------------------------------------------------------------------"

zk-kafka-multiple-secured-stack.yml → zk-kafka-multiple-nodes-secured-stack.yml

@@ -94,16 +94,16 @@ services:
     depends_on:
       - zk1
     ports:
-      - 9092:9092
-      - 9992:9992
+      - 9091:9091
+      - 9991:9991
     container_name: kafka101
     volumes:
      - ./etc/secrets/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
     environment:
       KAFKA_BROKER_ID: 101
-      KAFKA_JMX_PORT: 9992
+      KAFKA_JMX_PORT: 9991
       KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
-      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka101:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka101:29092,PLAINTEXT_HOST://localhost:9091
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SASL_PLAINTEXT:SASL_PLAINTEXT,PLAINTEXT_HOST:SASL_PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: SASL_PLAINTEXT
       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
@@ -127,16 +127,16 @@ services:
     depends_on:
       - zk1
     ports:
-      - 9093:9093
-      - 9993:9993
+      - 9092:9092
+      - 9992:9992
     container_name: kafka102
     volumes:
      - ./etc/secrets/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
     environment:
       KAFKA_BROKER_ID: 102
-      KAFKA_JMX_PORT: 9993
+      KAFKA_JMX_PORT: 9092
       KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
-      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka102:29093,PLAINTEXT_HOST://localhost:9093
+      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka102:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SASL_PLAINTEXT:SASL_PLAINTEXT,PLAINTEXT_HOST:SASL_PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: SASL_PLAINTEXT
       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
@@ -150,7 +150,7 @@ services:
       KAFKA_ZOOKEEPER_SET_ACL: "true"
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka102:29093
+      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka102:29092
       CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper1:2181
       CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
       CONFLUENT_METRICS_ENABLE: 'false'
@@ -160,16 +160,16 @@ services:
     depends_on:
       - zk1
     ports:
-      - 9094:9094
-      - 9994:9994
+      - 9093:9093
+      - 9993:9993
     container_name: kafka103
     volumes:
      - ./etc/secrets/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
     environment:
       KAFKA_BROKER_ID: 103
       KAFKA_JMX_PORT: 9993
       KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
-      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka103:29094,PLAINTEXT_HOST://localhost:9094
+      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka103:29092,PLAINTEXT_HOST://localhost:9993
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SASL_PLAINTEXT:SASL_PLAINTEXT,PLAINTEXT_HOST:SASL_PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: SASL_PLAINTEXT
       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
@@ -183,7 +183,7 @@ services:
       KAFKA_ZOOKEEPER_SET_ACL: "true"
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
-      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka103:29094
+      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka103:29092
       CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper1:2181
       CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
       CONFLUENT_METRICS_ENABLE: 'false'

zk-kafka-multiple-nodes-stack.yml

@@ -93,14 +93,14 @@ services:
     depends_on:
       - zk1
     ports:
-      - 9092:9092
-      - 9992:9992
+      - 9091:9091
+      - 9991:9991
     container_name: kafka101
     environment:
       KAFKA_BROKER_ID: 101
-      KAFKA_JMX_PORT: 9992
+      KAFKA_JMX_PORT: 9991
       KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka101:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka101:29092,PLAINTEXT_HOST://localhost:9091
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
@@ -116,19 +116,19 @@ services:
     depends_on:
       - zk1
     ports:
-      - 9093:9093
-      - 9993:9993
+      - 9092:9092
+      - 9992:9992
     container_name: kafka102
     environment:
       KAFKA_BROKER_ID: 102
-      KAFKA_JMX_PORT: 9993
+      KAFKA_JMX_PORT: 9992
       KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka102:29093,PLAINTEXT_HOST://localhost:9093
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka102:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
-      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka102:29093
+      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka102:29092
       CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper1:2181
       CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
       CONFLUENT_METRICS_ENABLE: 'false'
@@ -139,19 +139,19 @@ services:
     depends_on:
       - zk1
     ports:
-      - 9094:9094
-      - 9994:9994
+      - 9093:9093
+      - 9993:9993
     container_name: kafka103
     environment:
       KAFKA_BROKER_ID: 103
-      KAFKA_JMX_PORT: 9994
+      KAFKA_JMX_PORT: 9993
       KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka103:29094,PLAINTEXT_HOST://localhost:9094
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka103:29092,PLAINTEXT_HOST://localhost:9093
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
       KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
-      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka103:29094
+      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka103:29092
       CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper1:2181
       CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
       CONFLUENT_METRICS_ENABLE: 'false'      

zk-kafka-single-node-full-stack.yml

@@ -67,11 +67,11 @@ services:
       - zk1
     ports:
       - 9092:9092
-      - 9992:9992
+      - 9991:9991
     container_name: kafka101
     environment:
       KAFKA_BROKER_ID: 101
-      KAFKA_JMX_PORT: 9992
+      KAFKA_JMX_PORT: 9991
       KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka101:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT

zk-kafka-single-node-secured-stack.yml

@@ -71,23 +71,24 @@ services:
       - zk1
     ports:
       - 9092:9092
-      - 9992:9992
+      - 9991:9991
     container_name: kafka101
     volumes:
      - ./etc/secrets/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
     environment:
       KAFKA_BROKER_ID: 101
-      KAFKA_JMX_PORT: 9992
+      KAFKA_JMX_PORT: 9991
       KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
       KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka101:29092,PLAINTEXT_HOST://localhost:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SASL_PLAINTEXT:SASL_PLAINTEXT,PLAINTEXT_HOST:SASL_PLAINTEXT
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SASL_PLAINTEXT:SASL_PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: SASL_PLAINTEXT
       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.auth.SimpleAclAuthorizer
       KAFKA_SUPER_USERS: User:kafka
       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true" 
       KAFKA_SASL_ENABLED_MECHANISMS: SCRAM-SHA-256
       KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: SCRAM-SHA-256
       KAFKA_OPTS: -Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf -Dzookeeper.sasl.clientconfig=ZkClient
+      KAFKA_LOG4J_LOGGERS: "kafka.authorizer.logger=DEBUG"
       #KAFKA_LOG4J_ROOT_LOGLEVEL: DEBUG
       #KAFKA_TOOLS_LOG4J_LOGLEVEL: DEBUG
       KAFKA_ZOOKEEPER_SET_ACL: "true"

zk-kafka-single-node-stack.yml

@@ -66,11 +66,11 @@ services:
       - zk1
     ports:
       - 9092:9092
-      - 9992:9992
+      - 9991:9991
     container_name: kafka101
     environment:
       KAFKA_BROKER_ID: 101
-      KAFKA_JMX_PORT: 9992
+      KAFKA_JMX_PORT: 9991
       KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka101:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT