Enhance Kafka integration tests to support multiple Kafka versions (#…

…6400)

## Which problem is this PR solving?
- Resolves #3068
## Description of the changes
- Introduce old version kafka env
- Update e2e CI script
## How was this change tested?
- integration tests

## Checklist
- [x] I have read
https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md
- [x] I have signed all commits
- [ ] I have added unit tests for the new functionality
- [x] I have run lint and test steps successfully
  - for `jaeger`: `make lint test`
  - for `jaeger-ui`: `npm run lint` and `npm run test`

---------

Signed-off-by: zzzk1 <[email protected]>

.github/workflows/ci-e2e-kafka.yml

@@ -17,8 +17,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jaeger-version: [v1, v2]  # Adjust if there are specific versions of Jaeger
-    name: kafka ${{ matrix.jaeger-version }}
+        jaeger-version: [v1, v2]
+        kafka-version: ["3.x", "2.x"]
+    name: kafka ${{matrix.kafka-version }} ${{ matrix.jaeger-version }}
     steps:
     - name: Harden Runner
       uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
@@ -31,12 +32,12 @@ jobs:
       with:
         go-version: 1.23.x
 
-    - name: Run Kafka integration tests
+    - name: Run kafka integration tests
       id: test-execution
-      run: bash scripts/kafka-integration-test.sh -j ${{ matrix.jaeger-version }}
+      run: bash scripts/kafka-integration-test.sh -j ${{ matrix.jaeger-version }} -v ${{ matrix.kafka-version }}
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:
         files: cover.out
-        flags: kafka-${{ matrix.jaeger-version }}
+        flags: kafka-${{ matrix.kafka-version }}-${{ matrix.jaeger-version }}

docker-compose/kafka/docker-compose.yml

@@ -108,4 +108,4 @@ services:
       jaeger-remote-storage:
         condition: service_healthy
     links:
-      - jaeger-remote-storage
+      - jaeger-remote-storage

docker-compose/kafka/v2/docker-compose.yml

@@ -0,0 +1,21 @@
+services:
+  zookeeper:
+    image: bitnami/zookeeper:3.8.4
+    ports:
+      - "2181:2181"
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+
+  kafka:
+    image: bitnami/kafka:2.8.0
+    ports:
+      - "9092:9092"
+    environment:
+      - KAFKA_BROKER_ID=1
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_LISTENERS=PLAINTEXT://:9092
+      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - ALLOW_PLAINTEXT_LISTENER=yes
+    depends_on:
+      - zookeeper

renovate.json

@@ -22,7 +22,8 @@
         "docker-compose/elasticsearch/v6/docker-compose.yml",
         "docker-compose/elasticsearch/v7/docker-compose.yml",
         "docker-compose/elasticsearch/v8/docker-compose.yml",
-        "docker-compose/kafka-integration-test/docker-compose.yml",
+        "docker-compose/kafka/v2/docker-compose.yml",
+        "docker-compose/kafka/v3/docker-compose.yml",
         "docker-compose/kafka/docker-compose.yml",
         "docker-compose/monitor/docker-compose.yml",
         "docker-compose/opensearch/v1/docker-compose.yml",

scripts/kafka-integration-test.sh

@@ -5,38 +5,63 @@
 
 set -euf -o pipefail
 
-compose_file="docker-compose/kafka-integration-test/docker-compose.yml"
+compose_file=""
 jaeger_version=""
+kafka_version=""
 manage_kafka="true"
 success="false"
 
-print_help() {
-  echo "Usage: $0 [-K] -j <jaeger_version>"
+usage() {
+  echo "Usage: $0 [-K] -j <jaeger_version> -v <kafka_version>"
   echo "  -K: do not start or stop Kafka container (useful for local testing)"
   echo "  -j: major version of Jaeger to test (v1|v2)"
+  echo "  -v: kafka major version (3.x|2.x)"
   exit 1
 }
 
+check_arg() {
+  if [ ! $# -eq 3 ]; then
+      echo "ERROR: need exactly three arguments"
+      usage
+    fi
+}
+
 parse_args() {
-  while getopts "j:Kh" opt; do
+  while getopts "j:v:Kh" opt; do
     case "${opt}" in
     j)
       jaeger_version=${OPTARG}
       ;;
+    v)
+      case ${OPTARG} in
+      3.x)
+        kafka_version="v3"
+        ;;
+      2.x)
+        kafka_version="v2"
+        ;;
+      *)
+        echo "Error: Invalid Kafka version. Valid options are 3.x or 2.x"
+        usage
+        ;;
+      esac
+      ;;
     K)
       manage_kafka="false"
       ;;
     *)
-      print_help
+      usage
       ;;
     esac
   done
-  if [ "$jaeger_version" != "v1" ] && [ "$jaeger_version" != "v2" ]; then
+  if [[ "$jaeger_version" != "v1" && "$jaeger_version" != "v2" ]]; then
     echo "Error: Invalid Jaeger version. Valid options are v1 or v2"
-    print_help
+    usage
   fi
+  compose_file="docker-compose/kafka/${kafka_version}/docker-compose.yml"
 }
 
+
 setup_kafka() {
   echo "Starting Kafka using Docker Compose..."
   docker compose -f "${compose_file}" up -d kafka
@@ -89,7 +114,7 @@ run_integration_test() {
     make jaeger-v2-storage-integration-test
   else
     echo "Unknown Jaeger version ${jaeger_version}."
-    print_help
+    usage
   fi
 }