From e51d1aecfb7829e90a9acd0197c580d60777a488 Mon Sep 17 00:00:00 2001 From: Zhengke Zhou Date: Thu, 26 Dec 2024 00:49:02 +0800 Subject: [PATCH] Enhance Kafka integration tests to support multiple Kafka versions (#6400) ## Which problem is this PR solving? - Resolves #3068 ## Description of the changes - Introduce old version kafka env - Update e2e CI script ## How was this change tested? - integration tests ## Checklist - [x] I have read https://github.com/jaegertracing/jaeger/blob/master/CONTRIBUTING_GUIDELINES.md - [x] I have signed all commits - [ ] I have added unit tests for the new functionality - [x] I have run lint and test steps successfully - for `jaeger`: `make lint test` - for `jaeger-ui`: `npm run lint` and `npm run test` --------- Signed-off-by: zzzk1 --- .github/workflows/ci-e2e-kafka.yml | 11 ++--- docker-compose/kafka/docker-compose.yml | 2 +- docker-compose/kafka/v2/docker-compose.yml | 21 ++++++++++ .../v3}/docker-compose.yml | 0 renovate.json | 3 +- scripts/kafka-integration-test.sh | 41 +++++++++++++++---- 6 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 docker-compose/kafka/v2/docker-compose.yml rename docker-compose/{kafka-integration-test => kafka/v3}/docker-compose.yml (100%) diff --git a/.github/workflows/ci-e2e-kafka.yml b/.github/workflows/ci-e2e-kafka.yml index d56d0046803..c79af13d1bc 100644 --- a/.github/workflows/ci-e2e-kafka.yml +++ b/.github/workflows/ci-e2e-kafka.yml @@ -17,8 +17,9 @@ jobs: strategy: fail-fast: false matrix: - jaeger-version: [v1, v2] # Adjust if there are specific versions of Jaeger - name: kafka ${{ matrix.jaeger-version }} + jaeger-version: [v1, v2] + kafka-version: ["3.x", "2.x"] + name: kafka ${{matrix.kafka-version }} ${{ matrix.jaeger-version }} steps: - name: Harden Runner uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 @@ -31,12 +32,12 @@ jobs: with: go-version: 1.23.x - - name: Run Kafka integration tests + - name: Run kafka integration tests id: test-execution - run: bash scripts/kafka-integration-test.sh -j ${{ matrix.jaeger-version }} + run: bash scripts/kafka-integration-test.sh -j ${{ matrix.jaeger-version }} -v ${{ matrix.kafka-version }} - name: Upload coverage to codecov uses: ./.github/actions/upload-codecov with: files: cover.out - flags: kafka-${{ matrix.jaeger-version }} + flags: kafka-${{ matrix.kafka-version }}-${{ matrix.jaeger-version }} diff --git a/docker-compose/kafka/docker-compose.yml b/docker-compose/kafka/docker-compose.yml index 58e3061fa8f..dad341b3a9f 100644 --- a/docker-compose/kafka/docker-compose.yml +++ b/docker-compose/kafka/docker-compose.yml @@ -108,4 +108,4 @@ services: jaeger-remote-storage: condition: service_healthy links: - - jaeger-remote-storage + - jaeger-remote-storage \ No newline at end of file diff --git a/docker-compose/kafka/v2/docker-compose.yml b/docker-compose/kafka/v2/docker-compose.yml new file mode 100644 index 00000000000..31cd9f7ab18 --- /dev/null +++ b/docker-compose/kafka/v2/docker-compose.yml @@ -0,0 +1,21 @@ +services: + zookeeper: + image: bitnami/zookeeper:3.8.4 + ports: + - "2181:2181" + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + + kafka: + image: bitnami/kafka:2.8.0 + ports: + - "9092:9092" + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENERS=PLAINTEXT://:9092 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - ALLOW_PLAINTEXT_LISTENER=yes + depends_on: + - zookeeper \ No newline at end of file diff --git a/docker-compose/kafka-integration-test/docker-compose.yml b/docker-compose/kafka/v3/docker-compose.yml similarity index 100% rename from docker-compose/kafka-integration-test/docker-compose.yml rename to docker-compose/kafka/v3/docker-compose.yml diff --git a/renovate.json b/renovate.json index ee3da55b62c..d7433a85f67 100644 --- a/renovate.json +++ b/renovate.json @@ -22,7 +22,8 @@ "docker-compose/elasticsearch/v6/docker-compose.yml", "docker-compose/elasticsearch/v7/docker-compose.yml", "docker-compose/elasticsearch/v8/docker-compose.yml", - "docker-compose/kafka-integration-test/docker-compose.yml", + "docker-compose/kafka/v2/docker-compose.yml", + "docker-compose/kafka/v3/docker-compose.yml", "docker-compose/kafka/docker-compose.yml", "docker-compose/monitor/docker-compose.yml", "docker-compose/opensearch/v1/docker-compose.yml", diff --git a/scripts/kafka-integration-test.sh b/scripts/kafka-integration-test.sh index 0777e7efb1d..c62f73a44f5 100755 --- a/scripts/kafka-integration-test.sh +++ b/scripts/kafka-integration-test.sh @@ -5,38 +5,63 @@ set -euf -o pipefail -compose_file="docker-compose/kafka-integration-test/docker-compose.yml" +compose_file="" jaeger_version="" +kafka_version="" manage_kafka="true" success="false" -print_help() { - echo "Usage: $0 [-K] -j " +usage() { + echo "Usage: $0 [-K] -j -v " echo " -K: do not start or stop Kafka container (useful for local testing)" echo " -j: major version of Jaeger to test (v1|v2)" + echo " -v: kafka major version (3.x|2.x)" exit 1 } +check_arg() { + if [ ! $# -eq 3 ]; then + echo "ERROR: need exactly three arguments" + usage + fi +} + parse_args() { - while getopts "j:Kh" opt; do + while getopts "j:v:Kh" opt; do case "${opt}" in j) jaeger_version=${OPTARG} ;; + v) + case ${OPTARG} in + 3.x) + kafka_version="v3" + ;; + 2.x) + kafka_version="v2" + ;; + *) + echo "Error: Invalid Kafka version. Valid options are 3.x or 2.x" + usage + ;; + esac + ;; K) manage_kafka="false" ;; *) - print_help + usage ;; esac done - if [ "$jaeger_version" != "v1" ] && [ "$jaeger_version" != "v2" ]; then + if [[ "$jaeger_version" != "v1" && "$jaeger_version" != "v2" ]]; then echo "Error: Invalid Jaeger version. Valid options are v1 or v2" - print_help + usage fi + compose_file="docker-compose/kafka/${kafka_version}/docker-compose.yml" } + setup_kafka() { echo "Starting Kafka using Docker Compose..." docker compose -f "${compose_file}" up -d kafka @@ -89,7 +114,7 @@ run_integration_test() { make jaeger-v2-storage-integration-test else echo "Unknown Jaeger version ${jaeger_version}." - print_help + usage fi }