From 6d4bb44d2198d5adb0a539300f96a1b0d221daea Mon Sep 17 00:00:00 2001 From: Brett Randall Date: Sun, 20 Dec 2020 17:13:52 +1100 Subject: [PATCH] Run create-topics.sh from tools, reuse get_kafka_cluster_id_from_container. --- scripts/helper/create-topics.sh | 21 ++++++++++----------- scripts/helper/functions.sh | 5 ++++- scripts/start.sh | 2 +- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/scripts/helper/create-topics.sh b/scripts/helper/create-topics.sh index 453dd4a13..7adfca43d 100755 --- a/scripts/helper/create-topics.sh +++ b/scripts/helper/create-topics.sh @@ -5,17 +5,16 @@ IFS=$'\n\t' DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" source ${DIR}/functions.sh -KAFKA_CLUSTER_ID=$(curl -s --insecure --location --request GET 'https://localhost:8091/kafka/v3/clusters' --header 'Authorization: Basic c3VwZXJVc2VyOnN1cGVyVXNlcg==' | jq -r '.data[0].cluster_id') - -echo "cluster_id: ${KAFKA_CLUSTER_ID}" +KAFKA_CLUSTER_ID=$(get_kafka_cluster_id_from_container) +echo "KAFKA_CLUSTER_ID: ${KAFKA_CLUSTER_ID}" auth="superUser:superUser" -create_topic localhost:8091 ${KAFKA_CLUSTER_ID} users true ${auth} -create_topic localhost:8091 ${KAFKA_CLUSTER_ID} wikipedia.parsed true ${auth} -create_topic localhost:8091 ${KAFKA_CLUSTER_ID} wikipedia.parsed.count-by-domain false ${auth} -create_topic localhost:8091 ${KAFKA_CLUSTER_ID} wikipedia.failed false ${auth} -create_topic localhost:8091 ${KAFKA_CLUSTER_ID} WIKIPEDIABOT false ${auth} -create_topic localhost:8091 ${KAFKA_CLUSTER_ID} WIKIPEDIANOBOT false ${auth} -create_topic localhost:8091 ${KAFKA_CLUSTER_ID} EN_WIKIPEDIA_GT_1 false ${auth} -create_topic localhost:8091 ${KAFKA_CLUSTER_ID} EN_WIKIPEDIA_GT_1_COUNTS false ${auth} +create_topic kafka1:8091 ${KAFKA_CLUSTER_ID} users true ${auth} +create_topic kafka1:8091 ${KAFKA_CLUSTER_ID} wikipedia.parsed true ${auth} +create_topic kafka1:8091 ${KAFKA_CLUSTER_ID} wikipedia.parsed.count-by-domain false ${auth} +create_topic kafka1:8091 ${KAFKA_CLUSTER_ID} wikipedia.failed false ${auth} +create_topic kafka1:8091 ${KAFKA_CLUSTER_ID} WIKIPEDIABOT false ${auth} +create_topic kafka1:8091 ${KAFKA_CLUSTER_ID} WIKIPEDIANOBOT false ${auth} +create_topic kafka1:8091 ${KAFKA_CLUSTER_ID} EN_WIKIPEDIA_GT_1 false ${auth} +create_topic kafka1:8091 ${KAFKA_CLUSTER_ID} EN_WIKIPEDIA_GT_1_COUNTS false ${auth} diff --git a/scripts/helper/functions.sh b/scripts/helper/functions.sh index 8414e8481..bf1b5f86f 100644 --- a/scripts/helper/functions.sh +++ b/scripts/helper/functions.sh @@ -259,6 +259,9 @@ END } create_topic() { + + local DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + broker_host_port=$1 cluster_id=$2 topic_name=$3 @@ -267,7 +270,7 @@ create_topic() { RESULT=$(curl -sS -X POST \ -u ${auth} \ - --cacert ${DIR}/../security/snakeoil-ca-1.crt \ + --cacert /etc/kafka/secrets/snakeoil-ca-1.crt \ --header 'Content-Type: application/json' \ --data-binary @<(jq -n --arg topic_name "${topic_name}" --arg confluent_value_schema_validation "${confluent_value_schema_validation}" -f ${DIR}/topic.jq) \ "https://${broker_host_port}/kafka/v3/clusters/${cluster_id}/topics") && RC=$? || RC=$? diff --git a/scripts/start.sh b/scripts/start.sh index 896bcd77d..2afdc3669 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -77,7 +77,7 @@ docker-compose up -d schemaregistry connect control-center echo echo -e "Create topics in Kafka cluster:" -${DIR}/helper/create-topics.sh || exit 1 +docker-compose exec tools bash -c "/tmp/helper/create-topics.sh" || exit 1 # Verify Confluent Control Center has started MAX_WAIT=300