diff --git a/generate-dashboards.sh b/generate-dashboards.sh index 6709197619..5568a37247 100755 --- a/generate-dashboards.sh +++ b/generate-dashboards.sh @@ -28,7 +28,7 @@ fi usage="$(basename "$0") [-h] [-v comma separated versions ] [-D] [-j additional dashboard to load to Grafana, multiple params are supported] [-M scylla-manager version ] [-t] [-F force update] [-S start with a system specific dashboard set] -- Generates the grafana dashboards and their load files" BASE_DASHBOARD_DIR="grafana/provisioning/dashboards" -while getopts ':htDv:j:M:S:B:P:F' option; do +while getopts ':htDv:j:M:S:B:P:s:F' option; do case "$option" in h) echo "$usage" exit @@ -48,6 +48,9 @@ while getopts ':htDv:j:M:S:B:P:F' option; do ;; S) SPECIFIC_SOLUTION="$OPTARG" ;; + s) STACK="$OPTARG" + BASE_DASHBOARD_DIR="grafana/stack/$OPTARG/provisioning/dashboards" + ;; D) VERSIONS=${SUPPORTED_VERSIONS[$BRANCH_VERSION]} FORMAT_COMAND="$FORMAT_COMAND -v $VERSIONS" MANAGER_VERSION=${MANAGER_SUPPORTED_VERSIONS[$BRANCH_VERSION]} @@ -62,7 +65,6 @@ done if [[ -z "$TEST_ONLY" ]]; then mkdir -p grafana/build fi - mkdir -p $BASE_DASHBOARD_DIR rm -f $BASE_DASHBOARD_DIR/load.*.yaml diff --git a/grafana-datasource.sh b/grafana-datasource.sh index a4ebacf750..df941d92da 100755 --- a/grafana-datasource.sh +++ b/grafana-datasource.sh @@ -5,13 +5,13 @@ if [ "$1" = "" ]; then echo "$usage" exit fi - +BASE_DIR="grafana/provisioning/datasources" if [ "$1" = "--compose" ]; then DB_ADDRESS="aprom:9090" ALERT_MANAGER_ADDRESS="aalert:9093" LOKI_ADDRESS="loki:3100" else - while getopts ':hlEg:n:p:v:a:x:c:j:m:G:M:D:A:S:P:L:Q:' option; do + while getopts ':hlEg:n:p:v:a:x:c:j:m:G:M:D:A:S:P:L:Q:s:' option; do case "$option" in h) echo "$usage" exit @@ -23,6 +23,8 @@ else ;; L) LOKI_ADDRESS=$OPTARG ;; + s) BASE_DIR="grafana/stack/$OPTARG/provisioning/datasources" + ;; :) printf "missing argument for -%s\n" "$OPTARG" >&2 echo "$usage" >&2 exit 1 @@ -34,16 +36,16 @@ else esac done fi -mkdir -p grafana/provisioning/datasources -rm -f grafana/provisioning/datasources/datasource.yaml -sed "s/DB_ADDRESS/$DB_ADDRESS/" grafana/datasource.yml | sed "s/AM_ADDRESS/$ALERT_MANAGER_ADDRESS/" > grafana/provisioning/datasources/datasource.yaml +mkdir -p $BASE_DIR +rm -f $BASE_DIR/datasource.yaml +sed "s/DB_ADDRESS/$DB_ADDRESS/" grafana/datasource.yml | sed "s/AM_ADDRESS/$ALERT_MANAGER_ADDRESS/" > $BASE_DIR/datasource.yaml if [ -z "$LOKI_ADDRESS" ]; then - rm -f grafana/provisioning/datasources/datasource.loki.yaml + rm -f $BASE_DIR/datasource.loki.yaml else - sed "s/LOKI_ADDRESS/$LOKI_ADDRESS/" grafana/datasource.loki.yml> grafana/provisioning/datasources/datasource.loki.yaml + sed "s/LOKI_ADDRESS/$LOKI_ADDRESS/" grafana/datasource.loki.yml> $BASE_DIR/datasource.loki.yaml fi if [ -z "$SCYLLA_USER" ] || [ -z "$SCYLLA_PSSWD" ]; then - cp grafana/datasource.scylla.yml grafana/provisioning/datasources/datasource.scylla.yml + cp grafana/datasource.scylla.yml $BASE_DIR/datasource.scylla.yml else - sed "s/SCYLLA_USER/$SCYLLA_USER/" grafana/datasource.psswd.scylla.yml |sed "s/SCYLLA_PSSWD/$SCYLLA_PSSWD/">grafana/provisioning/datasources/datasource.scylla.yml + sed "s/SCYLLA_USER/$SCYLLA_USER/" grafana/datasource.psswd.scylla.yml |sed "s/SCYLLA_PSSWD/$SCYLLA_PSSWD/">$BASE_DIR/datasource.scylla.yml fi diff --git a/prometheus-config.sh b/prometheus-config.sh index 7046cc9fc7..897ca2c5e8 100755 --- a/prometheus-config.sh +++ b/prometheus-config.sh @@ -2,6 +2,7 @@ usage="$(basename "$0") [-h] [-m alert_manager address] [-L] [-T additional-prometheus-targets] [--compose] -- Generate grafna's datasource file" CONSUL_ADDRESS="" COMPOSE=0 +BASE_DIR="$PWD/prometheus/build" if [ -f env.sh ]; then . env.sh fi @@ -37,7 +38,7 @@ for arg; do esac done -while getopts ':hL:m:T:E:' option; do +while getopts ':hL:m:T:E:s:' option; do case "$option" in h) echo "$usage" exit @@ -48,6 +49,8 @@ while getopts ':hL:m:T:E:' option; do ;; m) AM_ADDRESS="$OPTARG" ;; + s) BASE_DIR="$PWD/prometheus/build/stack/$OPTARG" + ;; E) EVALUATION_INTERVAL="$OPTARG" ;; :) printf "missing argument for -%s\n" "$OPTARG" >&2 @@ -61,31 +64,31 @@ while getopts ':hL:m:T:E:' option; do esac done -mkdir -p $PWD/prometheus/build/ +mkdir -p $BASE_DIR/ if [ -z $CONSUL_ADDRESS ]; then - sed "s/AM_ADDRESS/$AM_ADDRESS/" $PWD/prometheus/prometheus.yml.template > $PWD/prometheus/build/prometheus.yml + sed "s/AM_ADDRESS/$AM_ADDRESS/" $PWD/prometheus/prometheus.yml.template > $BASE_DIR/prometheus.yml else if [[ ! $CONSUL_ADDRESS = *":"* ]]; then CONSUL_ADDRESS="$CONSUL_ADDRESS:5090" fi - sed "s/AM_ADDRESS/$AM_ADDRESS/" $PWD/prometheus/prometheus.consul.yml.template| sed "s/MANAGER_ADDRESS/$CONSUL_ADDRESS/" > $PWD/prometheus/build/prometheus.yml + sed "s/AM_ADDRESS/$AM_ADDRESS/" $PWD/prometheus/prometheus.consul.yml.template| sed "s/MANAGER_ADDRESS/$CONSUL_ADDRESS/" > $BASE_DIR/prometheus.yml fi if [[ "$EVALUATION_INTERVAL" != "" ]]; then - sed -i "s/ evaluation_interval: [[:digit:]]*.*/ evaluation_interval: ${EVALUATION_INTERVAL}/g" $PWD/prometheus/build/prometheus.yml + sed -i "s/ evaluation_interval: [[:digit:]]*.*/ evaluation_interval: ${EVALUATION_INTERVAL}/g" $BASE_DIR/prometheus.yml fi if [ "$NO_CAS" = "1" ] && [ "$NO_CDC" = "1" ]; then - sed -i "s/ *# FILTER_METRICS.*/ - source_labels: [__name__]\\n regex: '(.*_cdc_.*|.*_cas.*)'\\n action: drop/g" $PWD/prometheus/build/prometheus.yml + sed -i "s/ *# FILTER_METRICS.*/ - source_labels: [__name__]\\n regex: '(.*_cdc_.*|.*_cas.*)'\\n action: drop/g" $BASE_DIR/prometheus.yml elif [ "$NO_CAS" = "1" ]; then - sed -i "s/ *# FILTER_METRICS.*/ - source_labels: [__name__]\\n regex: '(.*_cas.*)'\\n action: drop/g" $PWD/prometheus/build/prometheus.yml + sed -i "s/ *# FILTER_METRICS.*/ - source_labels: [__name__]\\n regex: '(.*_cas.*)'\\n action: drop/g" $BASE_DIR/prometheus.yml elif [ "$NO_CDC" = "1" ]; then - sed -i "s/ *# FILTER_METRICS.*/ - source_labels: [__name__]\\n regex: '(.*_cdc_.*)'\\n action: drop/g" $PWD/prometheus/build/prometheus.yml + sed -i "s/ *# FILTER_METRICS.*/ - source_labels: [__name__]\\n regex: '(.*_cdc_.*)'\\n action: drop/g" $BASE_DIR/prometheus.yml fi if [ "$NO_NODE_EXPORTER_FILE" = "1" ]; then - sed -i "s/ *# NODE_EXPORTER_PORT_MAPPING.*/ - source_labels: [__address__]\\n regex: '(.*):\\\\d+'\\n target_label: __address__\\n replacement: '\$\{1\}'\\n/g" $PWD/prometheus/build/prometheus.yml + sed -i "s/ *# NODE_EXPORTER_PORT_MAPPING.*/ - source_labels: [__address__]\\n regex: '(.*):\\\\d+'\\n target_label: __address__\\n replacement: '\$\{1\}'\\n/g" $BASE_DIR/prometheus.yml fi if [ "$NO_MANAGER_AGENT_FILE" = "1" ]; then - sed -i "s/ *# MANAGER_AGENT_PORT_MAPPING.*/ - source_labels: [__address__]\\n regex: '(.*):\\\\d+'\\n target_label: __address__\\n replacement: \'\$\{1\}\'\\n/g" $PWD/prometheus/build/prometheus.yml + sed -i "s/ *# MANAGER_AGENT_PORT_MAPPING.*/ - source_labels: [__address__]\\n regex: '(.*):\\\\d+'\\n target_label: __address__\\n replacement: \'\$\{1\}\'\\n/g" $BASE_DIR/prometheus.yml fi for val in "${PROMETHEUS_TARGETS[@]}"; do @@ -93,5 +96,5 @@ for val in "${PROMETHEUS_TARGETS[@]}"; do echo "Target file $val does not exists" exit 1 fi - cat $val >> $PWD/prometheus/build/prometheus.yml + cat $val >> $BASE_DIR/prometheus.yml done diff --git a/start-all.sh b/start-all.sh index d8b36cbf07..84f1754277 100755 --- a/start-all.sh +++ b/start-all.sh @@ -31,18 +31,6 @@ if [ "$1" = "--version" ]; then exit fi -if [ "$CURRENT_VERSION" = "master" ]; then - echo "" - echo "*****************************************************" - echo "* WARNING: You are using the unstable master branch *" - echo "* Check the README.md file for the stable releases *" - echo "*****************************************************" - echo "" - echo "Make sure you run generate-dashboards.sh to generate your dashboards." - echo 'For example to use Scylla 2021.1 run `./generate-dashboards.sh -F -v 2021.1`' - echo "" -fi - if [ "`id -u`" -eq 0 ]; then echo "Running as root is not advised, please check the documentation on how to run as non-root user" USER_PERMISSIONS="-u 0:0" @@ -113,7 +101,7 @@ Options: --stack id - Use this option when running a secondary stack, id could be 1-4 --limit container,param - Allow to set a specific Docker parameter for a container, where container can be: prometheus, grafana, alertmanager, loki, sidecar, grafanarender - --archive - Treat data directory as an archive. This disables Prometheus time-to-live (infinite retention). + --archive data-directory - Treat data directory as an archive. This disables Prometheus time-to-live (infinite retention), and would run a minimal mode The script starts Scylla Monitoring stack. " echo "$__usage" @@ -262,7 +250,7 @@ for arg; do ;; (--param) LIMIT="1" - PARAM="1" + PARAM="param" ;; (--evaluation-interval) LIMIT="1" @@ -301,6 +289,8 @@ for arg; do ;; (--archive) ARCHIVE="1" + LIMIT="1" + PARAM="archive" ;; (*) set -- "$@" "$arg" ;; @@ -309,7 +299,13 @@ for arg; do DOCR=`echo $arg|cut -d',' -f1` VALUE=`echo $arg|cut -d',' -f2-|sed 's/#/ /g'` NOSPACE=`echo $arg|sed 's/ /#/g'` - if [ "$PARAM" = "1" ]; then + if [[ $NOSPACE == --* ]]; then + echo "Error: No value given to --$PARAM" + echo + usage + exit 1 + fi + if [ "$PARAM" = "param" ]; then if [ -z "${DOCKER_PARAMS[$DOCR]}" ]; then DOCKER_PARAMS[$DOCR]="" fi @@ -342,6 +338,11 @@ for arg; do unset PARAM elif [ "$PARAM" = "stack" ]; then STACK_ID="$NOSPACE" + STACK_CMD="-s $NOSPACE" + STACK="/stack/$NOSPACE" + unset PARAM + elif [ "$PARAM" = "archive" ]; then + DATA_DIR="$NOSPACE" unset PARAM else if [ -z "${DOCKER_LIMITS[$DOCR]}" ]; then @@ -363,6 +364,13 @@ for arg; do fi done +if [ ! -z $LIMIT ]; then + echo "Error: No value given to --$PARAM" + echo + usage + exit -1 +fi + while getopts ':hleEd:g:p:v:s:n:a:c:j:b:m:r:R:M:G:D:L:N:C:Q:A:f:P:S:T:k:' option; do case "$option" in h) usage @@ -440,14 +448,24 @@ while getopts ':hleEd:g:p:v:s:n:a:c:j:b:m:r:R:M:G:D:L:N:C:Q:A:f:P:S:T:k:' option esac done if [ "$ARCHIVE" == "1" ]; then - if [ -z $DATA_DIR ]; then - echo "Running --archive without an external Prometheus directory, make sure you are using the -d command line option" - exit -1 - fi PROMETHEUS_COMMAND_LINE_OPTIONS_ARRAY+=(--storage.tsdb.retention.time=100y) RUN_LOKI=0 RUN_RENDERER="" - CONSUL_ADDRESS="127.0.0.1:0" + CONSUL_ADDRESS="-L 127.0.0.1:0" + if [ ! -d $DATA_DIR/ ]; then + echo "The giving data directory $DATA_DIR does not exist" + exit 1 + fi + if [ -f $DATA_DIR/scylla.txt ]; then + . $DATA_DIR/scylla.txt + echo "Taking version from $DATA_DIR/scylla.txt" + echo "Version set to $VERSIONS" + else + echo "scylla.txt not found in $DATA_DIR/. You can use it to start the monitoring stack with a given version" + echo "For example, to start the monitoring stack with version 2014.1 and manager 3.3" + echo 'echo VERSIONS="2024.1">'$DATA_DIR/scylla.txt + echo 'echo MANAGER_VERSION="3.3">>'$DATA_DIR/scylla.txt + fi fi if [ -z "$VERSIONS" ]; then @@ -455,6 +473,18 @@ if [ -z "$VERSIONS" ]; then exit 1 fi +if [ "$CURRENT_VERSION" = "master" ]; then + if [ "$ARCHIVE" = "" ]; then + echo "" + echo "*****************************************************" + echo "* WARNING: You are using the unstable master branch *" + echo "* Check the README.md file for the stable releases *" + echo "*****************************************************" + fi + echo "Generating the dashaobard ./generate-dashboards.sh -v $VERSIONS -m $MANAGER_VERSION" $STACK_CMD + ./generate-dashboards.sh -v $VERSIONS +fi + if [[ $DOCKER_PARAM = *"--net=host"* ]]; then if [ ! -z "$ALERTMANAGER_PORT" ] || [ ! -z "$GRAFANA_PORT" ] || [ ! -z $PROMETHEUS_PORT ]; then echo "Port mapping is not supported with host network, remove the -l flag from the command line" @@ -616,15 +646,16 @@ for val in "${PROMETHEUS_COMMAND_LINE_OPTIONS_ARRAY[@]}"; do fi done -./prometheus-config.sh -m $AM_ADDRESS $CONSUL_ADDRESS $PROMETHEUS_TARGETS +./prometheus-config.sh -m $AM_ADDRESS $STACK_CMD $CONSUL_ADDRESS $PROMETHEUS_TARGETS if [ "$DATA_DIR" != "" ] && [ "$ARCHIVE" != "1" ]; then DATE=$(date +"%Y-%m-%d_%H_%M_%S") if [ -f $DATA_DIR/scylla.txt ]; then mv $DATA_DIR/scylla.txt $DATA_DIR/scylla.$DATE.txt fi - echo COMMAND_LINE='"'"$@"'"' > $DATA_DIR/scylla.txt + echo LAST_COMMAND_LINE='"'"$@"'"' > $DATA_DIR/scylla.txt echo VERSIONS='"'"$VERSIONS"'"' >> $DATA_DIR/scylla.txt - echo CURRENT_VERSION='"'"$CURRENT_VERSION"'"' >> $DATA_DIR/scylla.txt + echo MANAGER_VERSION='"'"$MANAGER_VERSION"'"' >> $DATA_DIR/scylla.txt + echo MONITORING_VERSION='"'"$CURRENT_VERSION"'"' >> $DATA_DIR/scylla.txt echo PROMETHEUS_VERSION='"'"$PROMETHEUS_VERSION"'"' >> $DATA_DIR/scylla.txt echo LAST_RUN='"'"$DATE"'"' >> $DATA_DIR/scylla.txt fi @@ -646,7 +677,7 @@ else docker run -d $DOCKER_PARAM ${DOCKER_LIMITS["prometheus"]} $USER_PERMISSIONS \ $DATA_DIR_CMD \ "${group_args[@]}" \ - -v $PWD/prometheus/build/prometheus.yml:/etc/prometheus/prometheus.yml:z \ + -v $PWD/prometheus/build$STACK/prometheus.yml:/etc/prometheus/prometheus.yml:z \ -v $PROMETHEUS_RULES:z \ $SCYLLA_TARGET_FILE \ $SCYLLA_MANGER_TARGET_FILE \ @@ -731,5 +762,4 @@ done if [ ! -z "$DATDOGPARAM" ]; then ./start-datadog.sh $DATDOGPARAM -p $DB_ADDRESS fi - -./start-grafana.sh $LDAP_FILE $LOKI_ADDRESS $LIMITS $VOLUMES $PARAMS $BIND_ADDRESS_CONFIG $RUN_RENDERER $SPECIFIC_SOLUTION -p $DB_ADDRESS $GRAFNA_ANONYMOUS_ROLE -D "$DOCKER_PARAM" $GRAFANA_PORT $EXTERNAL_VOLUME -m $AM_ADDRESS -M $MANAGER_VERSION -v $VERSIONS $GRAFANA_ENV_COMMAND $GRAFANA_DASHBOARD_COMMAND $GRAFANA_ADMIN_PASSWORD +./start-grafana.sh $LDAP_FILE $LOKI_ADDRESS $LIMITS $VOLUMES $PARAMS $BIND_ADDRESS_CONFIG $RUN_RENDERER $SPECIFIC_SOLUTION -p $DB_ADDRESS $GRAFNA_ANONYMOUS_ROLE -D "$DOCKER_PARAM" $GRAFANA_PORT $EXTERNAL_VOLUME -m $AM_ADDRESS -M $MANAGER_VERSION -v $VERSIONS $GRAFANA_ENV_COMMAND $GRAFANA_DASHBOARD_COMMAND $GRAFANA_ADMIN_PASSWORD $STACK_CMD diff --git a/start-grafana.sh b/start-grafana.sh index ec8aed1bae..3442361f7c 100755 --- a/start-grafana.sh +++ b/start-grafana.sh @@ -98,7 +98,7 @@ for arg; do done usage="$(basename "$0") [-h] [-v comma separated versions ] [-g grafana port ] [-G path to external dir] [-n grafana container name ] [-p ip:port address of prometheus ] [-j additional dashboard to load to Grafana, multiple params are supported] [-c grafana environment variable, multiple params are supported] [-x http_proxy_host:port] [-m alert_manager address] [-a admin password] [ -M scylla-manager version ] [-D encapsulate docker param] [-Q Grafana anonymous role (Admin/Editor/Viewer)] [-S start with a system specific dashboard set] [-P ldap_config_file] -- loads the prometheus datasource and the Scylla dashboards into an existing grafana installation" -while getopts ':hlEg:n:p:v:a:x:c:j:m:G:M:D:A:S:P:L:Q:' option; do +while getopts ':hlEg:n:p:v:a:x:c:j:m:G:M:D:A:S:P:L:Q:s:' option; do case "$option" in h) echo "$usage" exit @@ -147,6 +147,9 @@ while getopts ':hlEg:n:p:v:a:x:c:j:m:G:M:D:A:S:P:L:Q:' option; do ;; S) SPECIFIC_SOLUTION="-S $OPTARG" ;; + s) STACK="/stack/$OPTARG" + STACK_CMD="-s $OPTARG" + ;; E) RUN_RENDERER="-E" ;; :) printf "missing argument for -%s\n" "$OPTARG" >&2 @@ -234,9 +237,8 @@ fi for val in "${GRAFANA_DASHBOARD_ARRAY[@]}"; do GRAFANA_DASHBOARD_COMMAND="$GRAFANA_DASHBOARD_COMMAND -j $val" done - -./generate-dashboards.sh -t $SPECIFIC_SOLUTION -v $VERSIONS -M $MANAGER_VERSION $GRAFANA_DASHBOARD_COMMAND -./grafana-datasource.sh $DATA_SOURCES +./generate-dashboards.sh -t $SPECIFIC_SOLUTION -v $VERSIONS -M $MANAGER_VERSION $STACK_CMD $GRAFANA_DASHBOARD_COMMAND +./grafana-datasource.sh $DATA_SOURCES $STACK_CMD if [[ ! $DOCKER_PARAM = *"--net=host"* ]]; then PORT_MAPPING="-p $BIND_ADDRESS$GRAFANA_PORT:3000" @@ -272,6 +274,10 @@ if [ ! -z $RUN_RENDERER ]; then GRAFANA_ENV_COMMAND="$GRAFANA_ENV_COMMAND -e GF_RENDERING_SERVER_URL=http://$HOST_ADDRESS:8081/render -e GF_RENDERING_CALLBACK_URL=http://$HOST_ADDRESS:$GRAFANA_PORT/" fi +if [ -z $STACK ]; then + STACK="" +fi + docker run -d $DOCKER_PARAM ${DOCKER_LIMITS["grafana"]} -i $USER_PERMISSIONS $PORT_MAPPING \ -e "GF_AUTH_BASIC_ENABLED=$GRAFANA_AUTH" \ -e "GF_AUTH_ANONYMOUS_ENABLED=$GRAFANA_AUTH_ANONYMOUS" \ @@ -281,7 +287,7 @@ docker run -d $DOCKER_PARAM ${DOCKER_LIMITS["grafana"]} -i $USER_PERMISSIONS $PO "${group_args[@]}" \ -v $PWD/grafana/build:/var/lib/grafana/dashboards:z \ -v $PWD/grafana/plugins:/var/lib/grafana/plugins:z \ - -v $PWD/grafana/provisioning:/var/lib/grafana/provisioning:z $EXTERNAL_VOLUME \ + -v $PWD/grafana$STACK/provisioning:/var/lib/grafana/provisioning:z $EXTERNAL_VOLUME \ -e "GF_PATHS_PROVISIONING=/var/lib/grafana/provisioning" \ -e "GF_SECURITY_ADMIN_PASSWORD=$GRAFANA_ADMIN_PASSWORD" \ -e "GF_ANALYTICS_GOOGLE_ANALYTICS_UA_ID=$UA_ANALTYICS" \