Improved experience when working with Archive

generate-dashboards.sh

@@ -28,7 +28,7 @@ fi
 
 usage="$(basename "$0") [-h] [-v comma separated versions ]  [-D] [-j additional dashboard to load to Grafana, multiple params are supported] [-M scylla-manager version ] [-t] [-F force update] [-S start with a system specific dashboard set] -- Generates the grafana dashboards and their load files"
 BASE_DASHBOARD_DIR="grafana/provisioning/dashboards"
-while getopts ':htDv:j:M:S:B:P:F' option; do
+while getopts ':htDv:j:M:S:B:P:s:F' option; do
   case "$option" in
     h) echo "$usage"
        exit
@@ -48,6 +48,9 @@ while getopts ':htDv:j:M:S:B:P:F' option; do
        ;;
     S) SPECIFIC_SOLUTION="$OPTARG"
        ;;
+    s) STACK="$OPTARG"
+       BASE_DASHBOARD_DIR="grafana/stack/$OPTARG/provisioning/dashboards"
+       ;;
     D) VERSIONS=${SUPPORTED_VERSIONS[$BRANCH_VERSION]}
        FORMAT_COMAND="$FORMAT_COMAND -v $VERSIONS"
        MANAGER_VERSION=${MANAGER_SUPPORTED_VERSIONS[$BRANCH_VERSION]}
@@ -62,7 +65,6 @@ done
 if [[ -z "$TEST_ONLY" ]]; then
     mkdir -p grafana/build
 fi
-
 mkdir -p $BASE_DASHBOARD_DIR
 rm -f $BASE_DASHBOARD_DIR/load.*.yaml
 

grafana-datasource.sh

@@ -5,13 +5,13 @@ if [ "$1" = "" ]; then
     echo "$usage"
     exit
 fi
-
+BASE_DIR="grafana/provisioning/datasources"
 if [ "$1" = "--compose" ]; then
     DB_ADDRESS="aprom:9090"
     ALERT_MANAGER_ADDRESS="aalert:9093"
     LOKI_ADDRESS="loki:3100"
 else
-    while getopts ':hlEg:n:p:v:a:x:c:j:m:G:M:D:A:S:P:L:Q:' option; do
+    while getopts ':hlEg:n:p:v:a:x:c:j:m:G:M:D:A:S:P:L:Q:s:' option; do
       case "$option" in
         h) echo "$usage"
            exit
@@ -23,6 +23,8 @@ else
            ;;
         L) LOKI_ADDRESS=$OPTARG
            ;;
+        s) BASE_DIR="grafana/stack/$OPTARG/provisioning/datasources"
+           ;;
         :) printf "missing argument for -%s\n" "$OPTARG" >&2
            echo "$usage" >&2
            exit 1
@@ -34,16 +36,16 @@ else
       esac
     done
 fi
-mkdir -p grafana/provisioning/datasources
-rm -f grafana/provisioning/datasources/datasource.yaml
-sed "s/DB_ADDRESS/$DB_ADDRESS/" grafana/datasource.yml | sed "s/AM_ADDRESS/$ALERT_MANAGER_ADDRESS/" > grafana/provisioning/datasources/datasource.yaml
+mkdir -p $BASE_DIR
+rm -f $BASE_DIR/datasource.yaml
+sed "s/DB_ADDRESS/$DB_ADDRESS/" grafana/datasource.yml | sed "s/AM_ADDRESS/$ALERT_MANAGER_ADDRESS/" > $BASE_DIR/datasource.yaml
 if [ -z "$LOKI_ADDRESS" ]; then
-    rm -f grafana/provisioning/datasources/datasource.loki.yaml
+    rm -f $BASE_DIR/datasource.loki.yaml
 else
-    sed "s/LOKI_ADDRESS/$LOKI_ADDRESS/" grafana/datasource.loki.yml> grafana/provisioning/datasources/datasource.loki.yaml
+    sed "s/LOKI_ADDRESS/$LOKI_ADDRESS/" grafana/datasource.loki.yml> $BASE_DIR/datasource.loki.yaml
 fi
 if [ -z "$SCYLLA_USER" ] || [ -z "$SCYLLA_PSSWD" ]; then
-    cp grafana/datasource.scylla.yml grafana/provisioning/datasources/datasource.scylla.yml
+    cp grafana/datasource.scylla.yml $BASE_DIR/datasource.scylla.yml
 else
-    sed "s/SCYLLA_USER/$SCYLLA_USER/" grafana/datasource.psswd.scylla.yml |sed "s/SCYLLA_PSSWD/$SCYLLA_PSSWD/">grafana/provisioning/datasources/datasource.scylla.yml
+    sed "s/SCYLLA_USER/$SCYLLA_USER/" grafana/datasource.psswd.scylla.yml |sed "s/SCYLLA_PSSWD/$SCYLLA_PSSWD/">$BASE_DIR/datasource.scylla.yml
 fi

prometheus-config.sh

@@ -2,6 +2,7 @@
 usage="$(basename "$0") [-h] [-m alert_manager address]  [-L] [-T additional-prometheus-targets] [--compose] -- Generate grafna's datasource file"
 CONSUL_ADDRESS=""
 COMPOSE=0
+BASE_DIR="$PWD/prometheus/build"
 if [ -f  env.sh ]; then
     . env.sh
 fi
@@ -37,7 +38,7 @@ for arg; do
     esac
 done
 
-while getopts ':hL:m:T:E:' option; do
+while getopts ':hL:m:T:E:s:' option; do
   case "$option" in
     h) echo "$usage"
        exit
@@ -48,6 +49,8 @@ while getopts ':hL:m:T:E:' option; do
        ;;
     m) AM_ADDRESS="$OPTARG"
        ;;
+    s) BASE_DIR="$PWD/prometheus/build/stack/$OPTARG"
+       ;;
     E) EVALUATION_INTERVAL="$OPTARG"
        ;;
     :) printf "missing argument for -%s\n" "$OPTARG" >&2
@@ -61,37 +64,37 @@ while getopts ':hL:m:T:E:' option; do
   esac
 done
 
-mkdir -p $PWD/prometheus/build/
+mkdir -p $BASE_DIR/
 if [ -z $CONSUL_ADDRESS ]; then
-    sed "s/AM_ADDRESS/$AM_ADDRESS/" $PWD/prometheus/prometheus.yml.template > $PWD/prometheus/build/prometheus.yml
+    sed "s/AM_ADDRESS/$AM_ADDRESS/" $PWD/prometheus/prometheus.yml.template > $BASE_DIR/prometheus.yml
 else
     if [[ ! $CONSUL_ADDRESS = *":"* ]]; then
         CONSUL_ADDRESS="$CONSUL_ADDRESS:5090"
     fi
-    sed "s/AM_ADDRESS/$AM_ADDRESS/" $PWD/prometheus/prometheus.consul.yml.template| sed "s/MANAGER_ADDRESS/$CONSUL_ADDRESS/" > $PWD/prometheus/build/prometheus.yml
+    sed "s/AM_ADDRESS/$AM_ADDRESS/" $PWD/prometheus/prometheus.consul.yml.template| sed "s/MANAGER_ADDRESS/$CONSUL_ADDRESS/" > $BASE_DIR/prometheus.yml
 fi
 
 if [[ "$EVALUATION_INTERVAL" != "" ]]; then
-    sed -i "s/  evaluation_interval: [[:digit:]]*.*/  evaluation_interval: ${EVALUATION_INTERVAL}/g" $PWD/prometheus/build/prometheus.yml
+    sed -i "s/  evaluation_interval: [[:digit:]]*.*/  evaluation_interval: ${EVALUATION_INTERVAL}/g" $BASE_DIR/prometheus.yml
 fi
 if [ "$NO_CAS" = "1" ] && [ "$NO_CDC" = "1" ]; then
-    sed -i "s/ *# FILTER_METRICS.*/    - source_labels: [__name__]\\n      regex: '(.*_cdc_.*|.*_cas.*)'\\n      action: drop/g" $PWD/prometheus/build/prometheus.yml
+    sed -i "s/ *# FILTER_METRICS.*/    - source_labels: [__name__]\\n      regex: '(.*_cdc_.*|.*_cas.*)'\\n      action: drop/g" $BASE_DIR/prometheus.yml
 elif [ "$NO_CAS" = "1" ]; then
-    sed -i "s/ *# FILTER_METRICS.*/    - source_labels: [__name__]\\n      regex: '(.*_cas.*)'\\n      action: drop/g" $PWD/prometheus/build/prometheus.yml
+    sed -i "s/ *# FILTER_METRICS.*/    - source_labels: [__name__]\\n      regex: '(.*_cas.*)'\\n      action: drop/g" $BASE_DIR/prometheus.yml
 elif [ "$NO_CDC" = "1" ]; then
-    sed -i "s/ *# FILTER_METRICS.*/    - source_labels: [__name__]\\n      regex: '(.*_cdc_.*)'\\n      action: drop/g" $PWD/prometheus/build/prometheus.yml
+    sed -i "s/ *# FILTER_METRICS.*/    - source_labels: [__name__]\\n      regex: '(.*_cdc_.*)'\\n      action: drop/g" $BASE_DIR/prometheus.yml
 fi
 if [ "$NO_NODE_EXPORTER_FILE" = "1" ]; then
-    sed -i "s/ *# NODE_EXPORTER_PORT_MAPPING.*/    - source_labels: [__address__]\\n      regex:  '(.*):\\\\d+'\\n      target_label: __address__\\n      replacement: '\$\{1\}'\\n/g" $PWD/prometheus/build/prometheus.yml
+    sed -i "s/ *# NODE_EXPORTER_PORT_MAPPING.*/    - source_labels: [__address__]\\n      regex:  '(.*):\\\\d+'\\n      target_label: __address__\\n      replacement: '\$\{1\}'\\n/g" $BASE_DIR/prometheus.yml
 fi
 if [ "$NO_MANAGER_AGENT_FILE" = "1" ]; then
-    sed -i "s/ *# MANAGER_AGENT_PORT_MAPPING.*/    - source_labels: [__address__]\\n      regex:  '(.*):\\\\d+'\\n      target_label: __address__\\n      replacement: \'\$\{1\}\'\\n/g" $PWD/prometheus/build/prometheus.yml
+    sed -i "s/ *# MANAGER_AGENT_PORT_MAPPING.*/    - source_labels: [__address__]\\n      regex:  '(.*):\\\\d+'\\n      target_label: __address__\\n      replacement: \'\$\{1\}\'\\n/g" $BASE_DIR/prometheus.yml
 fi
 
 for val in "${PROMETHEUS_TARGETS[@]}"; do
     if [[ ! -f $val ]]; then
         echo "Target file $val does not exists"
         exit 1
     fi
-    cat $val >> $PWD/prometheus/build/prometheus.yml
+    cat $val >> $BASE_DIR/prometheus.yml
 done

start-all.sh

@@ -31,18 +31,6 @@ if [ "$1" = "--version" ]; then
     exit
 fi
 
-if [ "$CURRENT_VERSION" = "master" ]; then
-    echo ""
-    echo "*****************************************************"
-    echo "* WARNING: You are using the unstable master branch *"
-    echo "* Check the README.md file for the stable releases  *"
-    echo "*****************************************************"
-    echo ""
-    echo "Make sure you run generate-dashboards.sh to generate your dashboards."
-    echo 'For example to use Scylla 2021.1 run `./generate-dashboards.sh -F -v 2021.1`'
-    echo ""
-fi
-
 if [ "`id -u`" -eq 0 ]; then
     echo "Running as root is not advised, please check the documentation on how to run as non-root user"
     USER_PERMISSIONS="-u 0:0"
@@ -113,7 +101,7 @@ Options:
   --stack id                     - Use this option when running a secondary stack, id could be 1-4
   --limit container,param        - Allow to set a specific Docker parameter for a container, where container can be:
                                    prometheus, grafana, alertmanager, loki, sidecar, grafanarender
-  --archive                      - Treat data directory as an archive. This disables Prometheus time-to-live (infinite retention).
+  --archive  data-directory      - Treat data directory as an archive. This disables Prometheus time-to-live (infinite retention), and would run a minimal mode
 The script starts Scylla Monitoring stack.
 "
   echo "$__usage"
@@ -262,7 +250,7 @@ for arg; do
                 ;;
             (--param)
                 LIMIT="1"
-                PARAM="1"
+                PARAM="param"
                 ;;
             (--evaluation-interval)
                 LIMIT="1"
@@ -301,6 +289,8 @@ for arg; do
                 ;;
             (--archive)
                 ARCHIVE="1"
+                LIMIT="1"
+                PARAM="archive"
                 ;;
             (*) set -- "$@" "$arg"
                 ;;
@@ -309,7 +299,13 @@ for arg; do
         DOCR=`echo $arg|cut -d',' -f1`
         VALUE=`echo $arg|cut -d',' -f2-|sed 's/#/ /g'`
         NOSPACE=`echo $arg|sed 's/ /#/g'`
-        if [ "$PARAM" = "1" ]; then
+        if [[ $NOSPACE == --* ]]; then
+            echo "Error: No value given to --$PARAM"
+            echo
+            usage
+            exit 1
+        fi
+        if [ "$PARAM" = "param" ]; then
             if [ -z "${DOCKER_PARAMS[$DOCR]}" ]; then
                 DOCKER_PARAMS[$DOCR]=""
             fi
@@ -342,6 +338,11 @@ for arg; do
             unset PARAM
         elif [ "$PARAM" = "stack" ]; then
             STACK_ID="$NOSPACE"
+            STACK_CMD="-s $NOSPACE"
+            STACK="/stack/$NOSPACE"
+            unset PARAM
+        elif [ "$PARAM" = "archive" ]; then
+            DATA_DIR="$NOSPACE"
             unset PARAM
         else
             if [ -z "${DOCKER_LIMITS[$DOCR]}" ]; then
@@ -363,6 +364,13 @@ for arg; do
     fi
 done
 
+if [ ! -z $LIMIT ]; then
+    echo "Error: No value given to --$PARAM"
+    echo
+    usage
+    exit -1
+fi
+
 while getopts ':hleEd:g:p:v:s:n:a:c:j:b:m:r:R:M:G:D:L:N:C:Q:A:f:P:S:T:k:' option; do
   case "$option" in
     h) usage
@@ -440,21 +448,43 @@ while getopts ':hleEd:g:p:v:s:n:a:c:j:b:m:r:R:M:G:D:L:N:C:Q:A:f:P:S:T:k:' option
   esac
 done
 if [ "$ARCHIVE" == "1" ]; then
-    if [ -z $DATA_DIR ]; then
-        echo "Running --archive without an external Prometheus directory, make sure you are using the -d command line option"
-        exit -1
-    fi
     PROMETHEUS_COMMAND_LINE_OPTIONS_ARRAY+=(--storage.tsdb.retention.time=100y)
     RUN_LOKI=0
     RUN_RENDERER=""
-    CONSUL_ADDRESS="127.0.0.1:0"
+    CONSUL_ADDRESS="-L 127.0.0.1:0"
+    if [ ! -d $DATA_DIR/ ]; then
+        echo "The giving data directory $DATA_DIR does not exist"
+        exit 1
+    fi
+    if [ -f $DATA_DIR/scylla.txt ]; then
+        . $DATA_DIR/scylla.txt
+        echo "Taking version from $DATA_DIR/scylla.txt"
+        echo "Version set to $VERSIONS"
+    else
+        echo "scylla.txt not found in $DATA_DIR/. You can use it to start the monitoring stack with a given version"
+        echo "For example, to start the monitoring stack with version 2014.1 and manager 3.3"
+        echo 'echo VERSIONS="2024.1">'$DATA_DIR/scylla.txt
+        echo 'echo MANAGER_VERSION="3.3">>'$DATA_DIR/scylla.txt
+    fi
 fi
 
 if [ -z "$VERSIONS" ]; then
   echo "Scylla-version was not not found, add the -v command-line with a specific version (i.e. -v 2021.1)"
   exit 1
 fi
 
+if [ "$CURRENT_VERSION" = "master" ]; then
+    if [ "$ARCHIVE" = "" ]; then
+        echo ""
+        echo "*****************************************************"
+        echo "* WARNING: You are using the unstable master branch *"
+        echo "* Check the README.md file for the stable releases  *"
+        echo "*****************************************************"
+    fi
+    echo "Generating the dashaobard ./generate-dashboards.sh -v $VERSIONS -m $MANAGER_VERSION" $STACK_CMD
+    ./generate-dashboards.sh -v $VERSIONS
+fi
+
 if [[ $DOCKER_PARAM = *"--net=host"* ]]; then
     if [ ! -z "$ALERTMANAGER_PORT" ] || [ ! -z "$GRAFANA_PORT" ] || [ ! -z $PROMETHEUS_PORT ]; then
         echo "Port mapping is not supported with host network, remove the -l flag from the command line"
@@ -616,15 +646,16 @@ for val in "${PROMETHEUS_COMMAND_LINE_OPTIONS_ARRAY[@]}"; do
     fi
 done
 
-./prometheus-config.sh -m $AM_ADDRESS $CONSUL_ADDRESS $PROMETHEUS_TARGETS
+./prometheus-config.sh -m $AM_ADDRESS $STACK_CMD $CONSUL_ADDRESS $PROMETHEUS_TARGETS
 if [ "$DATA_DIR" != "" ] && [ "$ARCHIVE" != "1" ]; then
     DATE=$(date +"%Y-%m-%d_%H_%M_%S")
     if [ -f $DATA_DIR/scylla.txt ]; then
         mv $DATA_DIR/scylla.txt $DATA_DIR/scylla.$DATE.txt
     fi
-    echo COMMAND_LINE='"'"$@"'"' > $DATA_DIR/scylla.txt
+    echo LAST_COMMAND_LINE='"'"$@"'"' > $DATA_DIR/scylla.txt
     echo VERSIONS='"'"$VERSIONS"'"' >> $DATA_DIR/scylla.txt
-    echo CURRENT_VERSION='"'"$CURRENT_VERSION"'"' >> $DATA_DIR/scylla.txt
+    echo MANAGER_VERSION='"'"$MANAGER_VERSION"'"' >> $DATA_DIR/scylla.txt
+    echo MONITORING_VERSION='"'"$CURRENT_VERSION"'"' >> $DATA_DIR/scylla.txt
     echo PROMETHEUS_VERSION='"'"$PROMETHEUS_VERSION"'"' >> $DATA_DIR/scylla.txt
     echo LAST_RUN='"'"$DATE"'"' >> $DATA_DIR/scylla.txt
 fi
@@ -646,7 +677,7 @@ else
 docker run -d $DOCKER_PARAM ${DOCKER_LIMITS["prometheus"]} $USER_PERMISSIONS \
      $DATA_DIR_CMD \
      "${group_args[@]}" \
-     -v $PWD/prometheus/build/prometheus.yml:/etc/prometheus/prometheus.yml:z \
+     -v $PWD/prometheus/build$STACK/prometheus.yml:/etc/prometheus/prometheus.yml:z \
      -v $PROMETHEUS_RULES:z \
      $SCYLLA_TARGET_FILE \
      $SCYLLA_MANGER_TARGET_FILE \
@@ -731,5 +762,4 @@ done
 if [ ! -z "$DATDOGPARAM" ]; then
    ./start-datadog.sh $DATDOGPARAM -p $DB_ADDRESS
 fi
-
-./start-grafana.sh $LDAP_FILE $LOKI_ADDRESS $LIMITS $VOLUMES $PARAMS $BIND_ADDRESS_CONFIG $RUN_RENDERER $SPECIFIC_SOLUTION -p $DB_ADDRESS $GRAFNA_ANONYMOUS_ROLE -D "$DOCKER_PARAM" $GRAFANA_PORT $EXTERNAL_VOLUME -m $AM_ADDRESS -M $MANAGER_VERSION -v $VERSIONS $GRAFANA_ENV_COMMAND $GRAFANA_DASHBOARD_COMMAND $GRAFANA_ADMIN_PASSWORD
+./start-grafana.sh $LDAP_FILE $LOKI_ADDRESS $LIMITS $VOLUMES $PARAMS $BIND_ADDRESS_CONFIG $RUN_RENDERER $SPECIFIC_SOLUTION -p $DB_ADDRESS $GRAFNA_ANONYMOUS_ROLE -D "$DOCKER_PARAM" $GRAFANA_PORT $EXTERNAL_VOLUME -m $AM_ADDRESS -M $MANAGER_VERSION -v $VERSIONS $GRAFANA_ENV_COMMAND $GRAFANA_DASHBOARD_COMMAND $GRAFANA_ADMIN_PASSWORD $STACK_CMD

start-grafana.sh

@@ -98,7 +98,7 @@ for arg; do
 done
 usage="$(basename "$0") [-h] [-v comma separated versions ] [-g grafana port ] [-G path to external dir] [-n grafana container name ] [-p ip:port address of prometheus ] [-j additional dashboard to load to Grafana, multiple params are supported] [-c grafana environment variable, multiple params are supported] [-x http_proxy_host:port] [-m alert_manager address] [-a admin password] [ -M scylla-manager version ] [-D encapsulate docker param] [-Q Grafana anonymous role (Admin/Editor/Viewer)] [-S start with a system specific dashboard set] [-P ldap_config_file] -- loads the prometheus datasource and the Scylla dashboards into an existing grafana installation"
 
-while getopts ':hlEg:n:p:v:a:x:c:j:m:G:M:D:A:S:P:L:Q:' option; do
+while getopts ':hlEg:n:p:v:a:x:c:j:m:G:M:D:A:S:P:L:Q:s:' option; do
   case "$option" in
     h) echo "$usage"
        exit
@@ -147,6 +147,9 @@ while getopts ':hlEg:n:p:v:a:x:c:j:m:G:M:D:A:S:P:L:Q:' option; do
        ;;
     S) SPECIFIC_SOLUTION="-S $OPTARG"
        ;;
+    s) STACK="/stack/$OPTARG"
+       STACK_CMD="-s $OPTARG"
+       ;;
     E) RUN_RENDERER="-E"
        ;;
     :) printf "missing argument for -%s\n" "$OPTARG" >&2
@@ -234,9 +237,8 @@ fi
 for val in "${GRAFANA_DASHBOARD_ARRAY[@]}"; do
         GRAFANA_DASHBOARD_COMMAND="$GRAFANA_DASHBOARD_COMMAND -j $val"
 done
-
-./generate-dashboards.sh -t $SPECIFIC_SOLUTION -v $VERSIONS -M $MANAGER_VERSION $GRAFANA_DASHBOARD_COMMAND
-./grafana-datasource.sh $DATA_SOURCES
+./generate-dashboards.sh -t $SPECIFIC_SOLUTION -v $VERSIONS -M $MANAGER_VERSION $STACK_CMD $GRAFANA_DASHBOARD_COMMAND
+./grafana-datasource.sh $DATA_SOURCES $STACK_CMD
 
 if [[ ! $DOCKER_PARAM = *"--net=host"* ]]; then
     PORT_MAPPING="-p $BIND_ADDRESS$GRAFANA_PORT:3000"
@@ -272,6 +274,10 @@ if [ ! -z $RUN_RENDERER ]; then
     GRAFANA_ENV_COMMAND="$GRAFANA_ENV_COMMAND -e GF_RENDERING_SERVER_URL=http://$HOST_ADDRESS:8081/render -e GF_RENDERING_CALLBACK_URL=http://$HOST_ADDRESS:$GRAFANA_PORT/"
 fi
 
+if [ -z $STACK ]; then
+    STACK=""
+fi
+
 docker run -d $DOCKER_PARAM ${DOCKER_LIMITS["grafana"]} -i $USER_PERMISSIONS $PORT_MAPPING \
      -e "GF_AUTH_BASIC_ENABLED=$GRAFANA_AUTH" \
      -e "GF_AUTH_ANONYMOUS_ENABLED=$GRAFANA_AUTH_ANONYMOUS" \
@@ -281,7 +287,7 @@ docker run -d $DOCKER_PARAM ${DOCKER_LIMITS["grafana"]} -i $USER_PERMISSIONS $PO
      "${group_args[@]}" \
      -v $PWD/grafana/build:/var/lib/grafana/dashboards:z \
      -v $PWD/grafana/plugins:/var/lib/grafana/plugins:z \
-     -v $PWD/grafana/provisioning:/var/lib/grafana/provisioning:z $EXTERNAL_VOLUME \
+     -v $PWD/grafana$STACK/provisioning:/var/lib/grafana/provisioning:z $EXTERNAL_VOLUME \
      -e "GF_PATHS_PROVISIONING=/var/lib/grafana/provisioning" \
      -e "GF_SECURITY_ADMIN_PASSWORD=$GRAFANA_ADMIN_PASSWORD" \
      -e "GF_ANALYTICS_GOOGLE_ANALYTICS_UA_ID=$UA_ANALTYICS" \