From a712d4e221d78c12ba20a3ac88bcbfa74ce7728a Mon Sep 17 00:00:00 2001 From: BlancLoup Date: Wed, 2 May 2018 22:27:44 +0500 Subject: [PATCH 1/4] First dirty commit with bags: added list reading from config, functions for url choosing --- openwisp-config/files/openwisp-nossl.config | 4 ++- openwisp-config/files/openwisp-ssl.config | 4 ++- openwisp-config/files/openwisp.agent | 37 ++++++++++++++++++--- openwisp-config/files/openwisp.init | 12 ++++++- 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/openwisp-config/files/openwisp-nossl.config b/openwisp-config/files/openwisp-nossl.config index 015f2a64..93d7e85e 100644 --- a/openwisp-config/files/openwisp-nossl.config +++ b/openwisp-config/files/openwisp-nossl.config @@ -2,7 +2,9 @@ # or https://github.com/openwisp/openwisp-config#configuration-options config controller 'http' - #option url 'http://openwisp2.mynetwork.com' + #list url 'https://openwisp2.mynetwork.com' + #list url 'https://mirror1.openwisp2.mynetwork.com' + #list url 'https://mirror1.openwisp2.mynetwork.com' #option interval '120' option verify_ssl '0' #option shared_secret '' diff --git a/openwisp-config/files/openwisp-ssl.config b/openwisp-config/files/openwisp-ssl.config index 6f93ad95..7c5b807c 100644 --- a/openwisp-config/files/openwisp-ssl.config +++ b/openwisp-config/files/openwisp-ssl.config @@ -2,7 +2,9 @@ # or https://github.com/openwisp/openwisp-config#configuration-options config controller 'http' - #option url 'https://openwisp2.mynetwork.com' + #list url 'https://openwisp2.mynetwork.com' + #list url 'https://mirror1.openwisp2.mynetwork.com' + #list url 'https://mirror1.openwisp2.mynetwork.com' #option interval '120' #option verify_ssl '1' #option shared_secret '' diff --git a/openwisp-config/files/openwisp.agent b/openwisp-config/files/openwisp.agent index 11324c18..e9ff4fb1 100755 --- a/openwisp-config/files/openwisp.agent +++ b/openwisp-config/files/openwisp.agent @@ -4,7 +4,7 @@ while [ -n "$1" ]; do case "$1" in --version|-v) export VERSION=1; break;; - --url) export URL=${2%/}; shift;; # strip trailing slash + --url) export URLS=$(echo ${2%/} | tr '^' ' '); shift;; # strip trailing slash --interval) export INTERVAL=$2; shift;; --verify-ssl) export VERIFY_SSL=$2; shift;; --uuid) export UUID="$2"; shift;; @@ -30,6 +30,36 @@ while [ -n "$1" ]; do shift; done +## define functions for url redundancy +set_url() { # get next url and change url related variables after url changing + if [ -n "$(urls_count)"]; then + # first-time init + urls_count=$(echo $URLS | wc -w) + URL=$(echo $URLS | cut -d" " -f1) + url_index=1 + else + URL=$(get_next_url()) + fi + BASEURL="${URL}controller" + REGISTRATION_URL="${URL}controller/register/" +} +check_url_connection() { # check connection + # curl -Is --connect-timeout 10 -k -X POST https://fight.today/controller/register/ + # need something like https://server/controller/heartbeat/ + if [ -n "$($FETCH_COMMAND -X POST -I $REGISTRATION_URL | grep \"X-Openwisp-Controller: true\")" ]; then return true + else return false + fi +} +get_next_url() { # get next url in list + if [[ $url_index==$urls_count ]]; then url_index=1; else url_index=$url_index+1; fi + return $(echo $URLS | cut -d" " -f${url_index}) +} +get_next_url_rand() { # get random url in list... if you need then rename this one with "get_next_url" + local random_index=$(head /dev/urandom | tr -dc $(echo "123456789" | cut --bytes=-$urls_count) | head -c1) + return $(echo $URLS | cut -d" " -f${random_index}) +} +## end define + if [ "$VERSION" -eq "1" ]; then VERSION=$(cat /etc/openwisp/VERSION) echo "openwisp-config $VERSION" @@ -58,8 +88,9 @@ CONSISTENT_KEY=${CONSISTENT_KEY:-1} CONNECT_TIMEOUT=${CONNECT_TIMEOUT:-15} MAX_TIME=${MAX_TIME:-30} MAC_INTERFACE=${MAC_INTERFACE:-eth0} +set_url() +FETCH_COMMAND="curl -s --connect-timeout $CONNECT_TIMEOUT --max-time $MAX_TIME" WORKING_DIR="/tmp/openwisp" -BASEURL="$URL/controller" CONFIGURATION_ARCHIVE="$WORKING_DIR/configuration.tar.gz" CONFIGURATION_CHECKSUM="$WORKING_DIR/checksum" CONFIGURATION_BACKUP="$WORKING_DIR/backup.tar.gz" @@ -67,9 +98,7 @@ REGISTRATION_PARAMETERS="$WORKING_DIR/registration_parameters" TEST_CHECKSUM="$WORKING_DIR/test_checksum" STATUS_REPORT="$WORKING_DIR/status_report" APPLYING_CONF="$WORKING_DIR/applying_conf" -REGISTRATION_URL="$URL/controller/register/" UNMANAGED_DIR="$WORKING_DIR/unmanaged" -FETCH_COMMAND="curl -s --connect-timeout $CONNECT_TIMEOUT --max-time $MAX_TIME" mkdir -p $WORKING_DIR mkdir -p $UNMANAGED_DIR diff --git a/openwisp-config/files/openwisp.init b/openwisp-config/files/openwisp.init index 68a7a52a..6e702126 100755 --- a/openwisp-config/files/openwisp.init +++ b/openwisp-config/files/openwisp.init @@ -8,7 +8,17 @@ PROG_NAME="OpenWISP config agent" start_service() { config_load openwisp - url=$(config_get http url) + #handle_url() { + # urls_count=urls_count+1 + # if [[ -z "$url" ]]; then url="$1"; fi + # local delimiter="" + # if [[ ! -z "$urls" ]]; then local delimiter=" "; fi + # # TODO: add check whitespaces in url + # urls="$urls$delimiter$1" + #} + # == Variant with backward compatibility. Will use it (anoter way: using config_list_foreach http url handle_url) + url=$(config_get http url | tr ' ' '^') # replace space delimiter only for sending as argument to agent script + interval=$(config_get http interval) verify_ssl=$(config_get http verify_ssl) uuid=$(config_get http uuid) From ad52e4a08d7bffa2ad32a971263923859f1e61fd Mon Sep 17 00:00:00 2001 From: BlancLoup Date: Thu, 3 May 2018 00:19:19 +0500 Subject: [PATCH 2/4] Add infinite URL check for each curl --- openwisp-config/files/openwisp.agent | 33 +++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/openwisp-config/files/openwisp.agent b/openwisp-config/files/openwisp.agent index e9ff4fb1..d8050983 100755 --- a/openwisp-config/files/openwisp.agent +++ b/openwisp-config/files/openwisp.agent @@ -31,7 +31,7 @@ while [ -n "$1" ]; do done ## define functions for url redundancy -set_url() { # get next url and change url related variables after url changing +set_url() { # get next url till success connection and change url related variables after url changing if [ -n "$(urls_count)"]; then # first-time init urls_count=$(echo $URLS | wc -w) @@ -40,8 +40,19 @@ set_url() { # get next url and change url related variables after url changing else URL=$(get_next_url()) fi + while [ ! check_url_connection() ]; do + logger -s "Failed to connect to controller during check connection. Url: ${URL}" \ + -t openwisp \ + -p daemon.err + sleep $(expr $INTERVAL / 4) + done BASEURL="${URL}controller" REGISTRATION_URL="${URL}controller/register/" + if [ ! -z "$UUID" ] && [ ! -z "$KEY" ]; then + CONFIGURATION_URL="$BASEURL/download-config/$UUID/?key=$KEY" + CHECKSUM_URL="$BASEURL/checksum/$UUID/?key=$KEY" + REPORT_URL="$BASEURL/report-status/$UUID/" + fi } check_url_connection() { # check connection # curl -Is --connect-timeout 10 -k -X POST https://fight.today/controller/register/ @@ -159,6 +170,10 @@ register() { local consistent_key=$(echo -n "$macaddr+$SHARED_SECRET" | md5sum | awk '{print $1}') params="$params&key=$consistent_key" fi + while [ ! $(check_url_connection()) ]; do + sleep $(expr $INTERVAL / 4) + set_url() + done $($FETCH_COMMAND --data $params \ --data-urlencode secret="$SHARED_SECRET" \ --data-urlencode name="$hostname" \ @@ -210,6 +225,10 @@ register() { # gets checksum from controller get_checksum() { + while [ ! $(check_url_connection()) ]; do + sleep $(expr $INTERVAL / 4) + set_url() + done $($FETCH_COMMAND -i $CHECKSUM_URL > $1) local exit_code=$? @@ -291,6 +310,10 @@ apply_configuration() { report_status() { # retry several times for i in $(seq 1 10); do + while [ ! $(check_url_connection()) ]; do + sleep $(expr $INTERVAL / 4) + set_url() + done $($FETCH_COMMAND -i --data "key=$KEY&status=$1" $REPORT_URL > $STATUS_REPORT) local exit_code=$? if [ "$exit_code" == "0" ]; then @@ -364,6 +387,10 @@ test_configuration() { perform_default_test() { # max 3 attempts to get checksum for i in $(seq 1 3); do + while [ ! $(check_url_connection()) ]; do + sleep $(expr $INTERVAL / 4) + set_url() + done $($FETCH_COMMAND -i --connect-timeout 5 --max-time 5 $CHECKSUM_URL > $TEST_CHECKSUM) local result=$? if [ $result -gt 0 ]; then @@ -406,6 +433,10 @@ update_configuration() { -p daemon.info # download configuration + while [ ! $(check_url_connection()) ]; do + sleep $(expr $INTERVAL / 4) + set_url() + done $($FETCH_COMMAND $CONFIGURATION_URL -o $CONFIGURATION_ARCHIVE) local exit_code=$? From 9719663e95e2f605b89d62b89b0f581975f599e0 Mon Sep 17 00:00:00 2001 From: BlancLoup Date: Thu, 3 May 2018 00:44:39 +0500 Subject: [PATCH 3/4] Revert "Add infinite URL check for each curl" This reverts commit ad52e4a08d7bffa2ad32a971263923859f1e61fd. --- openwisp-config/files/openwisp.agent | 33 +--------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/openwisp-config/files/openwisp.agent b/openwisp-config/files/openwisp.agent index d8050983..e9ff4fb1 100755 --- a/openwisp-config/files/openwisp.agent +++ b/openwisp-config/files/openwisp.agent @@ -31,7 +31,7 @@ while [ -n "$1" ]; do done ## define functions for url redundancy -set_url() { # get next url till success connection and change url related variables after url changing +set_url() { # get next url and change url related variables after url changing if [ -n "$(urls_count)"]; then # first-time init urls_count=$(echo $URLS | wc -w) @@ -40,19 +40,8 @@ set_url() { # get next url till success connection and change url related variab else URL=$(get_next_url()) fi - while [ ! check_url_connection() ]; do - logger -s "Failed to connect to controller during check connection. Url: ${URL}" \ - -t openwisp \ - -p daemon.err - sleep $(expr $INTERVAL / 4) - done BASEURL="${URL}controller" REGISTRATION_URL="${URL}controller/register/" - if [ ! -z "$UUID" ] && [ ! -z "$KEY" ]; then - CONFIGURATION_URL="$BASEURL/download-config/$UUID/?key=$KEY" - CHECKSUM_URL="$BASEURL/checksum/$UUID/?key=$KEY" - REPORT_URL="$BASEURL/report-status/$UUID/" - fi } check_url_connection() { # check connection # curl -Is --connect-timeout 10 -k -X POST https://fight.today/controller/register/ @@ -170,10 +159,6 @@ register() { local consistent_key=$(echo -n "$macaddr+$SHARED_SECRET" | md5sum | awk '{print $1}') params="$params&key=$consistent_key" fi - while [ ! $(check_url_connection()) ]; do - sleep $(expr $INTERVAL / 4) - set_url() - done $($FETCH_COMMAND --data $params \ --data-urlencode secret="$SHARED_SECRET" \ --data-urlencode name="$hostname" \ @@ -225,10 +210,6 @@ register() { # gets checksum from controller get_checksum() { - while [ ! $(check_url_connection()) ]; do - sleep $(expr $INTERVAL / 4) - set_url() - done $($FETCH_COMMAND -i $CHECKSUM_URL > $1) local exit_code=$? @@ -310,10 +291,6 @@ apply_configuration() { report_status() { # retry several times for i in $(seq 1 10); do - while [ ! $(check_url_connection()) ]; do - sleep $(expr $INTERVAL / 4) - set_url() - done $($FETCH_COMMAND -i --data "key=$KEY&status=$1" $REPORT_URL > $STATUS_REPORT) local exit_code=$? if [ "$exit_code" == "0" ]; then @@ -387,10 +364,6 @@ test_configuration() { perform_default_test() { # max 3 attempts to get checksum for i in $(seq 1 3); do - while [ ! $(check_url_connection()) ]; do - sleep $(expr $INTERVAL / 4) - set_url() - done $($FETCH_COMMAND -i --connect-timeout 5 --max-time 5 $CHECKSUM_URL > $TEST_CHECKSUM) local result=$? if [ $result -gt 0 ]; then @@ -433,10 +406,6 @@ update_configuration() { -p daemon.info # download configuration - while [ ! $(check_url_connection()) ]; do - sleep $(expr $INTERVAL / 4) - set_url() - done $($FETCH_COMMAND $CONFIGURATION_URL -o $CONFIGURATION_ARCHIVE) local exit_code=$? From dc3dd5a595afe883dc4f06c36d3d43d1630a0e2c Mon Sep 17 00:00:00 2001 From: BlancLoup Date: Thu, 3 May 2018 00:51:30 +0500 Subject: [PATCH 4/4] Add infinite URL check for each curl --- openwisp-config/files/openwisp.agent | 35 ++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/openwisp-config/files/openwisp.agent b/openwisp-config/files/openwisp.agent index e9ff4fb1..c473465b 100755 --- a/openwisp-config/files/openwisp.agent +++ b/openwisp-config/files/openwisp.agent @@ -31,7 +31,7 @@ while [ -n "$1" ]; do done ## define functions for url redundancy -set_url() { # get next url and change url related variables after url changing +set_url() { # get next url till success connection and change url related variables after url changing if [ -n "$(urls_count)"]; then # first-time init urls_count=$(echo $URLS | wc -w) @@ -40,11 +40,22 @@ set_url() { # get next url and change url related variables after url changing else URL=$(get_next_url()) fi + while [ ! check_url_connection() ]; do + logger -s "Failed to connect to controller during check connection. Url: ${URL}" \ + -t openwisp \ + -p daemon.err + sleep $(expr $INTERVAL / 4) + done BASEURL="${URL}controller" REGISTRATION_URL="${URL}controller/register/" + if [ ! -z "$UUID" ] && [ ! -z "$KEY" ]; then + CONFIGURATION_URL="$BASEURL/download-config/$UUID/?key=$KEY" + CHECKSUM_URL="$BASEURL/checksum/$UUID/?key=$KEY" + REPORT_URL="$BASEURL/report-status/$UUID/" + fi } check_url_connection() { # check connection - # curl -Is --connect-timeout 10 -k -X POST https://fight.today/controller/register/ + # curl -Is --connect-timeout 10 -k -X POST https://server/controller/register/ # need something like https://server/controller/heartbeat/ if [ -n "$($FETCH_COMMAND -X POST -I $REGISTRATION_URL | grep \"X-Openwisp-Controller: true\")" ]; then return true else return false @@ -159,6 +170,10 @@ register() { local consistent_key=$(echo -n "$macaddr+$SHARED_SECRET" | md5sum | awk '{print $1}') params="$params&key=$consistent_key" fi + while [ ! $(check_url_connection()) ]; do + sleep $(expr $INTERVAL / 4) + set_url() + done $($FETCH_COMMAND --data $params \ --data-urlencode secret="$SHARED_SECRET" \ --data-urlencode name="$hostname" \ @@ -210,6 +225,10 @@ register() { # gets checksum from controller get_checksum() { + while [ ! $(check_url_connection()) ]; do + sleep $(expr $INTERVAL / 4) + set_url() + done $($FETCH_COMMAND -i $CHECKSUM_URL > $1) local exit_code=$? @@ -291,6 +310,10 @@ apply_configuration() { report_status() { # retry several times for i in $(seq 1 10); do + while [ ! $(check_url_connection()) ]; do + sleep $(expr $INTERVAL / 4) + set_url() + done $($FETCH_COMMAND -i --data "key=$KEY&status=$1" $REPORT_URL > $STATUS_REPORT) local exit_code=$? if [ "$exit_code" == "0" ]; then @@ -364,6 +387,10 @@ test_configuration() { perform_default_test() { # max 3 attempts to get checksum for i in $(seq 1 3); do + while [ ! $(check_url_connection()) ]; do + sleep $(expr $INTERVAL / 4) + set_url() + done $($FETCH_COMMAND -i --connect-timeout 5 --max-time 5 $CHECKSUM_URL > $TEST_CHECKSUM) local result=$? if [ $result -gt 0 ]; then @@ -406,6 +433,10 @@ update_configuration() { -p daemon.info # download configuration + while [ ! $(check_url_connection()) ]; do + sleep $(expr $INTERVAL / 4) + set_url() + done $($FETCH_COMMAND $CONFIGURATION_URL -o $CONFIGURATION_ARCHIVE) local exit_code=$?