Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Properly authenticate at proxy server #652

Merged
merged 4 commits into from
Feb 26, 2019
Merged

Properly authenticate at proxy server #652

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cc48853
Properly authenticate at proxy server
danielmitterdorfer Feb 22, 2019
421f07c
Add proxy integration test
danielmitterdorfer Feb 26, 2019
f8f2193
Don't use system tmp dir
danielmitterdorfer Feb 26, 2019
2f96a15
Don't please
danielmitterdorfer Feb 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions esrally/utils/net.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,16 @@ def init():
global __HTTP
proxy_url = os.getenv("http_proxy")
if proxy_url and len(proxy_url) > 0:
logger.info("Rally connects via proxy URL [%s] to the Internet (picked up from the environment variable [http_proxy]).", proxy_url)
__HTTP = urllib3.ProxyManager(proxy_url, cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())
parsed_url = urllib3.util.parse_url(proxy_url)
logger.info("Connecting via proxy URL [%s] to the Internet (picked up from the env variable [http_proxy]).",
proxy_url)
__HTTP = urllib3.ProxyManager(proxy_url,
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where(),
# appropriate headers will only be set if there is auth info
proxy_headers=urllib3.make_headers(proxy_basic_auth=parsed_url.auth))
else:
logger.info("Rally connects directly to the Internet (no proxy support).")
logger.info("Connecting directly to the Internet (no proxy support).")
__HTTP = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())


Expand Down
147 changes: 135 additions & 12 deletions integration-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ readonly ES_ARTIFACT="${ES_ARTIFACT_PATH}.tar.gz"
readonly MIN_CURL_VERSION=(7 12 3)

ES_PID=-1
PROXY_CONTAINER_ID=-1
PROXY_SERVER_AVAILABLE=0

function check_prerequisites {
local curl_major_version=$(curl --version | head -1 | cut -d ' ' -f 2,2 | cut -d '.' -f 1,1)
Expand All @@ -57,6 +59,14 @@ function info {
log "INFO" "${1}"
}

function warn {
log "WARN" "${1}"
}

function error {
log "ERROR" "${1}"
}

function kill_rally_processes {
# kill all lingering Rally instances that might still be hanging
set +e
Expand Down Expand Up @@ -84,19 +94,10 @@ function kill_related_es_processes {
set -e
}

function set_up {
info "setting up"
kill_rally_processes
kill_related_es_processes

function set_up_metrics_store {
local in_memory_config_file_path="${HOME}/.rally/rally-integration-test.ini"
local es_config_file_path="${HOME}/.rally/rally-es-integration-test.ini"

# configure for tests with an in-memory metrics store
esrally configure --assume-defaults --configuration-name="integration-test"
# configure for tests with an Elasticsearch metrics store
esrally configure --assume-defaults --configuration-name="es-integration-test"

# configure Elasticsearch instead of in-memory after the fact
# this is more portable than using sed's in-place editing which requires "-i" on GNU and "-i ''" elsewhere.
perl -i -pe "s/datastore\.type.*/datastore.type = elasticsearch/g" ${es_config_file_path}
Expand Down Expand Up @@ -134,7 +135,46 @@ function set_up {
sleep 1
done ;
info "ES metrics store is up and running."
popd
popd > /dev/null
}

function set_up_proxy_server {
# we want to see output to stderr for diagnosing problems
if docker ps > /dev/null; then
info "Docker is available. Proxy-related tests will be run"
# Portably create a temporary config directory for Squid on Linux or MacOS
local config_dir=$(mktemp -d 2>/dev/null || mktemp -d -t 'tmp_squid_cfg')
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For Docker on MacOS this requires to (manually) add /var/folders to the list of allowed paths (otherwise Docker will complain that it can't bind-mount the directory). /var/folders is actually symlinked to /private/var/folders and /private is already configured (recursively) for Docker but not /var/folders is not. I think we should add this to our developer docs. Wdyt?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As discussed via another channel we'll instead use an already existing directory tree that we use for integration tests anyway. I'll push a new commit addressing this.


cat > ${config_dir}/squid.conf <<"EOF"
auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/squidpasswords
auth_param basic realm proxy
acl authenticated proxy_auth REQUIRED
http_access allow authenticated
http_port 3128
EOF

cat > ${config_dir}/squidpasswords <<"EOF"
testuser:$apr1$GcQaaItl$lhi4JoDsWBpZbkXVbI51O/
EOF
PROXY_CONTAINER_ID=$(docker run --rm --name squid -d -v ${config_dir}/squidpasswords:/etc/squid/squidpasswords -v ${config_dir}/squid.conf:/etc/squid/squid.conf -p 3128:3128 datadog/squid)
PROXY_SERVER_AVAILABLE=1
else
warn "Docker is not available. Skipping proxy-related tests."
fi
}

function set_up {
info "setting up"
kill_rally_processes
kill_related_es_processes

# configure for tests with an in-memory metrics store
esrally configure --assume-defaults --configuration-name="integration-test"
# configure for tests with an Elasticsearch metrics store
esrally configure --assume-defaults --configuration-name="es-integration-test"

set_up_metrics_store
set_up_proxy_server
}

function random_configuration {
Expand Down Expand Up @@ -211,7 +251,84 @@ function test_benchmark_only {
--track-params="cluster_health:'yellow'"
}

function test_proxy_connection {
readonly rally_log="${HOME}/.rally/logs/rally.log"
readonly rally_log_backup="${HOME}/.rally/logs/rally.log.it.bak"
# isolate invocations so we see only the log output from the current invocation
set +e
mv -f ${rally_log} "${rally_log_backup}"
set -e

set +e
esrally list tracks
unset http_proxy
set -e

if grep -F -q "Connecting directly to the Internet" "$rally_log"; then
info "Successfully checked that direct internet connection is used."
rm -f ${rally_log}
else
error "Could not find indication that direct internet connection is used. Please check ${rally_log}."
exit 1
fi

# test that we cannot connect to the Internet if the proxy authentication is missing
export http_proxy=http://127.0.0.1:3128
# this invocation *may* lead to an error but this is ok
set +e
esrally list tracks
unset http_proxy
set -e
if grep -F -q "Connecting via proxy URL [http://127.0.0.1:3128] to the Internet" "$rally_log"; then
info "Successfully checked that proxy is used."
else
error "Could not find indication that proxy access is used. Please check ${rally_log}."
exit 1
fi

if grep -F -q "No Internet connection detected" "$rally_log"; then
info "Successfully checked that unauthenticated proxy access is prevented."
rm -f ${rally_log}
else
error "Could not find indication that unauthenticated proxy access is prevented. Please check ${rally_log}."
exit 1
fi

# test that we can connect to the Internet if the proxy authentication is set

export http_proxy=http://testuser:[email protected]:3128
# this invocation *may* lead to an error but this is ok
set +e
esrally list tracks
unset http_proxy
set -e

if grep -F -q "Connecting via proxy URL [http://testuser:[email protected]:3128] to the Internet" "$rally_log"; then
info "Successfully checked that proxy is used."
else
error "Could not find indication that proxy access is used. Please check ${rally_log}."
exit 1
fi

if grep -F -q "Detected a working Internet connection" "$rally_log"; then
info "Successfully checked that authenticated proxy access is allowed."
rm -f ${rally_log}
else
error "Could not find indication that authenticated proxy access is allowed. Please check ${rally_log}."
exit 1
fi
# restore original file (but only on success so we keep the test's Rally log file for inspection on errors).
set +e
mv -f ${rally_log_backup} "${rally_log}"
set -e

}

function run_test {
if [ "${PROXY_SERVER_AVAILABLE}" == "1" ]; then
echo "**************************************** TESTING PROXY CONNECTIONS *********************************"
test_proxy_connection
fi
echo "**************************************** TESTING CONFIGURATION OF RALLY ****************************************"
test_configure
echo "**************************************** TESTING RALLY LIST COMMANDS *******************************************"
Expand All @@ -230,7 +347,13 @@ function tear_down {
set +e
# terminate metrics store
if [ "${ES_PID}" != "-1" ]; then
kill -9 ${ES_PID}
info "Stopping Elasticsearch metrics store with PID [${ES_PID}]"
kill -9 ${ES_PID} > /dev/null
fi
# stop Docker container for tests
if [ "${PROXY_CONTAINER_ID}" != "-1" ]; then
info "Stopping Docker container [${PROXY_CONTAINER_ID}]"
docker stop ${PROXY_CONTAINER_ID} > /dev/null
fi

rm -f ~/.rally/rally*integration-test.ini
Expand Down