Migrate more integration tests (#966)

With this commit we migrate a few more integration tests from the shell script approach to the new approach based on `py.test`.

integration-test.sh

@@ -256,57 +256,6 @@ function test_configure {
     esrally configure --assume-defaults --configuration-name="config-integration-test"
 }
 
-function test_info {
-    local cfg
-    random_configuration cfg
-
-    info "test info [${cfg}]"
-    esrally info --configuration-name="${cfg}" --track=geonames --challenge=append-no-conflicts
-    info "test info can also use a track repository [${cfg}]"
-    esrally info --configuration-name="${cfg}" --track-repository=default --track=geonames
-    info "test info with task filter [${cfg}]"
-    esrally info --configuration-name="${cfg}" --track=geonames --challenge=append-no-conflicts --include-tasks="type:search"
-}
-
-function test_download {
-    local cfg
-    random_configuration cfg
-
-    for dist in "${DISTRIBUTIONS[@]}"
-    do
-        random_configuration cfg
-        info "test download [--configuration-name=${cfg}], [--distribution-version=${dist}]"
-        kill_rally_processes
-        esrally download --configuration-name="${cfg}" --distribution-version="${dist}" --quiet
-    done
-}
-
-function test_distributions {
-    local cfg
-
-    for dist in "${DISTRIBUTIONS[@]}"
-    do
-        for track in "${TRACKS[@]}"
-        do
-            random_configuration cfg
-            info "test distributions [--configuration-name=${cfg}], [--distribution-version=${dist}], [--track=${track}], [--car=4gheap]"
-            kill_rally_processes
-            wait_for_free_es_port
-            esrally --configuration-name="${cfg}" --on-error=abort --distribution-version="${dist}" --track="${track}" --test-mode --car=4gheap
-        done
-    done
-}
-
-function test_docker {
-    local cfg
-    # only test the most recent Docker distribution
-    local dist="${DISTRIBUTIONS[${#DISTRIBUTIONS[@]}-1]}"
-    random_configuration cfg
-    info "test docker [--configuration-name=${cfg}], [--distribution-version=${dist}], [--track=geonames], [--car=4gheap]"
-    kill_rally_processes
-    esrally --configuration-name="${cfg}" --on-error=abort --pipeline="docker" --distribution-version="${dist}" --track="geonames" --challenge="append-no-conflicts-index-only" --test-mode --car=4gheap --target-hosts=127.0.0.1:19200
-}
-
 function test_distribution_fails_with_wrong_track_params {
     local cfg
     local distribution
@@ -567,16 +516,8 @@ function run_test {
     fi
     echo "**************************************** TESTING CONFIGURATION OF RALLY ****************************************"
     test_configure
-    echo "**************************************** TESTING RALLY INFO COMMAND ********************************************"
-    test_info
     echo "**************************************** TESTING RALLY FAILS WITH UNUSED TRACK-PARAMS **************************"
     test_distribution_fails_with_wrong_track_params
-    echo "**************************************** TESTING RALLY DOWNLOAD COMMAND ***********************************"
-    test_download
-    echo "**************************************** TESTING RALLY WITH ES DISTRIBUTIONS ***********************************"
-    test_distributions
-    echo "**************************************** TESTING RALLY WITH ES DOCKER IMAGE ***********************************"
-    test_docker
     echo "**************************************** TESTING RALLY BENCHMARK-ONLY PIPELINE *********************************"
     test_benchmark_only
     echo "**************************************** TESTING RALLY DOCKER IMAGE ********************************************"

it/__init__.py

@@ -27,6 +27,8 @@
 from esrally.utils import process, io
 
 CONFIG_NAMES = ["in-memory-it", "es-it"]
+DISTRIBUTIONS = ["2.4.6", "5.6.16", "6.8.0", "7.1.1"]
+TRACKS = ["geonames", "nyc_taxis", "http_logs", "nested"]
 
 
 def all_rally_configs(t):
@@ -66,7 +68,7 @@ def wrapper(cfg, *args, **kwargs):
 
 
 def esrally(cfg, command_line):
-    return os.system("esrally {} --configuration-name=\"{}\"".format(command_line, cfg))
+    return os.system("esrally {} --kill-running-processes --configuration-name=\"{}\"".format(command_line, cfg))
 
 
 def wait_until_port_is_free(port_number=39200, timeout=120):

it/distribution_test.py

@@ -0,0 +1,37 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import it
+
+
+@it.random_rally_config
+def test_tar_distributions(cfg):
+    for dist in it.DISTRIBUTIONS:
+        for track in it.TRACKS:
+            it.wait_until_port_is_free()
+            assert it.esrally(cfg, f"--on-error=abort --distribution-version=\"{dist}\" --track=\"{track}\" "
+                                   f"--test-mode --car=4gheap") == 0
+
+
+@it.random_rally_config
+def test_docker_distribution(cfg):
+    # only test the most recent Docker distribution
+    dist = it.DISTRIBUTIONS[-1]
+    it.wait_until_port_is_free(port_number=19200)
+    assert it.esrally(cfg, f"--on-error=abort --pipeline=\"docker\" --distribution-version=\"{dist}\" "
+                           f"--track=\"geonames\" --challenge=\"append-no-conflicts-index-only\" --test-mode "
+                           f"--car=4gheap --target-hosts=127.0.0.1:19200") == 0

it/download_test.py

@@ -0,0 +1,24 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import it
+
+
+@it.random_rally_config
+def test_download_distribution(cfg):
+    for d in it.DISTRIBUTIONS:
+        assert it.esrally(cfg, f"download --distribution-version=\"{d}\" --quiet") == 0

it/info_test.py

@@ -0,0 +1,33 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import it
+
+
+@it.random_rally_config
+def test_track_info_with_challenge(cfg):
+    assert it.esrally(cfg, "info --track=geonames --challenge=append-no-conflicts") == 0
+
+
+@it.random_rally_config
+def test_track_info_with_track_repo(cfg):
+    assert it.esrally(cfg, "info --track-repository=default --track=geonames") == 0
+
+
+@it.random_rally_config
+def test_track_info_with_task_filter(cfg):
+    assert it.esrally(cfg, "info --track=geonames --challenge=append-no-conflicts --include-tasks=\"type:search\"") == 0