Add --reverse-order option and switch order after each run on citrine (…

…TechEmpower#8860)

toolset/benchmark/benchmarker.py

@@ -29,7 +29,8 @@ def __init__(self, config):
 
         # a list of all tests for this run
         self.tests = self.metadata.tests_to_run()
-
+        if self.config.reverse_order:
+            self.tests.reverse()
         self.results = Results(self)
         self.docker_helper = DockerHelper(self)
 

toolset/continuous/tfb-shutdown.sh

@@ -30,4 +30,10 @@ ssh techempower@$TFB_DATABASE_HOST "$(typeset -f docker_clean); docker_clean"
 echo "running docker_clean on client host"
 ssh techempower@$TFB_CLIENT_HOST "$(typeset -f docker_clean); docker_clean"
 
+if [ -z "$TFB_RUN_ORDER" ]; then
+  export TFB_RUN_ORDER="reverse"
+else
+  unset TFB_RUN_ORDER
+fi
+
 echo "done with tfb-shutdown script"

toolset/continuous/tfb-startup.sh

@@ -38,6 +38,7 @@ docker run \
   --results-name "$TFB_RUN_NAME" \
   --results-environment "$TFB_ENVIRONMENT" \
   --results-upload-uri "$TFB_UPLOAD_URI" \
+  $(if [ "$TFB_RUN_ORDER" = "reverse" ]; then echo "--reverse-order"; fi) \
   --quiet
 
 echo "zipping the results"

toolset/run-tests.py

@@ -84,6 +84,13 @@ def main(argv=None):
         help=
         'Only print a limited set of messages to stdout, keep the bulk of messages in log files only'
     )
+    parser.add_argument(
+        '--reverse-order',
+        action='store_true',
+        default=False,
+        help=
+        'Run the tests in reverse order, starting with the last test in the list'
+    )
     parser.add_argument(
         '--results-name',
         help='Gives a name to this set of results, formatted as a date',

toolset/utils/benchmark_config.py

@@ -25,6 +25,7 @@ def __init__(self, args):
         self.duration = args.duration
         self.exclude = args.exclude
         self.quiet = args.quiet
+        self.reverse_order = args.reverse_order
         self.server_host = args.server_host
         self.database_host = args.database_host
         self.client_host = args.client_host