From bf6999b58688d8cc40e91eb6a58d68a794051424 Mon Sep 17 00:00:00 2001 From: Jonatan Heyman Date: Tue, 26 May 2020 00:23:11 +0200 Subject: [PATCH 1/4] Add ability to control the Locust process' exit code from within locust test scripts --- docs/changelog.rst | 6 ++++++ docs/running-locust-without-web-ui.rst | 24 +++++++++++++++++++++ locust/env.py | 5 +++++ locust/event.py | 4 ++++ locust/main.py | 29 ++++++++++++++++---------- locust/runners.py | 4 ++-- locust/test/test_main.py | 23 ++++++++++++++++++++ 7 files changed, 82 insertions(+), 13 deletions(-) diff --git a/docs/changelog.rst b/docs/changelog.rst index 2024be4a44..d80ef06f02 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,12 @@ Changelog Highlights For full details of the Locust changelog, please see https://github.com/locustio/locust/blob/master/CHANGELOG.md +In development +============== + +* Ability to control the exit code of the Locust process by setting :py:attr:`Environment.process_exit_code ` + + 1.0.2 ===== diff --git a/docs/running-locust-without-web-ui.rst b/docs/running-locust-without-web-ui.rst index 25d461a0aa..f6a5460496 100644 --- a/docs/running-locust-without-web-ui.rst +++ b/docs/running-locust-without-web-ui.rst @@ -26,6 +26,7 @@ If you want to specify the run time for a test, you can do that with ``--run-tim Locust will shutdown once the time is up. + Allow tasks to finish their iteration on shutdown ------------------------------------------------- @@ -37,6 +38,7 @@ By default, locust will stop your tasks immediately. If you want to allow your t .. _running-locust-distributed-without-web-ui: + Running Locust distributed without the web UI --------------------------------------------- @@ -45,3 +47,25 @@ you should specify the ``--expect-workers`` option when starting the master node the number of worker nodes that are expected to connect. It will then wait until that many worker nodes have connected before starting the test. + +Controlling the exit code of the Locust process +----------------------------------------------- + +You can control the exit code of the Locust process by setting the +:py:attr:`process_exit_code ` of the +:py:class:`Environment ` instance. + +Here's an example that'll set the exit code to non zero if more than 1% of the requests failed +(this code could go into the locustfile.py or in any other file that is imported in the locustfile): + +.. code-block:: python + + from locust import events + + @events.quitting.add_listener + def _(environment, **kw): + if environment.stats.total.fail_ratio > 0.01: + environment.process_exit_code = 1 + else: + environment.process_exit_code = 0 + diff --git a/locust/env.py b/locust/env.py index deab381ac5..be81b88b6e 100644 --- a/locust/env.py +++ b/locust/env.py @@ -51,6 +51,11 @@ class Environment: If True exceptions that happen within running users will be catched (and reported in UI/console). If False, exeptions will be raised. """ + + process_exit_code: int = None + """ + If set it'll be the exit code of the Locust process + """ parsed_options = None """Optional reference to the parsed command line options (used to pre-populate fields in Web UI)""" diff --git a/locust/event.py b/locust/event.py index d05c4dfd1e..9d527df1f5 100644 --- a/locust/event.py +++ b/locust/event.py @@ -111,6 +111,10 @@ class Events: quitting = EventHook """ Fired when the locust process is exiting + + Event arguments: + + :param environment: Environment instance """ init = EventHook diff --git a/locust/main.py b/locust/main.py index bc28c8ad2c..aa37d737ac 100644 --- a/locust/main.py +++ b/locust/main.py @@ -297,18 +297,30 @@ def timelimit_stop(): gevent.spawn(stats_writer, environment, options.csv_prefix, full_history=options.stats_history_enabled).link_exception(greenlet_exception_handler) - def shutdown(code=0): + def shutdown(): """ Shut down locust by firing quitting event, printing/writing stats and exiting """ + logger.info("Running teardowns...") + environment.events.quitting.fire(environment=environment, reverse=True) + + # determine the process exit code + if log.unhandled_greenlet_exception: + code = 2 + elif environment.process_exit_code is not None: + code = environment.process_exit_code + elif len(runner.errors) or len(runner.exceptions): + code = options.exit_code_on_error + else: + code = 0 + logger.info("Shutting down (exit code %s), bye." % code) if stats_printer_greenlet is not None: stats_printer_greenlet.kill(block=False) logger.info("Cleaning up runner...") if runner is not None: runner.quit() - logger.info("Running teardowns...") - environment.events.quitting.fire(reverse=True) + print_stats(runner.stats, current=False) print_percentile_stats(runner.stats) if options.csv_prefix: @@ -319,17 +331,12 @@ def shutdown(code=0): # install SIGTERM handler def sig_term_handler(): logger.info("Got SIGTERM signal") - shutdown(0) + shutdown() gevent.signal_handler(signal.SIGTERM, sig_term_handler) try: logger.info("Starting Locust %s" % version) main_greenlet.join() - code = 0 - if log.unhandled_greenlet_exception: - code = 2 - elif len(runner.errors) or len(runner.exceptions): - code = options.exit_code_on_error - shutdown(code=code) + shutdown() except KeyboardInterrupt as e: - shutdown(0) + shutdown() diff --git a/locust/runners.py b/locust/runners.py index 617bac43b4..4424fd34ad 100644 --- a/locust/runners.py +++ b/locust/runners.py @@ -413,7 +413,7 @@ def on_worker_report(client_id, data): self.environment.events.worker_report.add_listener(on_worker_report) # register listener that sends quit message to worker nodes - def on_quitting(): + def on_quitting(environment, **kw): self.quit() self.environment.events.quitting.add_listener(on_quitting) @@ -615,7 +615,7 @@ def on_report_to_master(client_id, data): self.environment.events.report_to_master.add_listener(on_report_to_master) # register listener that sends quit message to master - def on_quitting(): + def on_quitting(environment, **kw): self.client.send(Message("quit", None, self.client_id)) self.environment.events.quitting.add_listener(on_quitting) diff --git a/locust/test/test_main.py b/locust/test/test_main.py index 13a0ebb3e0..4f03911e5c 100644 --- a/locust/test/test_main.py +++ b/locust/test/test_main.py @@ -133,6 +133,29 @@ def test_help_arg(self): self.assertIn("-f LOCUSTFILE, --locustfile LOCUSTFILE", output) self.assertIn("Logging options:", output) self.assertIn("--skip-log-setup Disable Locust's logging setup.", output) + + def test_custom_exit_code(self): + with temporary_file(content=textwrap.dedent(""" + from locust import User, task, constant, events + @events.quitting.add_listener + def _(environment, **kw): + print("ok") + environment.process_exit_code = 42 + class TestUser(User): + wait_time = constant(3) + @task + def my_task(): + print("running my_task()") + """)) as file_path: + proc = subprocess.Popen(["locust", "-f", file_path], stdout=PIPE, stderr=PIPE) + gevent.sleep(1) + proc.send_signal(signal.SIGTERM) + stdout, stderr = proc.communicate() + self.assertEqual(42, proc.returncode) + stderr = stderr.decode("utf-8") + self.assertIn("Starting web interface at", stderr) + self.assertIn("Starting Locust", stderr) + self.assertIn("Shutting down (exit code 42), bye", stderr) def test_webserver(self): with temporary_file(content=textwrap.dedent(""" From c77dee16a82004552a63cdf0e172dbf5f8a39895 Mon Sep 17 00:00:00 2001 From: Jonatan Heyman Date: Tue, 26 May 2020 00:27:04 +0200 Subject: [PATCH 2/4] Remove debug print --- locust/test/test_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/locust/test/test_main.py b/locust/test/test_main.py index 4f03911e5c..78e3556117 100644 --- a/locust/test/test_main.py +++ b/locust/test/test_main.py @@ -139,7 +139,6 @@ def test_custom_exit_code(self): from locust import User, task, constant, events @events.quitting.add_listener def _(environment, **kw): - print("ok") environment.process_exit_code = 42 class TestUser(User): wait_time = constant(3) From 502ba0b09cc94cdee31755012dc48b111078de2c Mon Sep 17 00:00:00 2001 From: Jonatan Heyman Date: Tue, 26 May 2020 09:25:53 +0200 Subject: [PATCH 3/4] Extend code example for setting process exit code with checks for average and and percentile response times --- docs/running-locust-without-web-ui.rst | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/running-locust-without-web-ui.rst b/docs/running-locust-without-web-ui.rst index f6a5460496..9d900b0df8 100644 --- a/docs/running-locust-without-web-ui.rst +++ b/docs/running-locust-without-web-ui.rst @@ -51,12 +51,16 @@ nodes have connected before starting the test. Controlling the exit code of the Locust process ----------------------------------------------- -You can control the exit code of the Locust process by setting the +When running Locust in a CI environment, you might want to control the exit code of the Locust +process. You can do that in your test scripts by setting the :py:attr:`process_exit_code ` of the :py:class:`Environment ` instance. -Here's an example that'll set the exit code to non zero if more than 1% of the requests failed -(this code could go into the locustfile.py or in any other file that is imported in the locustfile): +Below is an example that'll set the exit code to non zero if any of the following conditions are met: + +* More than 1% of the requests failed +* The average response time is longer than 200 ms +* The 95th percentile for response time is larger than 800 ms .. code-block:: python @@ -66,6 +70,11 @@ Here's an example that'll set the exit code to non zero if more than 1% of the r def _(environment, **kw): if environment.stats.total.fail_ratio > 0.01: environment.process_exit_code = 1 + elif environment.stats.total.avg_response_time > 200: + environment.process_exit_code = 1 + elif environment.stats.total.get_response_time_percentile(0.95) > 800: + environment.process_exit_code = 1 else: environment.process_exit_code = 0 +(this code could go into the locustfile.py or in any other file that is imported in the locustfile) From 48fd3f43d533bee2d241797a80f83f4071d2ea79 Mon Sep 17 00:00:00 2001 From: Jonatan Heyman Date: Tue, 26 May 2020 09:35:25 +0200 Subject: [PATCH 4/4] Add logging messages to custom exit code example --- docs/running-locust-without-web-ui.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/running-locust-without-web-ui.rst b/docs/running-locust-without-web-ui.rst index 9d900b0df8..79dbdd4cb4 100644 --- a/docs/running-locust-without-web-ui.rst +++ b/docs/running-locust-without-web-ui.rst @@ -64,15 +64,19 @@ Below is an example that'll set the exit code to non zero if any of the followin .. code-block:: python + import logging from locust import events @events.quitting.add_listener def _(environment, **kw): if environment.stats.total.fail_ratio > 0.01: + logging.error("Test failed due to failure ratio > 1%") environment.process_exit_code = 1 elif environment.stats.total.avg_response_time > 200: + logging.error("Test failed due to average response time ratio > 200 ms") environment.process_exit_code = 1 elif environment.stats.total.get_response_time_percentile(0.95) > 800: + logging.error("Test failed due to 95th percentil response time > 800 ms") environment.process_exit_code = 1 else: environment.process_exit_code = 0