Island: Redirect gevent tracebacks to file and log exceptions

By default, gevent prints exceptions and tracebacks to stderr. This is
obnoxious as it results in large tracebacks intermixed with the output
that the logger prints to the console. This commit redirects this data
to {DATA_DIR}/gevent_exceptions.log. Unfortunately, this would mean that
the user might be left without any indication these exceptions had
occurred, unless they take the time to inspect the
gevent_exceptions.log. Therefore, when an excepion occurs, a message
with just the exception (not the traceback) is logged to WARNING.

Fixes #859

CHANGELOG.md

@@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Fixed
 - Attempted to delete a directory when monkey config reset was called. #1054
 - An errant space in the windows commands to run monkey manually. #1153
+- gevent tracebacks in console output. #859
 
 ### Security
 - Address minor issues discovered by Dlint. #1075

monkey/monkey_island/cc/server_setup.py

@@ -1,10 +1,12 @@
+import atexit
 import json
 import logging
 import sys
 from pathlib import Path
 from threading import Thread
 from typing import Tuple
 
+import gevent.hub
 from gevent.pywsgi import WSGIServer
 
 # Add the monkey_island directory to the path, to make sure imports that don't start with
@@ -21,12 +23,14 @@
 from monkey_island.cc.arg_parser import parse_cli_args  # noqa: E402
 from monkey_island.cc.resources.monkey_download import MonkeyDownload  # noqa: E402
 from monkey_island.cc.server_utils.bootloader_server import BootloaderHttpServer  # noqa: E402
+from monkey_island.cc.server_utils.consts import GEVENT_EXCEPTION_LOG  # noqa: E402
 from monkey_island.cc.server_utils.encryptor import initialize_encryptor  # noqa: E402
 from monkey_island.cc.server_utils.island_logger import reset_logger, setup_logging  # noqa: E402
 from monkey_island.cc.services.initialize import initialize_services  # noqa: E402
 from monkey_island.cc.services.reporting.exporter_init import populate_exporter_list  # noqa: E402
 from monkey_island.cc.services.utils.network_utils import local_ip_addresses  # noqa: E402
 from monkey_island.cc.setup import island_config_options_validator  # noqa: E402
+from monkey_island.cc.setup.gevent_hub_error_handler import GeventHubErrorHandler  # noqa: E402
 from monkey_island.cc.setup.island_config_options import IslandConfigOptions  # noqa: E402
 from monkey_island.cc.setup.mongo.database_initializer import init_collections  # noqa: E402
 from monkey_island.cc.setup.mongo.mongo_setup import (  # noqa: E402
@@ -54,6 +58,7 @@ def run_monkey_island():
 
     connect_to_mongodb()
 
+    _configure_gevent_exception_handling(logger, Path(config_options.data_dir))
     _start_island_server(island_args.setup_only, config_options)
 
 
@@ -88,6 +93,18 @@ def _initialize_globals(config_options: IslandConfigOptions, server_config_path:
     initialize_services(config_options.data_dir)
 
 
+def _configure_gevent_exception_handling(logger, data_dir):
+    hub = gevent.hub.get_hub()
+
+    gevent_exception_log = open(data_dir / GEVENT_EXCEPTION_LOG, "a+")
+    atexit.register(gevent_exception_log.close)
+
+    # Send gevent's exception output to a log file.
+    # https://www.gevent.org/api/gevent.hub.html#gevent.hub.Hub.exception_stream
+    hub.exception_stream = gevent_exception_log
+    hub.handle_error = GeventHubErrorHandler(hub, logger)
+
+
 def _start_island_server(should_setup_only, config_options: IslandConfigOptions):
     populate_exporter_list()
     app = init_app(MONGO_URL)

monkey/monkey_island/cc/server_utils/consts.py

@@ -52,3 +52,5 @@ def _get_monkey_island_abs_path() -> str:
     "ssl_certificate_file": DEFAULT_CRT_PATH,
     "ssl_certificate_key_file": DEFAULT_KEY_PATH,
 }
+
+GEVENT_EXCEPTION_LOG = "gevent_exceptions.log"

vulture_allowlist.py

@@ -170,6 +170,7 @@
 MONKEY_LINUX_RUNNING  # unused variable (monkey/monkey_island/cc/services/utils/node_states.py:26)
 import_status  # monkey_island\cc\resources\configuration_import.py:19
 config_schema  # monkey_island\cc\resources\configuration_import.py:25
+exception_stream  # unused attribute (monkey_island/cc/server_setup.py:104)
 
 # these are not needed for it to work, but may be useful extra information to understand what's going on
 WINDOWS_PBA_TYPE  # unused variable (monkey/monkey_island/cc/resources/pba_file_upload.py:23)