Fix reload gunicorn workers (#32102)

* Toggle gunicorn --preload with reload_on_plugin_change Since gunicorn can't reload a new code if starts with ``--preload`` setting, we need to check ``reload_on_plugin_change`` before set it up. Gunicorn can't reload a new code because the code is preloaded into the master process and worker are launched with ``fork``, they will still have the old code. * added warning message * Improve warning message Co-authored-by: Jed Cunningham <[email protected]> * Improve warning message Co-authored-by: Jed Cunningham <[email protected]> * Improve warning message Co-authored-by: Jed Cunningham <[email protected]> * Minor tweak of warning message * Mention prod issue in config description --------- Co-authored-by: Jed Cunningham <[email protected]> Co-authored-by: Tzu-ping Chung <[email protected]> (cherry picked from commit f78a836)
apache · Aug 8, 2023 · c11e814 · c11e814
1 parent bc1d7b0
commit c11e814
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 6 deletions.
diff --git a/airflow/cli/commands/webserver_command.py b/airflow/cli/commands/webserver_command.py
@@ -422,11 +422,17 @@ def webserver(args):
 
         run_args += ["airflow.www.app:cached_app()"]
 
-        # To prevent different workers creating the web app and
-        # all writing to the database at the same time, we use the --preload option.
-        # With the preload option, the app is loaded before the workers are forked, and each worker will
-        # then have a copy of the app
-        run_args += ["--preload"]
+        if conf.getboolean("webserver", "reload_on_plugin_change", fallback=False):
+            log.warning(
+                "Setting reload_on_plugin_change = true prevents running Gunicorn with preloading. "
+                "This means the app cannot be loaded before workers are forked, and each worker has a "
+                "separate copy of the app. This may cause IntegrityError during webserver startup, and "
+                "should be avoided in production."
+            )
+        else:
+            # To prevent different workers creating the web app and
+            # all writing to the database at the same time, we use the --preload option.
+            run_args += ["--preload"]
 
         gunicorn_master_proc: psutil.Process | subprocess.Popen
 

diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
@@ -1452,7 +1452,8 @@ webserver:
     reload_on_plugin_change:
       description: |
         If set to True, Airflow will track files in plugins_folder directory. When it detects changes,
-        then reload the gunicorn.
+        then reload the gunicorn. If set to True, gunicorn starts without preloading, which is slower, uses
+        more memory, and may cause race conditions. Avoid setting this to True in production.
       version_added: 1.10.11
       type: boolean
       example: ~