[AIRFLOW-3540] Respect environment config when looking up config file. (

apache#4340)

UPDATING.md

@@ -24,6 +24,14 @@ assists users migrating to a new version.
 
 ## Airflow Master
 
+### Modification to config file discovery
+
+If the `AIRFLOW_CONFIG` environment variable was not set and the
+`~/airflow/airflow.cfg` file existed, airflow previously used
+`~/airflow/airflow.cfg` instead of `$AIRFLOW_HOME/airflow.cfg`. Now airflow
+will discover its config file using the `$AIRFLOW_CONFIG` and `$AIRFLOW_HOME`
+environment variables rather than checking for the presence of a file.
+
 ### Modification to `ts_nodash` macro
 `ts_nodash` previously contained TimeZone information alongwith execution date. For Example: `20150101T000000+0000`. This is not user-friendly for file or folder names which was a popular use case for `ts_nodash`. Hence this behavior has been changed and using `ts_nodash` will no longer contain TimeZone information, restoring the pre-1.10 behavior of this macro. And a new macro `ts_nodash_with_tz` has been added which can be used to get a string with execution date and timezone info without dashes. 
 

airflow/configuration.py

@@ -441,23 +441,23 @@ def mkdir_p(path):
                 'Error creating {}: {}'.format(path, exc.strerror))
 
 
-# Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using
-# "~/airflow" and "~/airflow/airflow.cfg" respectively as defaults.
+def get_airflow_home():
+    return expand_env_var(os.environ.get('AIRFLOW_HOME', '~/airflow'))
+
+
+def get_airflow_config(airflow_home):
+    if 'AIRFLOW_CONFIG' not in os.environ:
+        return os.path.join(airflow_home, 'airflow.cfg')
+    return expand_env_var(os.environ['AIRFLOW_CONFIG'])
 
-if 'AIRFLOW_HOME' not in os.environ:
-    AIRFLOW_HOME = expand_env_var('~/airflow')
-else:
-    AIRFLOW_HOME = expand_env_var(os.environ['AIRFLOW_HOME'])
 
+# Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using
+# "~/airflow" and "$AIRFLOW_HOME/airflow.cfg" respectively as defaults.
+
+AIRFLOW_HOME = get_airflow_home()
+AIRFLOW_CONFIG = get_airflow_config(AIRFLOW_HOME)
 mkdir_p(AIRFLOW_HOME)
 
-if 'AIRFLOW_CONFIG' not in os.environ:
-    if os.path.isfile(expand_env_var('~/airflow.cfg')):
-        AIRFLOW_CONFIG = expand_env_var('~/airflow.cfg')
-    else:
-        AIRFLOW_CONFIG = AIRFLOW_HOME + '/airflow.cfg'
-else:
-    AIRFLOW_CONFIG = expand_env_var(os.environ['AIRFLOW_CONFIG'])
 
 # Set up dags folder for unit tests
 # this directory won't exist if users install via pip

tests/test_configuration.py

@@ -21,6 +21,7 @@
 from __future__ import unicode_literals
 
 import os
+import contextlib
 from collections import OrderedDict
 
 import six
@@ -35,6 +36,23 @@
     import unittest
 
 
+@contextlib.contextmanager
+def env_vars(**vars):
+    original = {}
+    for key, value in vars.items():
+        original[key] = os.environ.get(key)
+        if value is not None:
+            os.environ[key] = value
+        else:
+            os.environ.pop(key, None)
+    yield
+    for key, value in original.items():
+        if value is not None:
+            os.environ[key] = value
+        else:
+            os.environ.pop(key, None)
+
+
 class ConfTest(unittest.TestCase):
 
     @classmethod
@@ -49,6 +67,30 @@ def tearDownClass(cls):
         del os.environ['AIRFLOW__TESTSECTION__TESTKEY']
         del os.environ['AIRFLOW__TESTSECTION__TESTPERCENT']
 
+    def test_airflow_home_default(self):
+        with env_vars(AIRFLOW_HOME=None):
+            self.assertEqual(
+                configuration.get_airflow_home(),
+                configuration.expand_env_var('~/airflow'))
+
+    def test_airflow_home_override(self):
+        with env_vars(AIRFLOW_HOME='/path/to/airflow'):
+            self.assertEqual(
+                configuration.get_airflow_home(),
+                '/path/to/airflow')
+
+    def test_airflow_config_default(self):
+        with env_vars(AIRFLOW_CONFIG=None):
+            self.assertEqual(
+                configuration.get_airflow_config('/home/airflow'),
+                configuration.expand_env_var('/home/airflow/airflow.cfg'))
+
+    def test_airflow_config_override(self):
+        with env_vars(AIRFLOW_CONFIG='/path/to/airflow/airflow.cfg'):
+            self.assertEqual(
+                configuration.get_airflow_config('/home//airflow'),
+                '/path/to/airflow/airflow.cfg')
+
     def test_env_var_config(self):
         opt = conf.get('testsection', 'testkey')
         self.assertEqual(opt, 'testvalue')