diff --git a/UPDATING.md b/UPDATING.md index 89f33543395ba..29c62bca2156c 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -25,7 +25,6 @@ assists users migrating to a new version. ## Airflow 1.10.3 ### RedisPy dependency updated to v3 series - If you are using the Redis Sensor or Hook you may have to update your code. See [redis-py porting instructions] to check if your code might be affected (MSET, MSETNX, ZADD, and ZINCRBY all were, but read the full doc). @@ -50,6 +49,14 @@ so you might need to update your config. `task_runner = StandardTaskRunner` +### Modification to config file discovery + +If the `AIRFLOW_CONFIG` environment variable was not set and the +`~/airflow/airflow.cfg` file existed, airflow previously used +`~/airflow/airflow.cfg` instead of `$AIRFLOW_HOME/airflow.cfg`. Now airflow +will discover its config file using the `$AIRFLOW_CONFIG` and `$AIRFLOW_HOME` +environment variables rather than checking for the presence of a file. + ## Airflow 1.10.2 ### Modification to `ts_nodash` macro diff --git a/airflow/configuration.py b/airflow/configuration.py index 751dce1c6da3d..76dceaf58fd81 100644 --- a/airflow/configuration.py +++ b/airflow/configuration.py @@ -441,23 +441,23 @@ def mkdir_p(path): 'Error creating {}: {}'.format(path, exc.strerror)) -# Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using -# "~/airflow" and "~/airflow/airflow.cfg" respectively as defaults. +def get_airflow_home(): + return expand_env_var(os.environ.get('AIRFLOW_HOME', '~/airflow')) + + +def get_airflow_config(airflow_home): + if 'AIRFLOW_CONFIG' not in os.environ: + return os.path.join(airflow_home, 'airflow.cfg') + return expand_env_var(os.environ['AIRFLOW_CONFIG']) -if 'AIRFLOW_HOME' not in os.environ: - AIRFLOW_HOME = expand_env_var('~/airflow') -else: - AIRFLOW_HOME = expand_env_var(os.environ['AIRFLOW_HOME']) +# Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using +# "~/airflow" and "$AIRFLOW_HOME/airflow.cfg" respectively as defaults. + +AIRFLOW_HOME = get_airflow_home() +AIRFLOW_CONFIG = get_airflow_config(AIRFLOW_HOME) mkdir_p(AIRFLOW_HOME) -if 'AIRFLOW_CONFIG' not in os.environ: - if os.path.isfile(expand_env_var('~/airflow.cfg')): - AIRFLOW_CONFIG = expand_env_var('~/airflow.cfg') - else: - AIRFLOW_CONFIG = AIRFLOW_HOME + '/airflow.cfg' -else: - AIRFLOW_CONFIG = expand_env_var(os.environ['AIRFLOW_CONFIG']) # Set up dags folder for unit tests # this directory won't exist if users install via pip diff --git a/tests/configuration.py b/tests/configuration.py index 278e2e597abbc..e3b81f769f7b9 100644 --- a/tests/configuration.py +++ b/tests/configuration.py @@ -21,6 +21,7 @@ from __future__ import unicode_literals import os +import contextlib from collections import OrderedDict import six @@ -35,6 +36,23 @@ import unittest +@contextlib.contextmanager +def env_vars(**vars): + original = {} + for key, value in vars.items(): + original[key] = os.environ.get(key) + if value is not None: + os.environ[key] = value + else: + os.environ.pop(key, None) + yield + for key, value in original.items(): + if value is not None: + os.environ[key] = value + else: + os.environ.pop(key, None) + + class ConfTest(unittest.TestCase): @classmethod @@ -49,6 +67,30 @@ def tearDownClass(cls): del os.environ['AIRFLOW__TESTSECTION__TESTKEY'] del os.environ['AIRFLOW__TESTSECTION__TESTPERCENT'] + def test_airflow_home_default(self): + with env_vars(AIRFLOW_HOME=None): + self.assertEqual( + configuration.get_airflow_home(), + configuration.expand_env_var('~/airflow')) + + def test_airflow_home_override(self): + with env_vars(AIRFLOW_HOME='/path/to/airflow'): + self.assertEqual( + configuration.get_airflow_home(), + '/path/to/airflow') + + def test_airflow_config_default(self): + with env_vars(AIRFLOW_CONFIG=None): + self.assertEqual( + configuration.get_airflow_config('/home/airflow'), + configuration.expand_env_var('/home/airflow/airflow.cfg')) + + def test_airflow_config_override(self): + with env_vars(AIRFLOW_CONFIG='/path/to/airflow/airflow.cfg'): + self.assertEqual( + configuration.get_airflow_config('/home//airflow'), + '/path/to/airflow/airflow.cfg') + def test_env_var_config(self): opt = conf.get('testsection', 'testkey') self.assertEqual(opt, 'testvalue')