From 04f94429983b97828e58f3887ac091b2c1a29b84 Mon Sep 17 00:00:00 2001 From: Joshua Carp Date: Tue, 9 Oct 2018 11:14:07 -0400 Subject: [PATCH] [AIRFLOW-3086] Add extras group for google auth to setup.py. (#3917) To clarify installation instructions for the google auth backend, add an install group to `setup.py` that installs dependencies google auth via `pip install apache-airflow[google_auth]`. --- docs/installation.rst | 134 ++++++++++++++++++++++-------------------- docs/security.rst | 18 ++++++ setup.py | 2 + 3 files changed, 89 insertions(+), 65 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 3db48e45ddf40..bb1365ff0071e 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -41,71 +41,75 @@ these extra dependencies. Here's the list of the subpackages and what they enable: -+---------------+----------------------------------------------+-------------------------------------------------+ -| subpackage | install command | enables | -+===============+==============================================+=================================================+ -| all | ``pip install apache-airflow[all]`` | All Airflow features known to man | -+---------------+----------------------------------------------+-------------------------------------------------+ -| all_dbs | ``pip install apache-airflow[all_dbs]`` | All databases integrations | -+---------------+----------------------------------------------+-------------------------------------------------+ -| async | ``pip install apache-airflow[async]`` | Async worker classes for Gunicorn | -+---------------+----------------------------------------------+-------------------------------------------------+ -| celery | ``pip install apache-airflow[celery]`` | CeleryExecutor | -+---------------+----------------------------------------------+-------------------------------------------------+ -| cloudant | ``pip install apache-airflow[cloudant]`` | Cloudant hook | -+---------------+----------------------------------------------+-------------------------------------------------+ -| crypto | ``pip install apache-airflow[crypto]`` | Encrypt connection passwords in metadata db | -+---------------+----------------------------------------------+-------------------------------------------------+ -| devel | ``pip install apache-airflow[devel]`` | Minimum dev tools requirements | -+---------------+----------------------------------------------+-------------------------------------------------+ -| devel_hadoop | ``pip install apache-airflow[devel_hadoop]`` | Airflow + dependencies on the Hadoop stack | -+---------------+----------------------------------------------+-------------------------------------------------+ -| druid | ``pip install apache-airflow[druid]`` | Druid related operators & hooks | -+---------------+----------------------------------------------+-------------------------------------------------+ -| gcp_api | ``pip install apache-airflow[gcp_api]`` | Google Cloud Platform hooks and operators | -| | | (using ``google-api-python-client``) | -+---------------+----------------------------------------------+-------------------------------------------------+ -| hdfs | ``pip install apache-airflow[hdfs]`` | HDFS hooks and operators | -+---------------+----------------------------------------------+-------------------------------------------------+ -| hive | ``pip install apache-airflow[hive]`` | All Hive related operators | -+---------------+----------------------------------------------+-------------------------------------------------+ -| jdbc | ``pip install apache-airflow[jdbc]`` | JDBC hooks and operators | -+---------------+----------------------------------------------+-------------------------------------------------+ -| kerberos | ``pip install apache-airflow[kerberos]`` | Kerberos integration for Kerberized Hadoop | -+---------------+----------------------------------------------+-------------------------------------------------+ -| ldap | ``pip install apache-airflow[ldap]`` | LDAP authentication for users | -+---------------+----------------------------------------------+-------------------------------------------------+ -| mssql | ``pip install apache-airflow[mssql]`` | Microsoft SQL Server operators and hook, | -| | | support as an Airflow backend | -+---------------+----------------------------------------------+-------------------------------------------------+ -| mysql | ``pip install apache-airflow[mysql]`` | MySQL operators and hook, support as an Airflow | -| | | backend. The version of MySQL server has to be | -| | | 5.6.4+. The exact version upper bound depends | -| | | on version of ``mysqlclient`` package. For | -| | | example, ``mysqlclient`` 1.3.12 can only be | -| | | used with MySQL server 5.6.4 through 5.7. | -+---------------+----------------------------------------------+-------------------------------------------------+ -| password | ``pip install apache-airflow[password]`` | Password authentication for users | -+---------------+----------------------------------------------+-------------------------------------------------+ -| postgres | ``pip install apache-airflow[postgres]`` | PostgreSQL operators and hook, support as an | -| | | Airflow backend | -+---------------+----------------------------------------------+-------------------------------------------------+ -| qds | ``pip install apache-airflow[qds]`` | Enable QDS (Qubole Data Service) support | -+---------------+----------------------------------------------+-------------------------------------------------+ -| rabbitmq | ``pip install apache-airflow[rabbitmq]`` | RabbitMQ support as a Celery backend | -+---------------+----------------------------------------------+-------------------------------------------------+ -| redis | ``pip install apache-airflow[redis]`` | Redis hooks and sensors | -+---------------+----------------------------------------------+-------------------------------------------------+ -| s3 | ``pip install apache-airflow[s3]`` | ``S3KeySensor``, ``S3PrefixSensor`` | -+---------------+----------------------------------------------+-------------------------------------------------+ -| samba | ``pip install apache-airflow[samba]`` | ``Hive2SambaOperator`` | -+---------------+----------------------------------------------+-------------------------------------------------+ -| slack | ``pip install apache-airflow[slack]`` | ``SlackAPIPostOperator`` | -+---------------+----------------------------------------------+-------------------------------------------------+ -| ssh | ``pip install apache-airflow[ssh]`` | SSH hooks and Operator | -+---------------+----------------------------------------------+-------------------------------------------------+ -| vertica | ``pip install apache-airflow[vertica]`` | Vertica hook support as an Airflow backend | -+---------------+----------------------------------------------+-------------------------------------------------+ ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| subpackage | install command | enables | ++=====================+===================================================+=================================================+ +| all | ``pip install apache-airflow[all]`` | All Airflow features known to man | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| all_dbs | ``pip install apache-airflow[all_dbs]`` | All databases integrations | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| async | ``pip install apache-airflow[async]`` | Async worker classes for Gunicorn | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| celery | ``pip install apache-airflow[celery]`` | CeleryExecutor | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| cloudant | ``pip install apache-airflow[cloudant]`` | Cloudant hook | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| crypto | ``pip install apache-airflow[crypto]`` | Encrypt connection passwords in metadata db | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| devel | ``pip install apache-airflow[devel]`` | Minimum dev tools requirements | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| devel_hadoop | ``pip install apache-airflow[devel_hadoop]`` | Airflow + dependencies on the Hadoop stack | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| druid | ``pip install apache-airflow[druid]`` | Druid related operators & hooks | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| gcp_api | ``pip install apache-airflow[gcp_api]`` | Google Cloud Platform hooks and operators | +| | | (using ``google-api-python-client``) | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| github_enterprise | ``pip install apache-airflow[github_enterprise]`` | Github Enterprise auth backend | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| google_auth | ``pip install apache-airflow[google_auth]`` | Google auth backend | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| hdfs | ``pip install apache-airflow[hdfs]`` | HDFS hooks and operators | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| hive | ``pip install apache-airflow[hive]`` | All Hive related operators | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| jdbc | ``pip install apache-airflow[jdbc]`` | JDBC hooks and operators | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| kerberos | ``pip install apache-airflow[kerberos]`` | Kerberos integration for Kerberized Hadoop | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| ldap | ``pip install apache-airflow[ldap]`` | LDAP authentication for users | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| mssql | ``pip install apache-airflow[mssql]`` | Microsoft SQL Server operators and hook, | +| | | support as an Airflow backend | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| mysql | ``pip install apache-airflow[mysql]`` | MySQL operators and hook, support as an Airflow | +| | | backend. The version of MySQL server has to be | +| | | 5.6.4+. The exact version upper bound depends | +| | | on version of ``mysqlclient`` package. For | +| | | example, ``mysqlclient`` 1.3.12 can only be | +| | | used with MySQL server 5.6.4 through 5.7. | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| password | ``pip install apache-airflow[password]`` | Password authentication for users | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| postgres | ``pip install apache-airflow[postgres]`` | PostgreSQL operators and hook, support as an | +| | | Airflow backend | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| qds | ``pip install apache-airflow[qds]`` | Enable QDS (Qubole Data Service) support | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| rabbitmq | ``pip install apache-airflow[rabbitmq]`` | RabbitMQ support as a Celery backend | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| redis | ``pip install apache-airflow[redis]`` | Redis hooks and sensors | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| s3 | ``pip install apache-airflow[s3]`` | ``S3KeySensor``, ``S3PrefixSensor`` | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| samba | ``pip install apache-airflow[samba]`` | ``Hive2SambaOperator`` | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| slack | ``pip install apache-airflow[slack]`` | ``SlackAPIPostOperator`` | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| ssh | ``pip install apache-airflow[ssh]`` | SSH hooks and Operator | ++---------------------+---------------------------------------------------+-------------------------------------------------+ +| vertica | ``pip install apache-airflow[vertica]`` | Vertica hook support as an Airflow backend | ++---------------------+---------------------------------------------------+-------------------------------------------------+ Initiating Airflow Database ''''''''''''''''''''''''''' diff --git a/docs/security.rst b/docs/security.rst index 9f6740d041ce0..76a11418a2e25 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -237,6 +237,12 @@ and in your DAG, when initializing the HiveOperator, specify: run_as_owner=True +To use kerberos authentication, you must install Airflow with the `kerberos` extras group: + +.. code-block:: base + + pip install airflow[kerberos] + OAuth Authentication -------------------- @@ -264,6 +270,12 @@ to only members of those teams. .. note:: If you do not specify a team whitelist, anyone with a valid account on your GHE installation will be able to login to Airflow. +To use GHE authentication, you must install Airflow with the `github_enterprise` extras group: + +.. code-block:: base + + pip install airflow[github_enterprise] + Setting up GHE Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -306,6 +318,12 @@ login, separated with a comma, to only members of those domains. oauth_callback_route = /oauth2callback domain = "example1.com,example2.com" +To use Google authentication, you must install Airflow with the `google_auth` extras group: + +.. code-block:: base + + pip install airflow[google_auth] + Setting up Google Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/setup.py b/setup.py index eded3c4e50230..76f55ab01b929 100644 --- a/setup.py +++ b/setup.py @@ -192,6 +192,7 @@ def write_version(filename=os.path.join(*['airflow', 'pandas-gbq' ] github_enterprise = ['Flask-OAuthlib>=0.9.1'] +google_auth = ['Flask-OAuthlib>=0.9.1'] hdfs = ['snakebite>=2.7.8'] hive = [ 'hmsclient>=0.1.0', @@ -358,6 +359,7 @@ def do_setup(): 'emr': emr, 'gcp_api': gcp_api, 'github_enterprise': github_enterprise, + 'google_auth': google_auth, 'hdfs': hdfs, 'hive': hive, 'jdbc': jdbc,