From 9a159ce94574a199ebdcc3975e0aec52079b20a6 Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Mon, 25 Mar 2019 20:14:43 +0800 Subject: [PATCH] [AIRFLOW-4062] Improve docs on install extra package commands (#4966) Some command for installing extra packages like `pip install apache-airflow[devel]` cause error in special situation/shell, We should clear them by add quotation like `pip install 'apache-airflow[devel]'` --- CONTRIBUTING.md | 8 +- .../example_kubernetes_operator.py | 4 +- airflow/contrib/utils/sendgrid.py | 2 +- docs/howto/secure-connections.rst | 4 +- docs/howto/write-logs.rst | 2 +- docs/installation.rst | 142 +++++++++--------- docs/security.rst | 6 +- 7 files changed, 85 insertions(+), 83 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3326018234c74..903f453d9faa2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,7 +85,9 @@ The latest API documentation is usually available you need to have set up an Airflow development environemnt (see below). Also install the `doc` extra. - pip install -e .[doc] +``` +pip install -e '.[doc]' +``` Generate the documentation by running: @@ -112,7 +114,7 @@ There are three ways to setup an Apache Airflow development environment. cd $AIRFLOW_HOME virtualenv env source env/bin/activate - pip install -e .[devel] + pip install -e '.[devel]' ``` 2. Using a Docker container @@ -126,7 +128,7 @@ There are three ways to setup an Apache Airflow development environment. # Install Airflow with all the required dependencies, # including the devel which will provide the development tools - pip install -e ".[hdfs,hive,druid,devel]" + pip install -e '.[hdfs,hive,druid,devel]' # Init the database airflow initdb diff --git a/airflow/contrib/example_dags/example_kubernetes_operator.py b/airflow/contrib/example_dags/example_kubernetes_operator.py index ba1f4433a78cc..4977890bc68f3 100644 --- a/airflow/contrib/example_dags/example_kubernetes_operator.py +++ b/airflow/contrib/example_dags/example_kubernetes_operator.py @@ -25,7 +25,7 @@ try: # Kubernetes is optional, so not available in vanilla Airflow - # pip install apache-airflow[kubernetes] + # pip install 'apache-airflow[kubernetes]' from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator args = { @@ -64,4 +64,4 @@ except ImportError as e: log.warn("Could not import KubernetesPodOperator: " + str(e)) log.warn("Install kubernetes dependencies with: " - " pip install apache-airflow[kubernetes]") + " pip install 'apache-airflow[kubernetes]'") diff --git a/airflow/contrib/utils/sendgrid.py b/airflow/contrib/utils/sendgrid.py index ceb3718ff584d..5082c7a3328bf 100644 --- a/airflow/contrib/utils/sendgrid.py +++ b/airflow/contrib/utils/sendgrid.py @@ -42,7 +42,7 @@ def send_email(to, subject, html_content, files=None, To use this plugin: 0. include sendgrid subpackage as part of your Airflow installation, e.g., - pip install apache-airflow[sendgrid] + pip install 'apache-airflow[sendgrid]' 1. update [email] backend in airflow.cfg, i.e., [email] email_backend = airflow.contrib.utils.sendgrid.send_email diff --git a/docs/howto/secure-connections.rst b/docs/howto/secure-connections.rst index 47b97c5fa4f47..eef49b3234982 100644 --- a/docs/howto/secure-connections.rst +++ b/docs/howto/secure-connections.rst @@ -27,8 +27,8 @@ If ``crypto`` package was not installed initially, it means that your Fernet key You can still enable encryption for passwords within connections by following below steps: -1. Install crypto package ``pip install apache-airflow[crypto]`` -2. Generate fernet_key, using this code snippet below. ``fernet_key`` must be a base64-encoded 32-byte key. +#. Install crypto package ``pip install 'apache-airflow[crypto]'`` +#. Generate fernet_key, using this code snippet below. ``fernet_key`` must be a base64-encoded 32-byte key: .. code:: python diff --git a/docs/howto/write-logs.rst b/docs/howto/write-logs.rst index f3cc4017f443d..b12b60ef86def 100644 --- a/docs/howto/write-logs.rst +++ b/docs/howto/write-logs.rst @@ -121,7 +121,7 @@ example: remote_base_log_folder = gs://my-bucket/path/to/logs remote_log_conn_id = MyGCSConn -#. Install the ``gcp_api`` package first, like so: ``pip install apache-airflow[gcp_api]``. +#. Install the ``gcp_api`` package first, like so: ``pip install 'apache-airflow[gcp_api]'``. #. Make sure a Google Cloud Platform connection hook has been defined in Airflow. The hook should have read and write access to the Google Cloud Storage bucket defined above in ``remote_base_log_folder``. #. Restart the Airflow webserver and scheduler, and trigger (or wait for) a new task execution. #. Verify that logs are showing up for newly executed tasks in the bucket you've defined. diff --git a/docs/installation.rst b/docs/installation.rst index 33c71439d241e..ad704c7dceebb 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -47,77 +47,77 @@ these extra dependencies. Here's the list of the subpackages and what they enable: -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| subpackage | install command | enables | -+=====================+===================================================+=================================================+ -| all | ``pip install apache-airflow[all]`` | All Airflow features known to man | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| all_dbs | ``pip install apache-airflow[all_dbs]`` | All databases integrations | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| async | ``pip install apache-airflow[async]`` | Async worker classes for Gunicorn | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| celery | ``pip install apache-airflow[celery]`` | CeleryExecutor | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| cloudant | ``pip install apache-airflow[cloudant]`` | Cloudant hook | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| crypto | ``pip install apache-airflow[crypto]`` | Encrypt connection passwords in metadata db | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| devel | ``pip install apache-airflow[devel]`` | Minimum dev tools requirements | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| devel_hadoop | ``pip install apache-airflow[devel_hadoop]`` | Airflow + dependencies on the Hadoop stack | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| druid | ``pip install apache-airflow[druid]`` | Druid related operators & hooks | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| gcp_api | ``pip install apache-airflow[gcp_api]`` | Google Cloud Platform hooks and operators | -| | | (using ``google-api-python-client``) | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| github_enterprise | ``pip install apache-airflow[github_enterprise]`` | GitHub Enterprise auth backend | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| google_auth | ``pip install apache-airflow[google_auth]`` | Google auth backend | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| hdfs | ``pip install apache-airflow[hdfs]`` | HDFS hooks and operators | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| hive | ``pip install apache-airflow[hive]`` | All Hive related operators | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| jdbc | ``pip install apache-airflow[jdbc]`` | JDBC hooks and operators | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| kerberos | ``pip install apache-airflow[kerberos]`` | Kerberos integration for Kerberized Hadoop | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| kubernetes | ``pip install apache-airflow[kubernetes]`` | Kubernetes Executor and operator | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| ldap | ``pip install apache-airflow[ldap]`` | LDAP authentication for users | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| mssql | ``pip install apache-airflow[mssql]`` | Microsoft SQL Server operators and hook, | -| | | support as an Airflow backend | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| mysql | ``pip install apache-airflow[mysql]`` | MySQL operators and hook, support as an Airflow | -| | | backend. The version of MySQL server has to be | -| | | 5.6.4+. The exact version upper bound depends | -| | | on version of ``mysqlclient`` package. For | -| | | example, ``mysqlclient`` 1.3.12 can only be | -| | | used with MySQL server 5.6.4 through 5.7. | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| password | ``pip install apache-airflow[password]`` | Password authentication for users | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| postgres | ``pip install apache-airflow[postgres]`` | PostgreSQL operators and hook, support as an | -| | | Airflow backend | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| qds | ``pip install apache-airflow[qds]`` | Enable QDS (Qubole Data Service) support | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| rabbitmq | ``pip install apache-airflow[rabbitmq]`` | RabbitMQ support as a Celery backend | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| redis | ``pip install apache-airflow[redis]`` | Redis hooks and sensors | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| s3 | ``pip install apache-airflow[s3]`` | ``S3KeySensor``, ``S3PrefixSensor`` | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| samba | ``pip install apache-airflow[samba]`` | ``Hive2SambaOperator`` | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| slack | ``pip install apache-airflow[slack]`` | ``SlackAPIPostOperator`` | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| ssh | ``pip install apache-airflow[ssh]`` | SSH hooks and Operator | -+---------------------+---------------------------------------------------+-------------------------------------------------+ -| vertica | ``pip install apache-airflow[vertica]`` | Vertica hook support as an Airflow backend | -+---------------------+---------------------------------------------------+-------------------------------------------------+ ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| subpackage | install command | enables | ++=====================+=====================================================+=================================================+ +| all | ``pip install 'apache-airflow[all]'`` | All Airflow features known to man | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| all_dbs | ``pip install 'apache-airflow[all_dbs]'`` | All databases integrations | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| async | ``pip install 'apache-airflow[async]'`` | Async worker classes for Gunicorn | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| celery | ``pip install 'apache-airflow[celery]'`` | CeleryExecutor | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| cloudant | ``pip install 'apache-airflow[cloudant]'`` | Cloudant hook | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| crypto | ``pip install 'apache-airflow[crypto]'`` | Encrypt connection passwords in metadata db | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| devel | ``pip install 'apache-airflow[devel]'`` | Minimum dev tools requirements | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| devel_hadoop | ``pip install 'apache-airflow[devel_hadoop]'`` | Airflow + dependencies on the Hadoop stack | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| druid | ``pip install 'apache-airflow[druid]'`` | Druid related operators & hooks | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| gcp_api | ``pip install 'apache-airflow[gcp_api]'`` | Google Cloud Platform hooks and operators | +| | | (using ``google-api-python-client``) | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| github_enterprise | ``pip install 'apache-airflow[github_enterprise]'`` | GitHub Enterprise auth backend | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| google_auth | ``pip install 'apache-airflow[google_auth]'`` | Google auth backend | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| hdfs | ``pip install 'apache-airflow[hdfs]'`` | HDFS hooks and operators | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| hive | ``pip install 'apache-airflow[hive]'`` | All Hive related operators | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| jdbc | ``pip install 'apache-airflow[jdbc]'`` | JDBC hooks and operators | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| kerberos | ``pip install 'apache-airflow[kerberos]'`` | Kerberos integration for Kerberized Hadoop | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| kubernetes | ``pip install 'apache-airflow[kubernetes]'`` | Kubernetes Executor and operator | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| ldap | ``pip install 'apache-airflow[ldap]'`` | LDAP authentication for users | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| mssql | ``pip install 'apache-airflow[mssql]'`` | Microsoft SQL Server operators and hook, | +| | | support as an Airflow backend | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| mysql | ``pip install 'apache-airflow[mysql]'`` | MySQL operators and hook, support as an Airflow | +| | | backend. The version of MySQL server has to be | +| | | 5.6.4+. The exact version upper bound depends | +| | | on version of ``mysqlclient`` package. For | +| | | example, ``mysqlclient`` 1.3.12 can only be | +| | | used with MySQL server 5.6.4 through 5.7. | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| password | ``pip install 'apache-airflow[password]'`` | Password authentication for users | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| postgres | ``pip install 'apache-airflow[postgres]'`` | PostgreSQL operators and hook, support as an | +| | | Airflow backend | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| qds | ``pip install 'apache-airflow[qds]'`` | Enable QDS (Qubole Data Service) support | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| rabbitmq | ``pip install 'apache-airflow[rabbitmq]'`` | RabbitMQ support as a Celery backend | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| redis | ``pip install 'apache-airflow[redis]'`` | Redis hooks and sensors | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| s3 | ``pip install 'apache-airflow[s3]'`` | ``S3KeySensor``, ``S3PrefixSensor`` | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| samba | ``pip install 'apache-airflow[samba]'`` | ``Hive2SambaOperator`` | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| slack | ``pip install 'apache-airflow[slack]'`` | ``SlackAPIPostOperator`` | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| ssh | ``pip install 'apache-airflow[ssh]'`` | SSH hooks and Operator | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ +| vertica | ``pip install 'apache-airflow[vertica]'`` | Vertica hook support as an Airflow backend | ++---------------------+-----------------------------------------------------+-------------------------------------------------+ Initiating Airflow Database ''''''''''''''''''''''''''' diff --git a/docs/security.rst b/docs/security.rst index 1234f90c2f364..4058ddfe61c0e 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -262,7 +262,7 @@ To use kerberos authentication, you must install Airflow with the `kerberos` ext .. code-block:: bash - pip install airflow[kerberos] + pip install 'apache-airflow[kerberos]' OAuth Authentication -------------------- @@ -295,7 +295,7 @@ To use GHE authentication, you must install Airflow with the `github_enterprise` .. code-block:: bash - pip install airflow[github_enterprise] + pip install 'apache-airflow[github_enterprise]' Setting up GHE Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -343,7 +343,7 @@ To use Google authentication, you must install Airflow with the `google_auth` ex .. code-block:: bash - pip install airflow[google_auth] + pip install 'apache-airflow[google_auth]' Setting up Google Authentication ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^