From e0fc9e55d07e84a6e53beac3a37b6aa6897c5373 Mon Sep 17 00:00:00 2001
From: Sumit Maheshwari <sumeet.manit@gmail.com>
Date: Mon, 1 Oct 2018 11:58:27 +0530
Subject: [PATCH] [AIRFLOW-3062] Add Qubole in integration docs (#3946)

---
 airflow/contrib/sensors/qubole_sensor.py | 32 ++++++++++++++++++++++++
 docs/integration.rst                     | 26 +++++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/airflow/contrib/sensors/qubole_sensor.py b/airflow/contrib/sensors/qubole_sensor.py
index d67fa8557e7ad..7065ae9262595 100644
--- a/airflow/contrib/sensors/qubole_sensor.py
+++ b/airflow/contrib/sensors/qubole_sensor.py
@@ -75,6 +75,22 @@ def poke(self, context):
 
 
 class QuboleFileSensor(QuboleSensor):
+    """
+    Wait for a file or folder to be present in cloud storage
+    and check for its presence via QDS APIs
+
+    :param qubole_conn_id: Connection id which consists of qds auth_token
+    :type qubole_conn_id: str
+    :param data: a JSON object containing payload, whose presence needs to be checked
+        Check this `example <https://github.com/apache/incubator-airflow/blob/master\
+        /airflow/contrib/example_dags/example_qubole_sensor.py>`_ for sample payload
+        structure.
+    :type data: a JSON object
+
+    .. note:: Both ``data`` and ``qubole_conn_id`` fields support templating. You can
+        also use ``.txt`` files for template-driven use cases.
+    """
+
     @apply_defaults
     def __init__(self, *args, **kwargs):
         self.sensor_class = FileSensor
@@ -82,6 +98,22 @@ def __init__(self, *args, **kwargs):
 
 
 class QubolePartitionSensor(QuboleSensor):
+    """
+    Wait for a Hive partition to show up in QHS (Qubole Hive Service)
+    and check for its presence via QDS APIs
+
+    :param qubole_conn_id: Connection id which consists of qds auth_token
+    :type qubole_conn_id: str
+    :param data: a JSON object containing payload, whose presence needs to be checked.
+        Check this `example <https://github.com/apache/incubator-airflow/blob/master\
+        /airflow/contrib/example_dags/example_qubole_sensor.py>`_ for sample payload
+        structure.
+    :type data: a JSON object
+
+    .. note:: Both ``data`` and ``qubole_conn_id`` fields support templating. You can
+        also use ``.txt`` files for template-driven use cases.
+    """
+
     @apply_defaults
     def __init__(self, *args, **kwargs):
         self.sensor_class = PartitionSensor
diff --git a/docs/integration.rst b/docs/integration.rst
index 9d3e6c9828f59..f3b878a3f9645 100644
--- a/docs/integration.rst
+++ b/docs/integration.rst
@@ -23,6 +23,7 @@ Integration
 - :ref:`AWS`
 - :ref:`Databricks`
 - :ref:`GCP`
+- :ref:`Qubole`
 
 .. _ReverseProxy:
 
@@ -972,3 +973,28 @@ Google Kubernetes Engine Hook
 
 .. autoclass:: airflow.contrib.hooks.gcp_container_hook.GKEClusterHook
     :members:
+
+
+.. _Qubole:
+
+Qubole
+------
+
+Apache Airflow has a native operator and hooks to talk to `Qubole <https://qubole.com/>`__,
+which lets you submit your big data jobs directly to Qubole from Apache Airflow.
+
+QuboleOperator
+''''''''''''''
+
+.. autoclass:: airflow.contrib.operators.qubole_operator.QuboleOperator
+
+QubolePartitionSensor
+'''''''''''''''''''''
+
+.. autoclass:: airflow.contrib.sensors.qubole_sensor.QubolePartitionSensor
+
+
+QuboleFileSensor
+''''''''''''''''
+
+.. autoclass:: airflow.contrib.sensors.qubole_sensor.QuboleFileSensor