From 049399e52f0d7f166854e23c43706067ee56d68f Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Tue, 16 Apr 2019 16:20:07 +0530 Subject: [PATCH 01/12] Added airflow engine configs --- qds_sdk/engine.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qds_sdk/engine.py b/qds_sdk/engine.py index 811eaee5..8878e701 100644 --- a/qds_sdk/engine.py +++ b/qds_sdk/engine.py @@ -199,4 +199,12 @@ def engine_parser(argparser): dest="overrides", default=None, help="overrides for airflow cluster", ) + airflow_settings_group.add_argument("--overrides", + dest="version", + default="1.10.0", + help="overrides for airflow cluster", ) + airflow_settings_group.add_argument("--overrides", + dest="airflow_python_version", + default="2.7", + help="overrides for airflow cluster", ) From 75470d875df22cdec6af35f9fb8dcc5ab4ecc4d8 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Wed, 17 Apr 2019 11:36:32 +0530 Subject: [PATCH 02/12] Added airflow engine configs --- qds_sdk/engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qds_sdk/engine.py b/qds_sdk/engine.py index 8878e701..b97e4bbf 100644 --- a/qds_sdk/engine.py +++ b/qds_sdk/engine.py @@ -199,11 +199,11 @@ def engine_parser(argparser): dest="overrides", default=None, help="overrides for airflow cluster", ) - airflow_settings_group.add_argument("--overrides", + airflow_settings_group.add_argument("--airflow-version", dest="version", default="1.10.0", help="overrides for airflow cluster", ) - airflow_settings_group.add_argument("--overrides", + airflow_settings_group.add_argument("--airflow-python-version", dest="airflow_python_version", default="2.7", help="overrides for airflow cluster", ) From f797d89e0645f865ba991ee5fa45911d160a4016 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Wed, 17 Apr 2019 11:39:58 +0530 Subject: [PATCH 03/12] Added airflow engine configs --- build/lib/qds_sdk/engine.py | 221 ++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 build/lib/qds_sdk/engine.py diff --git a/build/lib/qds_sdk/engine.py b/build/lib/qds_sdk/engine.py new file mode 100644 index 00000000..4ff81ee1 --- /dev/null +++ b/build/lib/qds_sdk/engine.py @@ -0,0 +1,221 @@ +from qds_sdk import util + + +class Engine: + ''' + Use this class to set engine config settings of cluster + qds_sdk.engine.Engine is the class which stores information about engine config settings. + You can use objects of this class to set engine_config settings while create/update/clone a cluster. + ''' + + def __init__(self, flavour=None): + self.flavour = flavour + self.hadoop_settings = {} + self.presto_settings = {} + self.spark_settings = {} + self.airflow_settings ={} + self.engine_config = {} + + def set_engine_config(self, + custom_hadoop_config=None, + use_qubole_placement_policy=None, + fairscheduler_config_xml=None, + default_pool=None, + presto_version=None, + custom_presto_config=None, + spark_version=None, + custom_spark_config=None, + dbtap_id=None, + fernet_key=None, + overrides=None, + is_ha=None, + enable_rubix=None): + ''' + + Args: + custom_hadoop_config: Custom Hadoop configuration overrides. + + use_qubole_placement_policy: Use Qubole Block Placement policy for + clusters with spot nodes. + + fairscheduler_config_xml: XML string with custom configuration + parameters for the fair scheduler. + + default_pool: The default pool for the fair scheduler. + + presto_version: Version of presto to be used in cluster + + custom_presto_config: Custom Presto configuration overrides. + + spark_version: Version of spark to be used in cluster + + custom_spark_config: Specify the custom Spark configuration overrides + + dbtap_id: ID of the data store inside QDS + + fernet_key: Encryption key for sensitive information inside airflow database. + For example, user passwords and connections. It must be a 32 url-safe base64 encoded bytes. + + overrides: Airflow configuration to override the default settings.Use the following syntax for overrides: +
.=\n
.=... + + is_ha: Enabling HA config for cluster + is_deeplearning : this is a deeplearning cluster config + enable_rubix: Enable rubix on the cluster + + ''' + + self.set_hadoop_settings(custom_hadoop_config, use_qubole_placement_policy, is_ha, fairscheduler_config_xml, default_pool, enable_rubix) + self.set_presto_settings(presto_version, custom_presto_config) + self.set_spark_settings(spark_version, custom_spark_config) + self.set_airflow_settings(dbtap_id, fernet_key, overrides) + + def set_fairscheduler_settings(self, + fairscheduler_config_xml=None, + default_pool=None): + self.hadoop_settings['fairscheduler_settings'] = {} + self.hadoop_settings['fairscheduler_settings']['fairscheduler_config_xml'] = \ + fairscheduler_config_xml + self.hadoop_settings['fairscheduler_settings']['default_pool'] = default_pool + + def set_hadoop_settings(self, + custom_hadoop_config=None, + use_qubole_placement_policy=None, + is_ha=None, + fairscheduler_config_xml=None, + default_pool=None, + enable_rubix=None): + self.hadoop_settings['custom_hadoop_config'] = custom_hadoop_config + self.hadoop_settings['use_qubole_placement_policy'] = use_qubole_placement_policy + self.hadoop_settings['is_ha'] = is_ha + self.set_fairscheduler_settings(fairscheduler_config_xml, default_pool) + self.hadoop_settings['enable_rubix'] = enable_rubix + + def set_presto_settings(self, + presto_version=None, + custom_presto_config=None): + self.presto_settings['presto_version'] = presto_version + self.presto_settings['custom_presto_config'] = custom_presto_config + + def set_spark_settings(self, + spark_version=None, + custom_spark_config=None): + self.spark_settings['spark_version'] = spark_version + self.spark_settings['custom_spark_config'] = custom_spark_config + + def set_airflow_settings(self, + dbtap_id=None, + fernet_key=None, + overrides=None, + airflow_version="1.10.0", + airflow_python_version="2.7"): + self.airflow_settings['dbtap_id'] = dbtap_idl + self.airflow_settings['overrides'] = overrides + self.airflow_settings['version'] = airflow_version + self.airflow_settings['airflow_python_version'] = airflow_python_version + + def set_engine_config_settings(self, arguments): + custom_hadoop_config = util._read_file(arguments.custom_hadoop_config_file) + fairscheduler_config_xml = util._read_file(arguments.fairscheduler_config_xml_file) + custom_presto_config = util._read_file(arguments.presto_custom_config_file) + is_deeplearning=False + + self.set_engine_config(custom_hadoop_config=custom_hadoop_config, + use_qubole_placement_policy=arguments.use_qubole_placement_policy, + fairscheduler_config_xml=fairscheduler_config_xml, + default_pool=arguments.default_pool, + presto_version=arguments.presto_version, + custom_presto_config=custom_presto_config, + spark_version=arguments.spark_version, + custom_spark_config=arguments.custom_spark_config, + dbtap_id=arguments.dbtap_id, + fernet_key=arguments.fernet_key, + overrides=arguments.overrides, + enable_rubix=arguments.enable_rubix) + + @staticmethod + def engine_parser(argparser): + engine_group = argparser.add_argument_group("engine settings") + engine_group.add_argument("--flavour", + dest="flavour", + choices=["hadoop", "hadoop2", "presto", "spark", "hbase", "airflow", "deeplearning"], + default=None, + help="Set engine flavour") + + hadoop_settings_group = argparser.add_argument_group("hadoop settings") + hadoop_settings_group.add_argument("--custom-hadoop-config", + dest="custom_hadoop_config_file", + default=None, + help="location of file containing custom" + + " hadoop configuration overrides") + qubole_placement_policy_group = hadoop_settings_group.add_mutually_exclusive_group() + qubole_placement_policy_group.add_argument("--use-qubole-placement-policy", + dest="use_qubole_placement_policy", + action="store_true", + default=None, + help="Use Qubole Block Placement policy" + + " for clusters with spot nodes", ) + qubole_placement_policy_group.add_argument("--no-use-qubole-placement-policy", + dest="use_qubole_placement_policy", + action="store_false", + default=None, + help="Do not use Qubole Block Placement policy" + + " for clusters with spot nodes", ) + enable_rubix_group = hadoop_settings_group.add_mutually_exclusive_group() + enable_rubix_group.add_argument("--enable-rubix", + dest="enable_rubix", + action="store_true", + default=None, + help="Enable rubix for cluster", ) + enable_rubix_group.add_argument("--no-enable-rubix", + dest="enable_rubix", + action="store_false", + default=None, + help="Do not enable rubix for cluster", ) + + fairscheduler_group = argparser.add_argument_group( + "fairscheduler configuration options") + fairscheduler_group.add_argument("--fairscheduler-config-xml", + dest="fairscheduler_config_xml_file", + help="location for file containing" + + " xml with custom configuration" + + " for the fairscheduler", ) + fairscheduler_group.add_argument("--fairscheduler-default-pool", + dest="default_pool", + help="default pool for the" + + " fairscheduler", ) + + presto_settings_group = argparser.add_argument_group("presto settings") + presto_settings_group.add_argument("--presto-version", + dest="presto_version", + default=None, + help="Version of presto for this cluster", ) + presto_settings_group.add_argument("--presto-custom-config", + dest="presto_custom_config_file", + help="location of file containg custom" + + " presto configuration overrides") + + spark_settings_group = argparser.add_argument_group("spark settings") + spark_settings_group.add_argument("--spark-version", + dest="spark_version", + default=None, + help="Version of spark for the cluster", ) + spark_settings_group.add_argument("--custom-spark-config", + dest="custom_spark_config", + default=None, + help="Custom config spark for this cluster", ) + + airflow_settings_group = argparser.add_argument_group("airflow settings") + airflow_settings_group.add_argument("--dbtap-id", + dest="dbtap_id", + default=None, + help="dbtap id for airflow cluster", ) + airflow_settings_group.add_argument("--fernet-key", + dest="fernet_key", + default=None, + help="fernet key for airflow cluster", ) + airflow_settings_group.add_argument("--overrides", + dest="overrides", + default=None, + help="overrides for airflow cluster", ) + From 9c9d97012ae06c947b2a85ebf8c230f919eb3be9 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Wed, 17 Apr 2019 11:42:18 +0530 Subject: [PATCH 04/12] Added airflow engine configs --- build/lib/qds_sdk/engine.py | 221 ------------------------------------ 1 file changed, 221 deletions(-) delete mode 100644 build/lib/qds_sdk/engine.py diff --git a/build/lib/qds_sdk/engine.py b/build/lib/qds_sdk/engine.py deleted file mode 100644 index 4ff81ee1..00000000 --- a/build/lib/qds_sdk/engine.py +++ /dev/null @@ -1,221 +0,0 @@ -from qds_sdk import util - - -class Engine: - ''' - Use this class to set engine config settings of cluster - qds_sdk.engine.Engine is the class which stores information about engine config settings. - You can use objects of this class to set engine_config settings while create/update/clone a cluster. - ''' - - def __init__(self, flavour=None): - self.flavour = flavour - self.hadoop_settings = {} - self.presto_settings = {} - self.spark_settings = {} - self.airflow_settings ={} - self.engine_config = {} - - def set_engine_config(self, - custom_hadoop_config=None, - use_qubole_placement_policy=None, - fairscheduler_config_xml=None, - default_pool=None, - presto_version=None, - custom_presto_config=None, - spark_version=None, - custom_spark_config=None, - dbtap_id=None, - fernet_key=None, - overrides=None, - is_ha=None, - enable_rubix=None): - ''' - - Args: - custom_hadoop_config: Custom Hadoop configuration overrides. - - use_qubole_placement_policy: Use Qubole Block Placement policy for - clusters with spot nodes. - - fairscheduler_config_xml: XML string with custom configuration - parameters for the fair scheduler. - - default_pool: The default pool for the fair scheduler. - - presto_version: Version of presto to be used in cluster - - custom_presto_config: Custom Presto configuration overrides. - - spark_version: Version of spark to be used in cluster - - custom_spark_config: Specify the custom Spark configuration overrides - - dbtap_id: ID of the data store inside QDS - - fernet_key: Encryption key for sensitive information inside airflow database. - For example, user passwords and connections. It must be a 32 url-safe base64 encoded bytes. - - overrides: Airflow configuration to override the default settings.Use the following syntax for overrides: -
.=\n
.=... - - is_ha: Enabling HA config for cluster - is_deeplearning : this is a deeplearning cluster config - enable_rubix: Enable rubix on the cluster - - ''' - - self.set_hadoop_settings(custom_hadoop_config, use_qubole_placement_policy, is_ha, fairscheduler_config_xml, default_pool, enable_rubix) - self.set_presto_settings(presto_version, custom_presto_config) - self.set_spark_settings(spark_version, custom_spark_config) - self.set_airflow_settings(dbtap_id, fernet_key, overrides) - - def set_fairscheduler_settings(self, - fairscheduler_config_xml=None, - default_pool=None): - self.hadoop_settings['fairscheduler_settings'] = {} - self.hadoop_settings['fairscheduler_settings']['fairscheduler_config_xml'] = \ - fairscheduler_config_xml - self.hadoop_settings['fairscheduler_settings']['default_pool'] = default_pool - - def set_hadoop_settings(self, - custom_hadoop_config=None, - use_qubole_placement_policy=None, - is_ha=None, - fairscheduler_config_xml=None, - default_pool=None, - enable_rubix=None): - self.hadoop_settings['custom_hadoop_config'] = custom_hadoop_config - self.hadoop_settings['use_qubole_placement_policy'] = use_qubole_placement_policy - self.hadoop_settings['is_ha'] = is_ha - self.set_fairscheduler_settings(fairscheduler_config_xml, default_pool) - self.hadoop_settings['enable_rubix'] = enable_rubix - - def set_presto_settings(self, - presto_version=None, - custom_presto_config=None): - self.presto_settings['presto_version'] = presto_version - self.presto_settings['custom_presto_config'] = custom_presto_config - - def set_spark_settings(self, - spark_version=None, - custom_spark_config=None): - self.spark_settings['spark_version'] = spark_version - self.spark_settings['custom_spark_config'] = custom_spark_config - - def set_airflow_settings(self, - dbtap_id=None, - fernet_key=None, - overrides=None, - airflow_version="1.10.0", - airflow_python_version="2.7"): - self.airflow_settings['dbtap_id'] = dbtap_idl - self.airflow_settings['overrides'] = overrides - self.airflow_settings['version'] = airflow_version - self.airflow_settings['airflow_python_version'] = airflow_python_version - - def set_engine_config_settings(self, arguments): - custom_hadoop_config = util._read_file(arguments.custom_hadoop_config_file) - fairscheduler_config_xml = util._read_file(arguments.fairscheduler_config_xml_file) - custom_presto_config = util._read_file(arguments.presto_custom_config_file) - is_deeplearning=False - - self.set_engine_config(custom_hadoop_config=custom_hadoop_config, - use_qubole_placement_policy=arguments.use_qubole_placement_policy, - fairscheduler_config_xml=fairscheduler_config_xml, - default_pool=arguments.default_pool, - presto_version=arguments.presto_version, - custom_presto_config=custom_presto_config, - spark_version=arguments.spark_version, - custom_spark_config=arguments.custom_spark_config, - dbtap_id=arguments.dbtap_id, - fernet_key=arguments.fernet_key, - overrides=arguments.overrides, - enable_rubix=arguments.enable_rubix) - - @staticmethod - def engine_parser(argparser): - engine_group = argparser.add_argument_group("engine settings") - engine_group.add_argument("--flavour", - dest="flavour", - choices=["hadoop", "hadoop2", "presto", "spark", "hbase", "airflow", "deeplearning"], - default=None, - help="Set engine flavour") - - hadoop_settings_group = argparser.add_argument_group("hadoop settings") - hadoop_settings_group.add_argument("--custom-hadoop-config", - dest="custom_hadoop_config_file", - default=None, - help="location of file containing custom" + - " hadoop configuration overrides") - qubole_placement_policy_group = hadoop_settings_group.add_mutually_exclusive_group() - qubole_placement_policy_group.add_argument("--use-qubole-placement-policy", - dest="use_qubole_placement_policy", - action="store_true", - default=None, - help="Use Qubole Block Placement policy" + - " for clusters with spot nodes", ) - qubole_placement_policy_group.add_argument("--no-use-qubole-placement-policy", - dest="use_qubole_placement_policy", - action="store_false", - default=None, - help="Do not use Qubole Block Placement policy" + - " for clusters with spot nodes", ) - enable_rubix_group = hadoop_settings_group.add_mutually_exclusive_group() - enable_rubix_group.add_argument("--enable-rubix", - dest="enable_rubix", - action="store_true", - default=None, - help="Enable rubix for cluster", ) - enable_rubix_group.add_argument("--no-enable-rubix", - dest="enable_rubix", - action="store_false", - default=None, - help="Do not enable rubix for cluster", ) - - fairscheduler_group = argparser.add_argument_group( - "fairscheduler configuration options") - fairscheduler_group.add_argument("--fairscheduler-config-xml", - dest="fairscheduler_config_xml_file", - help="location for file containing" + - " xml with custom configuration" + - " for the fairscheduler", ) - fairscheduler_group.add_argument("--fairscheduler-default-pool", - dest="default_pool", - help="default pool for the" + - " fairscheduler", ) - - presto_settings_group = argparser.add_argument_group("presto settings") - presto_settings_group.add_argument("--presto-version", - dest="presto_version", - default=None, - help="Version of presto for this cluster", ) - presto_settings_group.add_argument("--presto-custom-config", - dest="presto_custom_config_file", - help="location of file containg custom" + - " presto configuration overrides") - - spark_settings_group = argparser.add_argument_group("spark settings") - spark_settings_group.add_argument("--spark-version", - dest="spark_version", - default=None, - help="Version of spark for the cluster", ) - spark_settings_group.add_argument("--custom-spark-config", - dest="custom_spark_config", - default=None, - help="Custom config spark for this cluster", ) - - airflow_settings_group = argparser.add_argument_group("airflow settings") - airflow_settings_group.add_argument("--dbtap-id", - dest="dbtap_id", - default=None, - help="dbtap id for airflow cluster", ) - airflow_settings_group.add_argument("--fernet-key", - dest="fernet_key", - default=None, - help="fernet key for airflow cluster", ) - airflow_settings_group.add_argument("--overrides", - dest="overrides", - default=None, - help="overrides for airflow cluster", ) - From ca735fd42f41efaef49821fa489b04277b8417e6 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Wed, 17 Apr 2019 11:57:14 +0530 Subject: [PATCH 05/12] Added airflow engine configs --- qds_sdk/engine.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qds_sdk/engine.py b/qds_sdk/engine.py index b97e4bbf..97c33655 100644 --- a/qds_sdk/engine.py +++ b/qds_sdk/engine.py @@ -28,6 +28,8 @@ def set_engine_config(self, dbtap_id=None, fernet_key=None, overrides=None, + version=None, + airflow_python_version=None, is_ha=None): ''' @@ -66,7 +68,7 @@ def set_engine_config(self, self.set_hadoop_settings(custom_hadoop_config, use_qubole_placement_policy, is_ha, fairscheduler_config_xml, default_pool) self.set_presto_settings(presto_version, custom_presto_config) self.set_spark_settings(spark_version, custom_spark_config) - self.set_airflow_settings(dbtap_id, fernet_key, overrides) + self.set_airflow_settings(dbtap_id, fernet_key, overrides, version, airflow_python_version) def set_fairscheduler_settings(self, fairscheduler_config_xml=None, @@ -102,10 +104,14 @@ def set_spark_settings(self, def set_airflow_settings(self, dbtap_id=None, fernet_key=None, - overrides=None): + overrides=None, + version="1.10.0", + airflow_python_version="2.7"): self.airflow_settings['dbtap_id'] = dbtap_id self.airflow_settings['fernet_key'] = fernet_key self.airflow_settings['overrides'] = overrides + self.airflow_settings['version'] = version + self.airflow_settings['airflow_python_version'] = airflow_python_version def set_engine_config_settings(self, arguments): custom_hadoop_config = util._read_file(arguments.custom_hadoop_config_file) @@ -123,7 +129,9 @@ def set_engine_config_settings(self, arguments): custom_spark_config=arguments.custom_spark_config, dbtap_id=arguments.dbtap_id, fernet_key=arguments.fernet_key, - overrides=arguments.overrides) + overrides=arguments.overrides, + version=arguments.version, + airflow_python_version=arguments.airflow_python_version) @staticmethod def engine_parser(argparser): From 87c532d9627f4d3d90bdb75155c854ec6c789b71 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Wed, 17 Apr 2019 12:05:53 +0530 Subject: [PATCH 06/12] Added airflow engine configs --- qds_sdk/engine.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/qds_sdk/engine.py b/qds_sdk/engine.py index 97c33655..1135b066 100644 --- a/qds_sdk/engine.py +++ b/qds_sdk/engine.py @@ -28,7 +28,7 @@ def set_engine_config(self, dbtap_id=None, fernet_key=None, overrides=None, - version=None, + airflow_version=None, airflow_python_version=None, is_ha=None): ''' @@ -68,7 +68,7 @@ def set_engine_config(self, self.set_hadoop_settings(custom_hadoop_config, use_qubole_placement_policy, is_ha, fairscheduler_config_xml, default_pool) self.set_presto_settings(presto_version, custom_presto_config) self.set_spark_settings(spark_version, custom_spark_config) - self.set_airflow_settings(dbtap_id, fernet_key, overrides, version, airflow_python_version) + self.set_airflow_settings(dbtap_id, fernet_key, overrides, airflow_version, airflow_python_version) def set_fairscheduler_settings(self, fairscheduler_config_xml=None, @@ -105,12 +105,12 @@ def set_airflow_settings(self, dbtap_id=None, fernet_key=None, overrides=None, - version="1.10.0", + airflow_version="1.10.0", airflow_python_version="2.7"): self.airflow_settings['dbtap_id'] = dbtap_id self.airflow_settings['fernet_key'] = fernet_key self.airflow_settings['overrides'] = overrides - self.airflow_settings['version'] = version + self.airflow_settings['version'] = airflow_version self.airflow_settings['airflow_python_version'] = airflow_python_version def set_engine_config_settings(self, arguments): @@ -130,7 +130,7 @@ def set_engine_config_settings(self, arguments): dbtap_id=arguments.dbtap_id, fernet_key=arguments.fernet_key, overrides=arguments.overrides, - version=arguments.version, + airflow_version=arguments.airflow_version, airflow_python_version=arguments.airflow_python_version) @staticmethod From cfaf7fed71144789d734b2fbd3f735e646db97b5 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Fri, 19 Apr 2019 11:51:46 +0530 Subject: [PATCH 07/12] Set default value to NONE --- qds_sdk/engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qds_sdk/engine.py b/qds_sdk/engine.py index abfe88d4..49075660 100644 --- a/qds_sdk/engine.py +++ b/qds_sdk/engine.py @@ -225,10 +225,10 @@ def engine_parser(argparser): help="overrides for airflow cluster", ) airflow_settings_group.add_argument("--airflow-version", dest="version", - default="1.10.0", + default=None, help="overrides for airflow cluster", ) airflow_settings_group.add_argument("--airflow-python-version", dest="airflow_python_version", - default="2.7", + default=None, help="overrides for airflow cluster", ) From fd1d1fd1d15ded23b0654ffbd21ddcf3e3586646 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Fri, 19 Apr 2019 11:56:47 +0530 Subject: [PATCH 08/12] Set default value to NONE --- qds_sdk/engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qds_sdk/engine.py b/qds_sdk/engine.py index 49075660..c3050487 100644 --- a/qds_sdk/engine.py +++ b/qds_sdk/engine.py @@ -224,11 +224,11 @@ def engine_parser(argparser): default=None, help="overrides for airflow cluster", ) airflow_settings_group.add_argument("--airflow-version", - dest="version", - default=None, + dest="airflow_version", + default="1.10.0", help="overrides for airflow cluster", ) airflow_settings_group.add_argument("--airflow-python-version", dest="airflow_python_version", - default=None, + default="2.7", help="overrides for airflow cluster", ) From 38867ce3c42e06cc6ed9560a8cefb78153e54514 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Fri, 19 Apr 2019 12:16:11 +0530 Subject: [PATCH 09/12] Changed name to fix broken test cases --- qds_sdk/engine.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qds_sdk/engine.py b/qds_sdk/engine.py index c3050487..9f65c400 100644 --- a/qds_sdk/engine.py +++ b/qds_sdk/engine.py @@ -225,10 +225,10 @@ def engine_parser(argparser): help="overrides for airflow cluster", ) airflow_settings_group.add_argument("--airflow-version", dest="airflow_version", - default="1.10.0", + default=None, help="overrides for airflow cluster", ) airflow_settings_group.add_argument("--airflow-python-version", dest="airflow_python_version", - default="2.7", + default=None, help="overrides for airflow cluster", ) From fafdb455a9e50d7480e2498db3daefb6e935aba1 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Fri, 19 Apr 2019 13:53:50 +0530 Subject: [PATCH 10/12] Added test for airflow engine config --- tests/test_clusterv2.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_clusterv2.py b/tests/test_clusterv2.py index 1b33f4e2..2234573a 100644 --- a/tests/test_clusterv2.py +++ b/tests/test_clusterv2.py @@ -444,6 +444,27 @@ def test_spark_engine_config(self): 'custom_spark_config': 'spark-overrides'}}, 'cluster_info': {'label': ['test_label'],}}) + def test_airflow_engine_config(self): + with tempfile.NamedTemporaryFile() as temp: + temp.write("config.properties:\na=1\nb=2".encode("utf8")) + temp.flush() + sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label', + '--flavour', 'airflow', '--dbtap-id', '1', '--fernet-key', '-1', '--overrides', 'airflow_overrides', '--airflow-version', '1.10.0', '--airflow-python-version', '2.7'] + Qubole.cloud = None + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with('POST', 'clusters', + {'engine_config': + {'flavour': 'airflow', + 'airflow_settings': { + 'dbtap_id': '1', + 'fernet_key': '-1', + 'overrides': 'airflow_overrides', + 'airflow_version': '1.10.0', + 'airflow_python_version': '2.7' + }}, + 'cluster_info': {'label': ['test_label'],}}) def test_persistent_security_groups_v2(self): sys.argv = ['qds.py', '--version', 'v2', 'cluster', 'create', '--label', 'test_label', From 8a2a0b4f094042a3c27e0184ea72f48425f83487 Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Fri, 19 Apr 2019 14:03:06 +0530 Subject: [PATCH 11/12] Added test for airflow engine config --- tests/test_clusterv2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_clusterv2.py b/tests/test_clusterv2.py index 2234573a..2c3ebaae 100644 --- a/tests/test_clusterv2.py +++ b/tests/test_clusterv2.py @@ -461,7 +461,7 @@ def test_airflow_engine_config(self): 'dbtap_id': '1', 'fernet_key': '-1', 'overrides': 'airflow_overrides', - 'airflow_version': '1.10.0', + 'version': '1.10.0', 'airflow_python_version': '2.7' }}, 'cluster_info': {'label': ['test_label'],}}) From bd2a5c0f3368ec81ab6cd149dee4616c591ac30c Mon Sep 17 00:00:00 2001 From: Aaditya Sharma Date: Fri, 19 Apr 2019 15:26:24 +0530 Subject: [PATCH 12/12] Added text --- qds_sdk/engine.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/qds_sdk/engine.py b/qds_sdk/engine.py index 9f65c400..3070f699 100644 --- a/qds_sdk/engine.py +++ b/qds_sdk/engine.py @@ -61,6 +61,10 @@ def set_engine_config(self, overrides: Airflow configuration to override the default settings.Use the following syntax for overrides:
.=\n
.=... + airflow_version: The airflow version. + + airflow_python_version: The python version for the environment on the cluster. + is_ha: Enabling HA config for cluster is_deeplearning : this is a deeplearning cluster config enable_rubix: Enable rubix on the cluster @@ -226,9 +230,9 @@ def engine_parser(argparser): airflow_settings_group.add_argument("--airflow-version", dest="airflow_version", default=None, - help="overrides for airflow cluster", ) + help="airflow version for airflow cluster", ) airflow_settings_group.add_argument("--airflow-python-version", dest="airflow_python_version", default=None, - help="overrides for airflow cluster", ) + help="python environment version for airflow cluster", )