From 96d16fd328e85a781a6ce10d1574942dc0a16b3e Mon Sep 17 00:00:00 2001 From: Harsh Date: Thu, 18 Jun 2020 23:26:17 -0700 Subject: [PATCH] Ability to set azure cluster composition (#323) --- bin/qds.py | 2 +- qds_sdk/cloud/aws_cloud.py | 141 +++++++++++++++++++++++++++ qds_sdk/cloud/azure_cloud.py | 85 ++++++++++++++++- qds_sdk/cloud/cloud.py | 6 ++ qds_sdk/cluster_info_v22.py | 180 ++++++++--------------------------- qds_sdk/qubole.py | 4 + tests/test_clusterv22.py | 55 +++++++++++ 7 files changed, 331 insertions(+), 142 deletions(-) diff --git a/bin/qds.py b/bin/qds.py index ea8deb3e..01cde904 100755 --- a/bin/qds.py +++ b/bin/qds.py @@ -587,7 +587,7 @@ def main(): help="skip verification of server SSL certificate. Insecure: use with caution.") optparser.add_option("--cloud_name", dest="cloud_name", - default=os.getenv('CLOUD_PROVIDER'), + default=os.getenv('CLOUD_PROVIDER', "AWS"), help="cloud", choices=["AWS", "AZURE", "ORACLE_BMC", "ORACLE_OPC", "GCP"]) optparser.add_option("--base_retry_delay", dest="base_retry_delay", diff --git a/qds_sdk/cloud/aws_cloud.py b/qds_sdk/cloud/aws_cloud.py index b4f56156..6a75ef8a 100755 --- a/qds_sdk/cloud/aws_cloud.py +++ b/qds_sdk/cloud/aws_cloud.py @@ -107,6 +107,147 @@ def set_cloud_config_from_arguments(self, arguments): bastion_node_public_dns=arguments.bastion_node_public_dns, master_elastic_ip=arguments.master_elastic_ip) + def set_composition_arguments(self, comp_group): + comp_group.add_argument( + "--master-type", + dest="master_type", + choices=["ondemand", "spot", "spotblock"], + default="ondemand", + help="type of master nodes. Valid values are:" + + " ('ondemand', 'spot', 'spotblock')" + + " default: ondemand") + comp_group.add_argument( + "--master-spot-block-duration", + dest="master_spot_block_duration", + type=int, + default=120, + help="spot block duration unit: minutes") + comp_group.add_argument( + "--master-maximum-bid-price-percentage", + dest="master_maximum_bid_price_percentage", + type=int, + default=100, + help="maximum value to bid for master spot instances" + + " expressed as a percentage of the base" + + " price for the master instance types") + comp_group.add_argument( + "--master-timeout-for-request", + dest="master_timeout_for_request", + type=int, + default=1, + help="timeout for a master spot instance request, unit: minutes") + comp_group.add_argument( + "--master-spot-fallback", + dest="master_spot_fallback", + choices=["ondemand", None], + default=None, + help="whether to fallback to on-demand instances for master nodes" + + " if spot instances aren't available") + comp_group.add_argument( + "--min-ondemand-percentage", + dest="min_ondemand_percentage", + type=int, + help="percentage of ondemand nodes in min config") + comp_group.add_argument( + "--min-spot-block-percentage", + dest="min_spot_block_percentage", + type=int, + help="percentage of spot block nodes in min config") + comp_group.add_argument( + "--min-spot-percentage", + dest="min_spot_percentage", + type=int, + help="percentage of spot nodes in min config") + comp_group.add_argument( + "--min-spot-block-duration", + dest="min_spot_block_duration", + type=int, + default=120, + help="spot block duration unit: minutes") + comp_group.add_argument( + "--min-maximum-bid-price-percentage", + dest="min_maximum_bid_price_percentage", + type=int, + default=100, + help="maximum value to bid for min spot instances" + + " expressed as a percentage of the base" + + " price for the master instance types") + comp_group.add_argument( + "--min-timeout-for-request", + dest="min_timeout_for_request", + type=int, + default=1, + help="timeout for a min spot instance request, unit: minutes") + comp_group.add_argument( + "--min-spot-fallback", + dest="min_spot_fallback", + choices=["ondemand", None], + default=None, + help="whether to fallback to on-demand instances for min nodes" + + " if spot instances aren't available") + comp_group.add_argument( + "--min-spot-allocation-strategy", + dest="min_spot_allocation_strategy", + choices=["lowestPrice", "capacityOptimized", None], + default=None, + help="allocation strategy for min spot nodes") + comp_group.add_argument( + "--autoscaling-ondemand-percentage", + dest="autoscaling_ondemand_percentage", + type=int, + help="percentage of ondemand nodes in autoscaling config") + comp_group.add_argument( + "--autoscaling-spot-block-percentage", + dest="autoscaling_spot_block_percentage", + type=int, + help="percentage of spot block nodes in autoscaling config") + comp_group.add_argument( + "--autoscaling-spot-percentage", + dest="autoscaling_spot_percentage", + type=int, + help="percentage of spot nodes in autoscaling config") + comp_group.add_argument( + "--autoscaling-spot-block-duration", + dest="autoscaling_spot_block_duration", + type=int, + default=120, + help="spot block duration unit: minutes") + comp_group.add_argument( + "--autoscaling-spot-block-fallback", + dest="autoscaling_spot_block_fallback", + choices=["ondemand", None], + default=None, + help="whether to fallback to on-demand instances for autoscaling" + + " nodes if spot block instances aren't available") + comp_group.add_argument( + "--autoscaling-maximum-bid-price-percentage", + dest="autoscaling_maximum_bid_price_percentage", + type=int, + default=100, + help="maximum value to bid for autoscaling spot" + + " instances expressed as a percentage of" + + " the base price for the master instance types") + comp_group.add_argument( + "--autoscaling-timeout-for-request", + dest="autoscaling_timeout_for_request", + type=int, + default=1, + help="timeout for a autoscaling spot instance request, unit: minutes") + comp_group.add_argument( + "--autoscaling-spot-fallback", + dest="autoscaling_spot_fallback", + choices=["ondemand", None], + default=None, + help="whether to fallback to on-demand instances for autoscaling nodes" + + " if spot instances aren't available") + comp_group.add_argument( + "--autoscaling-spot-allocation-strategy", + dest="autoscaling_spot_allocation_strategy", + choices=["lowestPrice", "capacityOptimized", None], + default=None, + help="allocation strategy for autoscaling" + + " spot nodes") + def create_parser(self, argparser): # compute settings parser compute_config = argparser.add_argument_group("compute config settings") diff --git a/qds_sdk/cloud/azure_cloud.py b/qds_sdk/cloud/azure_cloud.py index 435e4993..2214dee0 100755 --- a/qds_sdk/cloud/azure_cloud.py +++ b/qds_sdk/cloud/azure_cloud.py @@ -153,6 +153,89 @@ def set_cloud_config_from_arguments(self, arguments): master_static_public_ip_name=arguments.master_static_public_ip_name, resource_group_name=arguments.resource_group_name) + def set_composition_arguments(self, comp_group): + # composition arguments we want to accept for azure + comp_group.add_argument("--min-ondemand-percentage", + dest="min_ondemand_percentage", + type=int, default=0, + help="Percentage of ondemand nodes in min config.") + comp_group.add_argument("--min-spot-percentage", + dest="min_spot_percentage", + type=int, default=0, + help="Percentage of spot nodes in min config.") + comp_group.add_argument("--max-price-percentage", + dest="max_price_percentage", + type=int, default=100, + help="Percentage of maximum price percentage" + " for spot nodes.") + comp_group.add_argument("--min-spot-fallback", + dest="min_spot_fallback", + choices=["ondemand", None], + default=None, + help="Whether to fallback to on-demand instances" + + " for min nodes if spot instances" + + " aren't available.") + comp_group.add_argument("--autoscaling-ondemand-percentage", + dest="autoscaling_ondemand_percentage", + type=int, default=0, + help="Percentage of ondemand nodes" + + "in autoscaling config.") + comp_group.add_argument("--autoscaling-spot-percentage", + dest="autoscaling_spot_percentage", + type=int, default=0, + help="Percentage of spot nodes" + + "in autoscaling config.") + comp_group.add_argument("--autoscaling-spot-fallback", + dest="autoscaling_spot_fallback", + choices=["ondemand", None], + default=None, + help="Whether to fallback to on-demand instances" + + " for autoscaling nodes if spot instances" + + " aren't available.") + + # Ignore other key-value arguments. + def get_composition(self, + min_ondemand_percentage=0, + min_spot_percentage=0, + min_spot_fallback=None, + autoscaling_ondemand_percentage=0, + autoscaling_spot_percentage=0, + autoscaling_spot_fallback=None, + max_price_percentage=100, + **kwargs): + composition = {} + composition["min_nodes"] = {"nodes": []} + min_nodes = composition["min_nodes"]["nodes"] + if min_ondemand_percentage + min_spot_percentage != 100: + raise ValueError("Minimum nodes ondemand+spot percentage" + " should be 100: Ondemand pct: %d Spot pct: %d" + % (min_ondemand_percentage, + min_spot_percentage)) + if min_ondemand_percentage > 0: + min_nodes.append({"type": "ondemand", + "percentage": min_ondemand_percentage}) + if min_spot_percentage > 0: + min_nodes.append({"type": "spot", "percentage": min_spot_percentage, + "fallback": min_spot_fallback, + "max_price_percentage": max_price_percentage}) + + composition["autoscaling_nodes"] = {"nodes": []} + autoscaling_nodes = composition["autoscaling_nodes"]["nodes"] + if autoscaling_ondemand_percentage + autoscaling_spot_percentage != 100: + raise ValueError("Autoscaling nodes ondemand+spot percentage" + + " should be 100: Ondemand pct: %d Spot pct: %d" + % (autoscaling_ondemand_percentage, + autoscaling_spot_percentage)) + if autoscaling_ondemand_percentage > 0: + autoscaling_nodes.append({"type": "ondemand", + "percentage": autoscaling_ondemand_percentage}) + if autoscaling_spot_percentage > 0: + autoscaling_nodes.append({"type": "spot", + "percentage": autoscaling_spot_percentage, + "fallback": autoscaling_spot_fallback, + "max_price_percentage": max_price_percentage}) + return composition + def create_parser(self, argparser): # compute settings parser compute_config = argparser.add_argument_group("compute config settings") @@ -228,4 +311,4 @@ def create_parser(self, argparser): storage_config.add_argument("--disk-storage-account-resource-group-name", dest="disk_storage_account_resource_group_name", default=None, - help="disk storage account resource group for azure cluster") \ No newline at end of file + help="disk storage account resource group for azure cluster") diff --git a/qds_sdk/cloud/cloud.py b/qds_sdk/cloud/cloud.py index 49053d87..31831283 100755 --- a/qds_sdk/cloud/cloud.py +++ b/qds_sdk/cloud/cloud.py @@ -5,3 +5,9 @@ def create_parser(self, argparser): def set_cloud_config_from_arguments(self, arguments): return NotImplemented + + def set_composition_arguments(self, comp_group): + pass + + def get_composition(self, **kwargs): + pass diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py index 9f946a3f..9a98ea43 100644 --- a/qds_sdk/cluster_info_v22.py +++ b/qds_sdk/cluster_info_v22.py @@ -1,6 +1,7 @@ import json from qds_sdk import util +from qds_sdk.qubole import Qubole def str2bool(v): @@ -56,29 +57,33 @@ def set_cluster_info_from_arguments(self, arguments): paused_autoscale_node_timeout_mins=arguments.paused_autoscale_node_timeout_mins, parent_cluster_id=arguments.parent_cluster_id, image_version=arguments.image_version) - - self.set_composition(master_type=arguments.master_type, - master_spot_block_duration=arguments.master_spot_block_duration, - master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage, - master_timeout_for_request=arguments.master_timeout_for_request, - master_spot_fallback=arguments.master_spot_fallback, - min_ondemand_percentage=arguments.min_ondemand_percentage, - min_spot_block_percentage=arguments.min_spot_block_percentage, - min_spot_block_duration=arguments.min_spot_block_duration, - min_spot_percentage=arguments.min_spot_percentage, - min_maximum_bid_price_percentage=arguments.min_maximum_bid_price_percentage, - min_timeout_for_request=arguments.min_timeout_for_request, - min_spot_allocation_strategy=arguments.min_spot_allocation_strategy, - min_spot_fallback=arguments.min_spot_fallback, - autoscaling_ondemand_percentage=arguments.autoscaling_ondemand_percentage, - autoscaling_spot_block_percentage=arguments.autoscaling_spot_block_percentage, - autoscaling_spot_percentage=arguments.autoscaling_spot_percentage, - autoscaling_spot_block_duration=arguments.autoscaling_spot_block_duration, - autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage, - autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request, - autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy, - autoscaling_spot_fallback=arguments.autoscaling_spot_fallback, - autoscaling_spot_block_fallback=arguments.autoscaling_spot_block_fallback) + if Qubole.get_cloud_name() == "aws": + # Need to move to aws cloud. + self.set_composition( + master_type=arguments.master_type, + master_spot_block_duration=arguments.master_spot_block_duration, + master_maximum_bid_price_percentage=arguments.master_maximum_bid_price_percentage, + master_timeout_for_request=arguments.master_timeout_for_request, + master_spot_fallback=arguments.master_spot_fallback, + min_ondemand_percentage=arguments.min_ondemand_percentage, + min_spot_block_percentage=arguments.min_spot_block_percentage, + min_spot_block_duration=arguments.min_spot_block_duration, + min_spot_percentage=arguments.min_spot_percentage, + min_maximum_bid_price_percentage=arguments.min_maximum_bid_price_percentage, + min_timeout_for_request=arguments.min_timeout_for_request, + min_spot_allocation_strategy=arguments.min_spot_allocation_strategy, + min_spot_fallback=arguments.min_spot_fallback, + autoscaling_ondemand_percentage=arguments.autoscaling_ondemand_percentage, + autoscaling_spot_block_percentage=arguments.autoscaling_spot_block_percentage, + autoscaling_spot_percentage=arguments.autoscaling_spot_percentage, + autoscaling_spot_block_duration=arguments.autoscaling_spot_block_duration, + autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage, + autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request, + autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy, + autoscaling_spot_fallback=arguments.autoscaling_spot_fallback, + autoscaling_spot_block_fallback=arguments.autoscaling_spot_block_fallback) + else: + self.set_composition_from_cloud_using_parser(arguments) def set_cluster_info(self, disallow_cluster_termination=None, @@ -228,6 +233,16 @@ def set_cluster_info(self, self.set_start_stop_settings(disable_cluster_pause, paused_cluster_timeout_mins, disable_autoscale_node_pause, paused_autoscale_node_timeout_mins) + def set_composition_from_cloud_using_parser(self, arguments): + self.set_composition_for_cluster(**{k: v for k, v in arguments.__dict__.items() + if v is not None}) + + def set_composition_for_cluster(self, **kwargs): + cloud = Qubole.get_cloud() + composition = cloud.get_composition(**kwargs) + if composition is not None: + self.cluster_info["composition"] = composition + def set_composition(self, master_type="ondemand", master_spot_block_duration=None, @@ -614,123 +629,8 @@ def cluster_info_parser(argparser, action): dest="heterogeneous_config", help="heterogeneous config for the cluster") - composition_group = argparser.add_argument_group("cluster composition settings") - composition_group.add_argument("--master-type", - dest="master_type", - choices=["ondemand", "spot", "spotblock"], - default="ondemand", - help="type of master nodes. Valid values are: ('ondemand', 'spot', 'spotblock')" + - "default: ondemand") - composition_group.add_argument("--master-spot-block-duration", - dest="master_spot_block_duration", - type=int, - default=120, - help="spot block duration unit: minutes") - composition_group.add_argument("--master-maximum-bid-price-percentage", - dest="master_maximum_bid_price_percentage", - type=int, - default=100, - help="maximum value to bid for master spot instances" + - " expressed as a percentage of the base" + - " price for the master instance types") - composition_group.add_argument("--master-timeout-for-request", - dest="master_timeout_for_request", - type=int, - default=1, - help="timeout for a master spot instance request, unit: minutes") - composition_group.add_argument("--master-spot-fallback", - dest="master_spot_fallback", - choices=["ondemand", None], - default=None, - help="whether to fallback to on-demand instances for master nodes" + - " if spot instances aren't available") - composition_group.add_argument("--min-ondemand-percentage", - dest="min_ondemand_percentage", - type=int, - help="percentage of ondemand nodes in min config") - composition_group.add_argument("--min-spot-block-percentage", - dest="min_spot_block_percentage", - type=int, - help="percentage of spot block nodes in min config") - composition_group.add_argument("--min-spot-percentage", - dest="min_spot_percentage", - type=int, - help="percentage of spot nodes in min config") - composition_group.add_argument("--min-spot-block-duration", - dest="min_spot_block_duration", - type=int, - default=120, - help="spot block duration unit: minutes") - composition_group.add_argument("--min-maximum-bid-price-percentage", - dest="min_maximum_bid_price_percentage", - type=int, - default=100, - help="maximum value to bid for min spot instances" + - " expressed as a percentage of the base" + - " price for the master instance types") - composition_group.add_argument("--min-timeout-for-request", - dest="min_timeout_for_request", - type=int, - default=1, - help="timeout for a min spot instance request, unit: minutes") - composition_group.add_argument("--min-spot-fallback", - dest="min_spot_fallback", - choices=["ondemand", None], - default=None, - help="whether to fallback to on-demand instances for min nodes" + - " if spot instances aren't available") - composition_group.add_argument("--min-spot-allocation-strategy", - dest="min_spot_allocation_strategy", - choices=["lowestPrice", "capacityOptimized", None], - default=None, - help="allocation strategy for min spot nodes") - composition_group.add_argument("--autoscaling-ondemand-percentage", - dest="autoscaling_ondemand_percentage", - type=int, - help="percentage of ondemand nodes in autoscaling config") - composition_group.add_argument("--autoscaling-spot-block-percentage", - dest="autoscaling_spot_block_percentage", - type=int, - help="percentage of spot block nodes in autoscaling config") - composition_group.add_argument("--autoscaling-spot-percentage", - dest="autoscaling_spot_percentage", - type=int, - help="percentage of spot nodes in autoscaling config") - composition_group.add_argument("--autoscaling-spot-block-duration", - dest="autoscaling_spot_block_duration", - type=int, - default=120, - help="spot block duration unit: minutes") - composition_group.add_argument("--autoscaling-spot-block-fallback", - dest="autoscaling_spot_block_fallback", - choices=["ondemand", None], - default=None, - help="whether to fallback to on-demand instances for autoscaling" + - " nodes if spot block instances aren't available") - composition_group.add_argument("--autoscaling-maximum-bid-price-percentage", - dest="autoscaling_maximum_bid_price_percentage", - type=int, - default=100, - help="maximum value to bid for autoscaling spot instances" + - " expressed as a percentage of the base" + - " price for the master instance types") - composition_group.add_argument("--autoscaling-timeout-for-request", - dest="autoscaling_timeout_for_request", - type=int, - default=1, - help="timeout for a autoscaling spot instance request, unit: minutes") - composition_group.add_argument("--autoscaling-spot-fallback", - dest="autoscaling_spot_fallback", - choices=["ondemand", None], - default=None, - help="whether to fallback to on-demand instances for autoscaling nodes" + - " if spot instances aren't available") - composition_group.add_argument("--autoscaling-spot-allocation-strategy", - dest="autoscaling_spot_allocation_strategy", - choices=["lowestPrice", "capacityOptimized", None], - default=None, - help="allocation strategy for autoscaling" + - " spot nodes") + composition_group = argparser.add_argument_group("Cluster composition settings") + Qubole.get_cloud().set_composition_arguments(composition_group) # monitoring settings monitoring_group = argparser.add_argument_group("monitoring settings") diff --git a/qds_sdk/qubole.py b/qds_sdk/qubole.py index 659516af..58b40304 100644 --- a/qds_sdk/qubole.py +++ b/qds_sdk/qubole.py @@ -143,3 +143,7 @@ def get_cloud_object(cls, cloud_name): elif cloud_name.lower() == "gcp": import qds_sdk.cloud.gcp_cloud return qds_sdk.cloud.gcp_cloud.GcpCloud() + + @classmethod + def get_cloud_name(cls): + return Qubole.cloud_name diff --git a/tests/test_clusterv22.py b/tests/test_clusterv22.py index 928ace03..2b10fc0a 100644 --- a/tests/test_clusterv22.py +++ b/tests/test_clusterv22.py @@ -227,6 +227,61 @@ def test_hive_settings(self): 'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': {'nodes': [{'percentage': 50, 'type': 'ondemand'}, {'percentage': 50, 'type': 'spot', 'maximum_bid_price_percentage': 100, 'timeout_for_request': 1, 'allocation_strategy': None, 'fallback': 'ondemand'}]}}}}) +class TestAzureClusterComposition(QdsCliTestCase): + def test_od_od(self): + sys.argv = ['qds.py', '--version', 'v2.2', '--cloud','AZURE', 'cluster', + 'create', '--label', 'test_label', + '--min-ondemand-percentage', '100', + '--autoscaling-ondemand-percentage', '100'] + Qubole.cloud = None + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { + 'composition': {'min_nodes': {'nodes': [{'type': 'ondemand', 'percentage': 100}]}, + 'autoscaling_nodes': {'nodes': [{'type': 'ondemand', 'percentage': 100}]}}, + 'label': ['test_label']}}) + + def test_spot_spot(self): + sys.argv = ['qds.py', '--version', 'v2.2', '--cloud','AZURE', 'cluster', 'create', + '--label', 'test_label', + '--min-spot-percentage', '100', '--min-spot-fallback', 'ondemand', + '--max-price-percentage', '50', + '--autoscaling-spot-percentage', '100', "--autoscaling-spot-fallback", 'ondemand'] + Qubole.cloud = None + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { + 'composition': { + 'min_nodes': {'nodes': [{'percentage': 100, 'type': 'spot', + 'fallback': "ondemand", 'max_price_percentage': 50}]}, + 'autoscaling_nodes': {'nodes': [{'percentage': 100, 'type': 'spot', + 'fallback': "ondemand", 'max_price_percentage': 50}]}}, + 'label': ['test_label']}}) + + + def test_od_spot(self): + sys.argv = ['qds.py', '--version', 'v2.2', '--cloud','AZURE', 'cluster', 'create', + '--label', 'test_label', + '--min-ondemand-percentage', '50', + '--min-spot-percentage', '50', '--min-spot-fallback', 'ondemand', + '--max-price-percentage', '50', + '--autoscaling-ondemand-percentage', '50', + '--autoscaling-spot-percentage', '50', '--autoscaling-spot-fallback', 'ondemand'] + Qubole.cloud = None + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': { + 'composition': { + 'min_nodes': {'nodes': [{'type': 'ondemand', 'percentage': 50}, + {'percentage': 50, 'type': 'spot', + 'fallback': "ondemand", 'max_price_percentage': 50}]}, + 'autoscaling_nodes': {'nodes': [{'type': 'ondemand', 'percentage': 50}, + {'percentage': 50, 'type': 'spot', + 'fallback': "ondemand", 'max_price_percentage': 50}]}}, + 'label': ['test_label']}}) if __name__ == '__main__': unittest.main()