From 35ea401ae4d22ed6d61a94111b52058e10aef7e3 Mon Sep 17 00:00:00 2001 From: Swapnil Kumar Date: Wed, 19 Feb 2020 15:13:07 +0530 Subject: [PATCH] SDK-388: Support for spot allocation strategy on Cluster (#293) --- qds_sdk/cluster_info_v22.py | 44 ++++++++++++++++++++++++++++++------- tests/test_clusterv22.py | 32 +++++++++++++-------------- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/qds_sdk/cluster_info_v22.py b/qds_sdk/cluster_info_v22.py index 32aa52d8..43e699f6 100644 --- a/qds_sdk/cluster_info_v22.py +++ b/qds_sdk/cluster_info_v22.py @@ -68,6 +68,7 @@ def set_cluster_info_from_arguments(self, arguments): min_spot_percentage=arguments.min_spot_percentage, min_maximum_bid_price_percentage=arguments.min_maximum_bid_price_percentage, min_timeout_for_request=arguments.min_timeout_for_request, + min_spot_allocation_strategy=arguments.min_spot_allocation_strategy, min_spot_fallback=arguments.min_spot_fallback, autoscaling_ondemand_percentage=arguments.autoscaling_ondemand_percentage, autoscaling_spot_block_percentage=arguments.autoscaling_spot_block_percentage, @@ -75,6 +76,7 @@ def set_cluster_info_from_arguments(self, arguments): autoscaling_spot_block_duration=arguments.autoscaling_spot_block_duration, autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage, autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request, + autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy, autoscaling_spot_fallback=arguments.autoscaling_spot_fallback) def set_cluster_info(self, @@ -237,6 +239,7 @@ def set_composition(self, min_spot_percentage=None, min_maximum_bid_price_percentage=None, min_timeout_for_request=None, + min_spot_allocation_strategy=None, min_spot_fallback=None, autoscaling_ondemand_percentage=None, autoscaling_spot_block_percentage=None, @@ -244,6 +247,7 @@ def set_composition(self, autoscaling_spot_block_duration=None, autoscaling_maximum_bid_price_percentage=None, autoscaling_timeout_for_request=None, + autoscaling_spot_allocation_strategy=None, autoscaling_spot_fallback=None): self.cluster_info["composition"] = {} @@ -260,6 +264,7 @@ def set_composition(self, min_spot_percentage, min_maximum_bid_price_percentage, min_timeout_for_request, + min_spot_allocation_strategy, min_spot_fallback) self.set_autoscaling_config(autoscaling_ondemand_percentage, @@ -268,6 +273,7 @@ def set_composition(self, autoscaling_spot_percentage, autoscaling_maximum_bid_price_percentage, autoscaling_timeout_for_request, + autoscaling_spot_allocation_strategy, autoscaling_spot_fallback) def set_master_config(self, @@ -293,6 +299,7 @@ def set_min_config(self, min_spot_percentage, min_maximum_bid_price_percentage, min_timeout_for_request, + min_spot_allocation_strategy, min_spot_fallback): self.cluster_info["composition"]["min_nodes"] = {"nodes": []} if not min_ondemand_percentage and not min_spot_block_percentage and not min_spot_percentage: @@ -305,7 +312,8 @@ def set_min_config(self, min_spot_block_percentage, min_spot_block_duration) if min_spot_percentage: self.set_min_spot(min_spot_percentage, min_maximum_bid_price_percentage, - min_timeout_for_request, min_spot_fallback) + min_timeout_for_request, min_spot_allocation_strategy, + min_spot_fallback) def set_autoscaling_config(self, autoscaling_ondemand_percentage, @@ -314,11 +322,12 @@ def set_autoscaling_config(self, autoscaling_spot_percentage, autoscaling_maximum_bid_price_percentage, autoscaling_timeout_for_request, + autoscaling_spot_allocation_strategy, autoscaling_spot_fallback): self.cluster_info["composition"]["autoscaling_nodes"] = {"nodes": []} if not autoscaling_ondemand_percentage and not autoscaling_spot_block_percentage and not autoscaling_spot_percentage: self.set_autoscaling_ondemand(50) - self.set_autoscaling_spot(50, 100, 1, 'ondemand') + self.set_autoscaling_spot(50, 100, 1, None, 'ondemand') else: if autoscaling_ondemand_percentage: self.set_autoscaling_ondemand(autoscaling_ondemand_percentage) @@ -326,8 +335,11 @@ def set_autoscaling_config(self, self.set_autoscaling_spot_block(autoscaling_spot_block_percentage, autoscaling_spot_block_duration) if autoscaling_spot_percentage: - self.set_autoscaling_spot(autoscaling_spot_percentage, autoscaling_maximum_bid_price_percentage, - autoscaling_timeout_for_request, autoscaling_spot_fallback) + self.set_autoscaling_spot(autoscaling_spot_percentage, + autoscaling_maximum_bid_price_percentage, + autoscaling_timeout_for_request, + autoscaling_spot_allocation_strategy, + autoscaling_spot_fallback) def set_master_ondemand(self, master_ondemand_percentage=None): ondemand = {"percentage": master_ondemand_percentage, "type": "ondemand"} @@ -360,11 +372,13 @@ def set_min_spot_block(self, min_spot_block_percentage=None, min_spot_block_dura self.cluster_info["composition"]["min_nodes"]["nodes"].append(spot_block) def set_min_spot(self, min_spot_percentage=None, min_maximum_bid_price_percentage=100, - min_timeout_for_request=1, min_spot_fallback=None): + min_timeout_for_request=1, min_spot_allocation_strategy=None, + min_spot_fallback=None): spot = {"percentage": min_spot_percentage, "type": "spot", "maximum_bid_price_percentage": min_maximum_bid_price_percentage, "timeout_for_request": min_timeout_for_request, + "allocation_strategy": min_spot_allocation_strategy, "fallback": min_spot_fallback } self.cluster_info["composition"]["min_nodes"]["nodes"].append(spot) @@ -380,12 +394,16 @@ def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, aut "timeout": autoscaling_spot_block_duration} self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(spot_block) - def set_autoscaling_spot(self, autoscaling_spot_percentage=None, autoscaling_maximum_bid_price_percentage=100, - autoscaling_timeout_for_request=1, autoscaling_spot_fallback=None): + def set_autoscaling_spot(self, autoscaling_spot_percentage=None, + autoscaling_maximum_bid_price_percentage=100, + autoscaling_timeout_for_request=1, + autoscaling_spot_allocation_strategy=None, + autoscaling_spot_fallback=None): spot = {"percentage": autoscaling_spot_percentage, "type": "spot", "maximum_bid_price_percentage": autoscaling_maximum_bid_price_percentage, "timeout_for_request": autoscaling_timeout_for_request, + "allocation_strategy": autoscaling_spot_allocation_strategy, "fallback": autoscaling_spot_fallback } self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(spot) @@ -653,7 +671,11 @@ def cluster_info_parser(argparser, action): default=None, help="whether to fallback to on-demand instances for min nodes" + " if spot instances aren't available") - + composition_group.add_argument("--min-spot-allocation-strategy", + dest="min_spot_allocation_strategy", + choices=["lowestPrice", "capacityOptimized", None], + default=None, + help="allocation strategy for min spot nodes") composition_group.add_argument("--autoscaling-ondemand-percentage", dest="autoscaling_ondemand_percentage", type=int, @@ -689,6 +711,12 @@ def cluster_info_parser(argparser, action): default=None, help="whether to fallback to on-demand instances for autoscaling nodes" + " if spot instances aren't available") + composition_group.add_argument("--autoscaling-spot-allocation-strategy", + dest="autoscaling_spot_allocation_strategy", + choices=["lowestPrice", "capacityOptimized", None], + default=None, + help="allocation strategy for autoscaling" + + " spot nodes") # monitoring settings monitoring_group = argparser.add_argument_group("monitoring settings") diff --git a/tests/test_clusterv22.py b/tests/test_clusterv22.py index 5b1e4ee7..ab4c03ec 100644 --- a/tests/test_clusterv22.py +++ b/tests/test_clusterv22.py @@ -42,7 +42,7 @@ def test_cluster_info(self): {'timeout_for_request': 1, 'percentage': 50, 'type': 'spot', 'fallback': 'ondemand', - 'maximum_bid_price_percentage': 100}]}}, + 'maximum_bid_price_percentage': 100, 'allocation_strategy': None}]}}, 'datadisk': {'encryption': True}}}) def test_od_od_od(self): @@ -64,7 +64,7 @@ def test_od_od_odspot(self): '--master-type', 'ondemand', '--min-ondemand-percentage', '100', '--autoscaling-ondemand-percentage', '50', '--autoscaling-spot-percentage', '50', '--autoscaling-maximum-bid-price-percentage', '50', - '--autoscaling-timeout-for-request', '3', '--autoscaling-spot-fallback', 'ondemand'] + '--autoscaling-timeout-for-request', '3', '--autoscaling-spot-fallback', 'ondemand', '--autoscaling-spot-allocation-strategy', 'lowestPrice'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) @@ -74,14 +74,14 @@ def test_od_od_odspot(self): 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': { 'nodes': [{'percentage': 50, 'type': 'ondemand'}, {'timeout_for_request': 3, 'percentage': 50, 'type': 'spot', 'fallback': 'ondemand', - 'maximum_bid_price_percentage': 50}]}}, 'label': ['test_label']}}) + 'maximum_bid_price_percentage': 50, 'allocation_strategy': 'lowestPrice'}]}}, 'label': ['test_label']}}) def test_od_od_odspot_nofallback(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--master-type', 'ondemand', '--min-ondemand-percentage', '100', '--autoscaling-ondemand-percentage', '50', '--autoscaling-spot-percentage', '50', '--autoscaling-maximum-bid-price-percentage', '50', - '--autoscaling-timeout-for-request', '3', '--autoscaling-spot-fallback', None] + '--autoscaling-timeout-for-request', '3', '--autoscaling-spot-fallback', None, '--autoscaling-spot-allocation-strategy', 'capacityOptimized'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) @@ -91,7 +91,7 @@ def test_od_od_odspot_nofallback(self): 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': { 'nodes': [{'percentage': 50, 'type': 'ondemand'}, {'timeout_for_request': 3, 'percentage': 50, 'type': 'spot', 'fallback': None, - 'maximum_bid_price_percentage': 50}]}}, 'label': ['test_label']}}) + 'maximum_bid_price_percentage': 50, 'allocation_strategy': 'capacityOptimized'}]}}, 'label': ['test_label']}}) def test_od_od_spotblock(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', @@ -114,7 +114,7 @@ def test_od_od_spotblockspot(self): '--autoscaling-spot-block-percentage', '50', '--autoscaling-spot-block-duration', '60', '--autoscaling-spot-percentage', '50', '--autoscaling-maximum-bid-price-percentage', '50', - '--autoscaling-timeout-for-request', '3', '--autoscaling-spot-fallback', None] + '--autoscaling-timeout-for-request', '3', '--autoscaling-spot-fallback', None, '--autoscaling-spot-allocation-strategy', 'capacityOptimized'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) @@ -124,14 +124,14 @@ def test_od_od_spotblockspot(self): 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': { 'nodes': [{'percentage': 50, 'type': 'spotblock', 'timeout': 60}, {'timeout_for_request': 3, 'percentage': 50, 'type': 'spot', 'fallback': None, - 'maximum_bid_price_percentage': 50}]}}, 'label': ['test_label']}}) + 'maximum_bid_price_percentage': 50, 'allocation_strategy': 'capacityOptimized'}]}}, 'label': ['test_label']}}) def test_od_od_spot(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', '--master-type', 'ondemand', '--min-ondemand-percentage', '100', '--autoscaling-spot-percentage', '100', '--autoscaling-maximum-bid-price-percentage', '50', '--autoscaling-timeout-for-request', '3', - '--autoscaling-spot-fallback', None] + '--autoscaling-spot-fallback', None, '--autoscaling-spot-allocation-strategy', 'lowestPrice'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) @@ -140,7 +140,7 @@ def test_od_od_spot(self): 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': { 'nodes': [{'timeout_for_request': 3, 'percentage': 100, 'type': 'spot', 'fallback': None, - 'maximum_bid_price_percentage': 50}]}}, 'label': ['test_label']}}) + 'maximum_bid_price_percentage': 50, 'allocation_strategy': 'lowestPrice'}]}}, 'label': ['test_label']}}) def test_od_spot_spot(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', @@ -148,17 +148,17 @@ def test_od_spot_spot(self): '--min-maximum-bid-price-percentage', '50', '--min-timeout-for-request', '3', '--min-spot-fallback', None, '--autoscaling-spot-percentage', '100', '--autoscaling-maximum-bid-price-percentage', '50', '--autoscaling-timeout-for-request', '3', - '--autoscaling-spot-fallback', None] + '--autoscaling-spot-fallback', None, '--autoscaling-spot-allocation-strategy', 'capacityOptimized', '--min-spot-allocation-strategy', 'lowestPrice'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': {'composition': {'min_nodes': { 'nodes': [{'timeout_for_request': 3, 'percentage': 100, 'type': 'spot', 'fallback': None, - 'maximum_bid_price_percentage': 50}]}, 'master': { + 'maximum_bid_price_percentage': 50, 'allocation_strategy': 'lowestPrice'}]}, 'master': { 'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': {'nodes': [ {'timeout_for_request': 3, 'percentage': 100, 'type': 'spot', 'fallback': None, - 'maximum_bid_price_percentage': 50}]}}, 'label': ['test_label']}}) + 'maximum_bid_price_percentage': 50, 'allocation_strategy': 'capacityOptimized'}]}}, 'label': ['test_label']}}) def test_spotblock_spotblock_spotblock(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', 'test_label', @@ -183,18 +183,18 @@ def test_spot_spot_spot(self): '--min-maximum-bid-price-percentage', '50', '--min-timeout-for-request', '3', '--min-spot-fallback', None, '--autoscaling-spot-percentage', '100', '--autoscaling-maximum-bid-price-percentage', '50', '--autoscaling-timeout-for-request', '3', - '--autoscaling-spot-fallback', None] + '--autoscaling-spot-fallback', None, '--autoscaling-spot-allocation-strategy', 'lowestPrice', '--min-spot-allocation-strategy', 'capacityOptimized'] Qubole.cloud = None print_command() Connection._api_call = Mock(return_value={}) qds.main() Connection._api_call.assert_called_with('POST', 'clusters', {'cluster_info': {'composition': {'min_nodes': { 'nodes': [{'timeout_for_request': 3, 'percentage': 100, 'type': 'spot', 'fallback': None, - 'maximum_bid_price_percentage': 50}]}, 'master': {'nodes': [ + 'maximum_bid_price_percentage': 50, 'allocation_strategy': 'capacityOptimized'}]}, 'master': {'nodes': [ {'timeout_for_request': 3, 'percentage': 100, 'type': 'spot', 'fallback': None, 'maximum_bid_price_percentage': 50}]}, 'autoscaling_nodes': {'nodes': [ {'timeout_for_request': 3, 'percentage': 100, 'type': 'spot', 'fallback': None, - 'maximum_bid_price_percentage': 50}]}}, 'label': ['test_label']}}) + 'maximum_bid_price_percentage': 50, 'allocation_strategy': 'lowestPrice'}]}}, 'label': ['test_label']}}) def test_image_version_v22(self): sys.argv = ['qds.py', '--version', 'v2.2', 'cluster', 'create', '--label', @@ -210,4 +210,4 @@ def test_image_version_v22(self): 'min_nodes': 3, 'slave_instance_type': 'c1.xlarge', 'cluster_image_version': '1.latest', - 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': {'nodes': [{'percentage': 50, 'type': 'ondemand'}, {'timeout_for_request': 1, 'percentage': 50, 'type': 'spot', 'fallback': 'ondemand', 'maximum_bid_price_percentage': 100}]}}, 'label': ['test_label']}}) + 'composition': {'min_nodes': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'master': {'nodes': [{'percentage': 100, 'type': 'ondemand'}]}, 'autoscaling_nodes': {'nodes': [{'percentage': 50, 'type': 'ondemand'}, {'timeout_for_request': 1, 'percentage': 50, 'type': 'spot', 'fallback': 'ondemand', 'maximum_bid_price_percentage': 100, 'allocation_strategy': None}]}}, 'label': ['test_label']}})