Skip to content

Commit

Permalink
SDK-388: Support for spot allocation strategy on Cluster (#293)
Browse files Browse the repository at this point in the history
  • Loading branch information
@kswap
kswap authored Feb 19, 2020
1 parent 75129d8 commit 35ea401
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 24 deletions.
44 changes: 36 additions & 8 deletions qds_sdk/cluster_info_v22.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,15 @@ def set_cluster_info_from_arguments(self, arguments):
min_spot_percentage=arguments.min_spot_percentage,
min_maximum_bid_price_percentage=arguments.min_maximum_bid_price_percentage,
min_timeout_for_request=arguments.min_timeout_for_request,
min_spot_allocation_strategy=arguments.min_spot_allocation_strategy,
min_spot_fallback=arguments.min_spot_fallback,
autoscaling_ondemand_percentage=arguments.autoscaling_ondemand_percentage,
autoscaling_spot_block_percentage=arguments.autoscaling_spot_block_percentage,
autoscaling_spot_percentage=arguments.autoscaling_spot_percentage,
autoscaling_spot_block_duration=arguments.autoscaling_spot_block_duration,
autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage,
autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request,
autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy,
autoscaling_spot_fallback=arguments.autoscaling_spot_fallback)

def set_cluster_info(self,
Expand Down Expand Up @@ -237,13 +239,15 @@ def set_composition(self,
min_spot_percentage=None,
min_maximum_bid_price_percentage=None,
min_timeout_for_request=None,
min_spot_allocation_strategy=None,
min_spot_fallback=None,
autoscaling_ondemand_percentage=None,
autoscaling_spot_block_percentage=None,
autoscaling_spot_percentage=None,
autoscaling_spot_block_duration=None,
autoscaling_maximum_bid_price_percentage=None,
autoscaling_timeout_for_request=None,
autoscaling_spot_allocation_strategy=None,
autoscaling_spot_fallback=None):

self.cluster_info["composition"] = {}
Expand All @@ -260,6 +264,7 @@ def set_composition(self,
min_spot_percentage,
min_maximum_bid_price_percentage,
min_timeout_for_request,
min_spot_allocation_strategy,
min_spot_fallback)

self.set_autoscaling_config(autoscaling_ondemand_percentage,
Expand All @@ -268,6 +273,7 @@ def set_composition(self,
autoscaling_spot_percentage,
autoscaling_maximum_bid_price_percentage,
autoscaling_timeout_for_request,
autoscaling_spot_allocation_strategy,
autoscaling_spot_fallback)

def set_master_config(self,
Expand All @@ -293,6 +299,7 @@ def set_min_config(self,
min_spot_percentage,
min_maximum_bid_price_percentage,
min_timeout_for_request,
min_spot_allocation_strategy,
min_spot_fallback):
self.cluster_info["composition"]["min_nodes"] = {"nodes": []}
if not min_ondemand_percentage and not min_spot_block_percentage and not min_spot_percentage:
Expand All @@ -305,7 +312,8 @@ def set_min_config(self,
min_spot_block_percentage, min_spot_block_duration)
if min_spot_percentage:
self.set_min_spot(min_spot_percentage, min_maximum_bid_price_percentage,
min_timeout_for_request, min_spot_fallback)
min_timeout_for_request, min_spot_allocation_strategy,
min_spot_fallback)

def set_autoscaling_config(self,
autoscaling_ondemand_percentage,
Expand All @@ -314,20 +322,24 @@ def set_autoscaling_config(self,
autoscaling_spot_percentage,
autoscaling_maximum_bid_price_percentage,
autoscaling_timeout_for_request,
autoscaling_spot_allocation_strategy,
autoscaling_spot_fallback):
self.cluster_info["composition"]["autoscaling_nodes"] = {"nodes": []}
if not autoscaling_ondemand_percentage and not autoscaling_spot_block_percentage and not autoscaling_spot_percentage:
self.set_autoscaling_ondemand(50)
self.set_autoscaling_spot(50, 100, 1, 'ondemand')
self.set_autoscaling_spot(50, 100, 1, None, 'ondemand')
else:
if autoscaling_ondemand_percentage:
self.set_autoscaling_ondemand(autoscaling_ondemand_percentage)
if autoscaling_spot_block_percentage:
self.set_autoscaling_spot_block(autoscaling_spot_block_percentage,
autoscaling_spot_block_duration)
if autoscaling_spot_percentage:
self.set_autoscaling_spot(autoscaling_spot_percentage, autoscaling_maximum_bid_price_percentage,
autoscaling_timeout_for_request, autoscaling_spot_fallback)
self.set_autoscaling_spot(autoscaling_spot_percentage,
autoscaling_maximum_bid_price_percentage,
autoscaling_timeout_for_request,
autoscaling_spot_allocation_strategy,
autoscaling_spot_fallback)

def set_master_ondemand(self, master_ondemand_percentage=None):
ondemand = {"percentage": master_ondemand_percentage, "type": "ondemand"}
Expand Down Expand Up @@ -360,11 +372,13 @@ def set_min_spot_block(self, min_spot_block_percentage=None, min_spot_block_dura
self.cluster_info["composition"]["min_nodes"]["nodes"].append(spot_block)

def set_min_spot(self, min_spot_percentage=None, min_maximum_bid_price_percentage=100,
min_timeout_for_request=1, min_spot_fallback=None):
min_timeout_for_request=1, min_spot_allocation_strategy=None,
min_spot_fallback=None):
spot = {"percentage": min_spot_percentage,
"type": "spot",
"maximum_bid_price_percentage": min_maximum_bid_price_percentage,
"timeout_for_request": min_timeout_for_request,
"allocation_strategy": min_spot_allocation_strategy,
"fallback": min_spot_fallback
}
self.cluster_info["composition"]["min_nodes"]["nodes"].append(spot)
Expand All @@ -380,12 +394,16 @@ def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, aut
"timeout": autoscaling_spot_block_duration}
self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(spot_block)

def set_autoscaling_spot(self, autoscaling_spot_percentage=None, autoscaling_maximum_bid_price_percentage=100,
autoscaling_timeout_for_request=1, autoscaling_spot_fallback=None):
def set_autoscaling_spot(self, autoscaling_spot_percentage=None,
autoscaling_maximum_bid_price_percentage=100,
autoscaling_timeout_for_request=1,
autoscaling_spot_allocation_strategy=None,
autoscaling_spot_fallback=None):
spot = {"percentage": autoscaling_spot_percentage,
"type": "spot",
"maximum_bid_price_percentage": autoscaling_maximum_bid_price_percentage,
"timeout_for_request": autoscaling_timeout_for_request,
"allocation_strategy": autoscaling_spot_allocation_strategy,
"fallback": autoscaling_spot_fallback
}
self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(spot)
Expand Down Expand Up @@ -653,7 +671,11 @@ def cluster_info_parser(argparser, action):
default=None,
help="whether to fallback to on-demand instances for min nodes" +
" if spot instances aren't available")

composition_group.add_argument("--min-spot-allocation-strategy",
dest="min_spot_allocation_strategy",
choices=["lowestPrice", "capacityOptimized", None],
default=None,
help="allocation strategy for min spot nodes")
composition_group.add_argument("--autoscaling-ondemand-percentage",
dest="autoscaling_ondemand_percentage",
type=int,
Expand Down Expand Up @@ -689,6 +711,12 @@ def cluster_info_parser(argparser, action):
default=None,
help="whether to fallback to on-demand instances for autoscaling nodes" +
" if spot instances aren't available")
composition_group.add_argument("--autoscaling-spot-allocation-strategy",
dest="autoscaling_spot_allocation_strategy",
choices=["lowestPrice", "capacityOptimized", None],
default=None,
help="allocation strategy for autoscaling" +
" spot nodes")

# monitoring settings
monitoring_group = argparser.add_argument_group("monitoring settings")
Expand Down
Loading

0 comments on commit 35ea401

Please sign in to comment.