Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SDK-388: Adding support for spot allocation strategy #293

Merged
merged 13 commits into from
Feb 19, 2020
Merged

SDK-388: Adding support for spot allocation strategy #293

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
3721977
Merge pull request #2 from qubole/unreleased
kswap Dec 13, 2019
2db59f3
Adding support for spot allocation strategy
Dec 13, 2019
6c43a99
Adding change
Dec 13, 2019
745ab01
Adding tests for spot allocation startegy
Dec 13, 2019
2ececdf
Adding test change
Dec 13, 2019
644c2da
Changing cost_optimized to lowestPrice
Dec 14, 2019
c3f3242
Merge branch 'unreleased' into qds-sdk-allocation-strategy
kswap Jan 31, 2020
ed831b7
Merge branch 'unreleased' into qds-sdk-allocation-strategy
kswap Feb 19, 2020
44b96d4
Adding style fix
Feb 19, 2020
1e885e9
Merge branch 'qds-sdk-allocation-strategy' of github.com:kswap/qds-sd…
Feb 19, 2020
83a3771
Adding style fix
Feb 19, 2020
0d2b638
Adding style fix
Feb 19, 2020
4432b8c
Adding change
Feb 19, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 36 additions & 8 deletions qds_sdk/cluster_info_v22.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,15 @@ def set_cluster_info_from_arguments(self, arguments):
min_spot_percentage=arguments.min_spot_percentage,
min_maximum_bid_price_percentage=arguments.min_maximum_bid_price_percentage,
min_timeout_for_request=arguments.min_timeout_for_request,
min_spot_allocation_strategy=arguments.min_spot_allocation_strategy,
min_spot_fallback=arguments.min_spot_fallback,
autoscaling_ondemand_percentage=arguments.autoscaling_ondemand_percentage,
autoscaling_spot_block_percentage=arguments.autoscaling_spot_block_percentage,
autoscaling_spot_percentage=arguments.autoscaling_spot_percentage,
autoscaling_spot_block_duration=arguments.autoscaling_spot_block_duration,
autoscaling_maximum_bid_price_percentage=arguments.autoscaling_maximum_bid_price_percentage,
autoscaling_timeout_for_request=arguments.autoscaling_timeout_for_request,
autoscaling_spot_allocation_strategy=arguments.autoscaling_spot_allocation_strategy,
autoscaling_spot_fallback=arguments.autoscaling_spot_fallback)

def set_cluster_info(self,
Expand Down Expand Up @@ -237,13 +239,15 @@ def set_composition(self,
min_spot_percentage=None,
min_maximum_bid_price_percentage=None,
min_timeout_for_request=None,
min_spot_allocation_strategy=None,
min_spot_fallback=None,
autoscaling_ondemand_percentage=None,
autoscaling_spot_block_percentage=None,
autoscaling_spot_percentage=None,
autoscaling_spot_block_duration=None,
autoscaling_maximum_bid_price_percentage=None,
autoscaling_timeout_for_request=None,
autoscaling_spot_allocation_strategy=None,
autoscaling_spot_fallback=None):

self.cluster_info["composition"] = {}
Expand All @@ -260,6 +264,7 @@ def set_composition(self,
min_spot_percentage,
min_maximum_bid_price_percentage,
min_timeout_for_request,
min_spot_allocation_strategy,
min_spot_fallback)

self.set_autoscaling_config(autoscaling_ondemand_percentage,
Expand All @@ -268,6 +273,7 @@ def set_composition(self,
autoscaling_spot_percentage,
autoscaling_maximum_bid_price_percentage,
autoscaling_timeout_for_request,
autoscaling_spot_allocation_strategy,
autoscaling_spot_fallback)

def set_master_config(self,
Expand All @@ -293,6 +299,7 @@ def set_min_config(self,
min_spot_percentage,
min_maximum_bid_price_percentage,
min_timeout_for_request,
min_spot_allocation_strategy,
min_spot_fallback):
self.cluster_info["composition"]["min_nodes"] = {"nodes": []}
if not min_ondemand_percentage and not min_spot_block_percentage and not min_spot_percentage:
Expand All @@ -305,7 +312,8 @@ def set_min_config(self,
min_spot_block_percentage, min_spot_block_duration)
if min_spot_percentage:
self.set_min_spot(min_spot_percentage, min_maximum_bid_price_percentage,
min_timeout_for_request, min_spot_fallback)
min_timeout_for_request, min_spot_allocation_strategy,
min_spot_fallback)

def set_autoscaling_config(self,
autoscaling_ondemand_percentage,
Expand All @@ -314,20 +322,24 @@ def set_autoscaling_config(self,
autoscaling_spot_percentage,
autoscaling_maximum_bid_price_percentage,
autoscaling_timeout_for_request,
autoscaling_spot_allocation_strategy,
autoscaling_spot_fallback):
self.cluster_info["composition"]["autoscaling_nodes"] = {"nodes": []}
if not autoscaling_ondemand_percentage and not autoscaling_spot_block_percentage and not autoscaling_spot_percentage:
self.set_autoscaling_ondemand(50)
self.set_autoscaling_spot(50, 100, 1, 'ondemand')
self.set_autoscaling_spot(50, 100, 1, None, 'ondemand')
else:
if autoscaling_ondemand_percentage:
self.set_autoscaling_ondemand(autoscaling_ondemand_percentage)
if autoscaling_spot_block_percentage:
self.set_autoscaling_spot_block(autoscaling_spot_block_percentage,
autoscaling_spot_block_duration)
if autoscaling_spot_percentage:
self.set_autoscaling_spot(autoscaling_spot_percentage, autoscaling_maximum_bid_price_percentage,
autoscaling_timeout_for_request, autoscaling_spot_fallback)
self.set_autoscaling_spot(autoscaling_spot_percentage,
autoscaling_maximum_bid_price_percentage,
autoscaling_timeout_for_request,
autoscaling_spot_allocation_strategy,
autoscaling_spot_fallback)

def set_master_ondemand(self, master_ondemand_percentage=None):
ondemand = {"percentage": master_ondemand_percentage, "type": "ondemand"}
Expand Down Expand Up @@ -360,11 +372,13 @@ def set_min_spot_block(self, min_spot_block_percentage=None, min_spot_block_dura
self.cluster_info["composition"]["min_nodes"]["nodes"].append(spot_block)

def set_min_spot(self, min_spot_percentage=None, min_maximum_bid_price_percentage=100,
min_timeout_for_request=1, min_spot_fallback=None):
min_timeout_for_request=1, min_spot_allocation_strategy=None,
min_spot_fallback=None):
spot = {"percentage": min_spot_percentage,
"type": "spot",
"maximum_bid_price_percentage": min_maximum_bid_price_percentage,
"timeout_for_request": min_timeout_for_request,
"allocation_strategy": min_spot_allocation_strategy,
"fallback": min_spot_fallback
}
self.cluster_info["composition"]["min_nodes"]["nodes"].append(spot)
Expand All @@ -380,12 +394,16 @@ def set_autoscaling_spot_block(self, autoscaling_spot_block_percentage=None, aut
"timeout": autoscaling_spot_block_duration}
self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(spot_block)

def set_autoscaling_spot(self, autoscaling_spot_percentage=None, autoscaling_maximum_bid_price_percentage=100,
autoscaling_timeout_for_request=1, autoscaling_spot_fallback=None):
def set_autoscaling_spot(self, autoscaling_spot_percentage=None,
autoscaling_maximum_bid_price_percentage=100,
autoscaling_timeout_for_request=1,
autoscaling_spot_allocation_strategy=None,
autoscaling_spot_fallback=None):
spot = {"percentage": autoscaling_spot_percentage,
"type": "spot",
"maximum_bid_price_percentage": autoscaling_maximum_bid_price_percentage,
"timeout_for_request": autoscaling_timeout_for_request,
"allocation_strategy": autoscaling_spot_allocation_strategy,
"fallback": autoscaling_spot_fallback
}
self.cluster_info["composition"]["autoscaling_nodes"]["nodes"].append(spot)
Expand Down Expand Up @@ -653,7 +671,11 @@ def cluster_info_parser(argparser, action):
default=None,
help="whether to fallback to on-demand instances for min nodes" +
" if spot instances aren't available")

composition_group.add_argument("--min-spot-allocation-strategy",
dest="min_spot_allocation_strategy",
choices=["lowestPrice", "capacityOptimized", None],
default=None,
help="allocation strategy for min spot nodes")
composition_group.add_argument("--autoscaling-ondemand-percentage",
dest="autoscaling_ondemand_percentage",
type=int,
Expand Down Expand Up @@ -689,6 +711,12 @@ def cluster_info_parser(argparser, action):
default=None,
help="whether to fallback to on-demand instances for autoscaling nodes" +
" if spot instances aren't available")
composition_group.add_argument("--autoscaling-spot-allocation-strategy",
dest="autoscaling_spot_allocation_strategy",
choices=["lowestPrice", "capacityOptimized", None],
default=None,
help="allocation strategy for autoscaling" +
" spot nodes")

# monitoring settings
monitoring_group = argparser.add_argument_group("monitoring settings")
Expand Down
Loading