Skip to content

Commit

Permalink
SDK-323: GCP changes for preemptible nodes (#266)
Browse files Browse the repository at this point in the history
  • Loading branch information
@akaranjkar-qu @chattarajoy
akaranjkar-qu authored and chattarajoy committed May 14, 2019
1 parent 3631d0e commit 62468d2
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 3 deletions.
71 changes: 68 additions & 3 deletions qds_sdk/cloud/gcp_cloud.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ def __init__(self):
self.location = {}
self.network_config = {}
self.storage_config = {}
self.cluster_composition = {}

def set_cloud_config(self,
qsa_client_id=None,
Expand All @@ -27,7 +28,12 @@ def set_cloud_config(self,
storage_disk_type=None,
bastion_node_public_dns=None,
vpc_id=None,
subnet_id=None):
subnet_id=None,
master_preemptible=None,
min_nodes_preemptible=None,
min_nodes_preemptible_percentage=None,
autoscaling_nodes_preemptible=None,
autoscaling_nodes_preemptible_percentage=None):
'''
Args:
Expand Down Expand Up @@ -56,13 +62,25 @@ def set_cloud_config(self,
vpc_id: Vpc id for gcp cluster
subnet_id: Subnet id for gcp cluster
master_preemptible: if the master node is preemptible
min_nodes_preemptible: if the min nodes are preemptible
min_nodes_preemptible_percentage: percentage of min nodes that are preemptible
autoscaling_nodes_preemptible: if the autoscaling nodes are preemptible
autoscaling_nodes_preemptible_percentage: percentage of autoscaling nodes that are preemptible
'''

self.set_compute_config(use_account_compute_creds, qsa_client_id, customer_project_id, qsa_client_email,
qsa_private_key_id, qsa_private_key, comp_client_email)
self.set_location(gcp_region, gcp_zone)
self.set_network_config(bastion_node_public_dns, vpc_id, subnet_id)
self.set_storage_config(inst_client_email, storage_disk_size_in_gb, storage_disk_count, storage_disk_type)
self.set_cluster_composition(master_preemptible, min_nodes_preemptible, min_nodes_preemptible_percentage,
autoscaling_nodes_preemptible, autoscaling_nodes_preemptible_percentage)

def set_compute_config(self,
use_account_compute_creds=None,
Expand Down Expand Up @@ -106,6 +124,21 @@ def set_storage_config(self,
self.storage_config['disk_count'] = storage_disk_count
self.storage_config['disk_type'] = storage_disk_type

def set_cluster_composition(self,
master_preemptible=None,
min_nodes_preemptible=None,
min_nodes_preemptible_percentage=None,
autoscaling_nodes_preemptible=None,
autoscaling_nodes_preemptible_percentage=None):
self.cluster_composition['master'] = {}
self.cluster_composition['master']['preemptible'] = master_preemptible
self.cluster_composition['min_nodes'] = {}
self.cluster_composition['min_nodes']['preemptible'] = min_nodes_preemptible
self.cluster_composition['min_nodes']['percentage'] = min_nodes_preemptible_percentage
self.cluster_composition['autoscaling_nodes'] = {}
self.cluster_composition['autoscaling_nodes']['preemptible'] = autoscaling_nodes_preemptible
self.cluster_composition['autoscaling_nodes']['percentage'] = autoscaling_nodes_preemptible_percentage

def set_cloud_config_from_arguments(self, arguments):
self.set_cloud_config(qsa_client_id=arguments.qsa_client_id,
customer_project_id=arguments.customer_project_id,
Expand All @@ -122,7 +155,12 @@ def set_cloud_config_from_arguments(self, arguments):
storage_disk_type=arguments.storage_disk_type,
bastion_node_public_dns=arguments.bastion_node_public_dns,
vpc_id=arguments.vpc_id,
subnet_id=arguments.subnet_id)
subnet_id=arguments.subnet_id,
master_preemptible=arguments.master_preemptible,
min_nodes_preemptible=arguments.min_nodes_preemptible,
min_nodes_preemptible_percentage=arguments.min_nodes_preemptible_percentage,
autoscaling_nodes_preemptible=arguments.autoscaling_nodes_preemptible,
autoscaling_nodes_preemptible_percentage=arguments.autoscaling_nodes_preemptible_percentage)

def create_parser(self, argparser):
# compute settings parser
Expand Down Expand Up @@ -202,4 +240,31 @@ def create_parser(self, argparser):
storage_config.add_argument("--storage-disk-type",
dest="storage_disk_type",
default=None,
help="disk type for gcp cluster")
help="disk type for gcp cluster")
# cluster composition settings parser
cluster_composition = argparser.add_argument_group("cluster composition settings")
cluster_composition.add_argument("--master-preemptible",
dest="master_preemptible",
action="store_true",
default=None,
help="if the master node is preemptible")
cluster_composition.add_argument("--min-nodes-preemptible",
dest="min_nodes_preemptible",
action="store_true",
default=None,
help="if the min nodes are preemptible")
cluster_composition.add_argument("--min-nodes-preemptible-percentage",
dest="min_nodes_preemptible_percentage",
type=int,
default=None,
help="percentage of min nodes that are preemptible")
cluster_composition.add_argument("--autoscaling-nodes-preemptible",
dest="autoscaling_nodes_preemptible",
action="store_true",
default=None,
help="if the autoscaling nodes are preemptible")
cluster_composition.add_argument("--autoscaling-nodes-preemptible-percentage",
dest="autoscaling_nodes_preemptible_percentage",
type=int,
default=None,
help="percentage of autoscaling nodes that are preemptible")
40 changes: 40 additions & 0 deletions tests/test_clusterv2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,46 @@ def test_gcp_location_config(self):
'zone': 'yyy'}},
'cluster_info': {'label': ['test_label']}})

def test_gcp_cluster_composition(self):
sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'GCP', 'cluster', 'create', '--label', 'test_label',
'--master-preemptible',
'--min-nodes-preemptible', '--min-nodes-preemptible-percentage', '50',
'--autoscaling-nodes-preemptible', '--autoscaling-nodes-preemptible-percentage', '75']
Qubole.cloud = None
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters',
{
'cloud_config': {
'cluster_composition': {
'master': {
'preemptible': True
},
'min_nodes': {
'preemptible': True,
'percentage': 50
},
'autoscaling_nodes': {
'preemptible': True,
'percentage': 75
}
}
},
'cluster_info': {
'label': ['test_label']
}
})

def test_gcp_cluster_composition_invalid(self):
sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'GCP', 'cluster', 'create', '--label', 'test_label',
'--master-preemptible',
'--min-nodes-preemptible', '--min-nodes-preemptible-percentage', 'invalid_value']
Qubole.cloud = None
print_command()
with self.assertRaises(SystemExit):
qds.main()

def test_presto_engine_config(self):
with tempfile.NamedTemporaryFile() as temp:
temp.write("config.properties:\na=1\nb=2".encode("utf8"))
Expand Down

0 comments on commit 62468d2

Please sign in to comment.