From 62468d2eea002889606db72a62904dad790500cd Mon Sep 17 00:00:00 2001 From: akaranjkar-qu <41927856+akaranjkar-qu@users.noreply.github.com> Date: Thu, 28 Feb 2019 21:23:04 -0800 Subject: [PATCH] SDK-323: GCP changes for preemptible nodes (#266) --- qds_sdk/cloud/gcp_cloud.py | 71 ++++++++++++++++++++++++++++++++++++-- tests/test_clusterv2.py | 40 +++++++++++++++++++++ 2 files changed, 108 insertions(+), 3 deletions(-) diff --git a/qds_sdk/cloud/gcp_cloud.py b/qds_sdk/cloud/gcp_cloud.py index 20897c8c..a2033246 100644 --- a/qds_sdk/cloud/gcp_cloud.py +++ b/qds_sdk/cloud/gcp_cloud.py @@ -10,6 +10,7 @@ def __init__(self): self.location = {} self.network_config = {} self.storage_config = {} + self.cluster_composition = {} def set_cloud_config(self, qsa_client_id=None, @@ -27,7 +28,12 @@ def set_cloud_config(self, storage_disk_type=None, bastion_node_public_dns=None, vpc_id=None, - subnet_id=None): + subnet_id=None, + master_preemptible=None, + min_nodes_preemptible=None, + min_nodes_preemptible_percentage=None, + autoscaling_nodes_preemptible=None, + autoscaling_nodes_preemptible_percentage=None): ''' Args: @@ -56,6 +62,16 @@ def set_cloud_config(self, vpc_id: Vpc id for gcp cluster subnet_id: Subnet id for gcp cluster + + master_preemptible: if the master node is preemptible + + min_nodes_preemptible: if the min nodes are preemptible + + min_nodes_preemptible_percentage: percentage of min nodes that are preemptible + + autoscaling_nodes_preemptible: if the autoscaling nodes are preemptible + + autoscaling_nodes_preemptible_percentage: percentage of autoscaling nodes that are preemptible ''' self.set_compute_config(use_account_compute_creds, qsa_client_id, customer_project_id, qsa_client_email, @@ -63,6 +79,8 @@ def set_cloud_config(self, self.set_location(gcp_region, gcp_zone) self.set_network_config(bastion_node_public_dns, vpc_id, subnet_id) self.set_storage_config(inst_client_email, storage_disk_size_in_gb, storage_disk_count, storage_disk_type) + self.set_cluster_composition(master_preemptible, min_nodes_preemptible, min_nodes_preemptible_percentage, + autoscaling_nodes_preemptible, autoscaling_nodes_preemptible_percentage) def set_compute_config(self, use_account_compute_creds=None, @@ -106,6 +124,21 @@ def set_storage_config(self, self.storage_config['disk_count'] = storage_disk_count self.storage_config['disk_type'] = storage_disk_type + def set_cluster_composition(self, + master_preemptible=None, + min_nodes_preemptible=None, + min_nodes_preemptible_percentage=None, + autoscaling_nodes_preemptible=None, + autoscaling_nodes_preemptible_percentage=None): + self.cluster_composition['master'] = {} + self.cluster_composition['master']['preemptible'] = master_preemptible + self.cluster_composition['min_nodes'] = {} + self.cluster_composition['min_nodes']['preemptible'] = min_nodes_preemptible + self.cluster_composition['min_nodes']['percentage'] = min_nodes_preemptible_percentage + self.cluster_composition['autoscaling_nodes'] = {} + self.cluster_composition['autoscaling_nodes']['preemptible'] = autoscaling_nodes_preemptible + self.cluster_composition['autoscaling_nodes']['percentage'] = autoscaling_nodes_preemptible_percentage + def set_cloud_config_from_arguments(self, arguments): self.set_cloud_config(qsa_client_id=arguments.qsa_client_id, customer_project_id=arguments.customer_project_id, @@ -122,7 +155,12 @@ def set_cloud_config_from_arguments(self, arguments): storage_disk_type=arguments.storage_disk_type, bastion_node_public_dns=arguments.bastion_node_public_dns, vpc_id=arguments.vpc_id, - subnet_id=arguments.subnet_id) + subnet_id=arguments.subnet_id, + master_preemptible=arguments.master_preemptible, + min_nodes_preemptible=arguments.min_nodes_preemptible, + min_nodes_preemptible_percentage=arguments.min_nodes_preemptible_percentage, + autoscaling_nodes_preemptible=arguments.autoscaling_nodes_preemptible, + autoscaling_nodes_preemptible_percentage=arguments.autoscaling_nodes_preemptible_percentage) def create_parser(self, argparser): # compute settings parser @@ -202,4 +240,31 @@ def create_parser(self, argparser): storage_config.add_argument("--storage-disk-type", dest="storage_disk_type", default=None, - help="disk type for gcp cluster") \ No newline at end of file + help="disk type for gcp cluster") + # cluster composition settings parser + cluster_composition = argparser.add_argument_group("cluster composition settings") + cluster_composition.add_argument("--master-preemptible", + dest="master_preemptible", + action="store_true", + default=None, + help="if the master node is preemptible") + cluster_composition.add_argument("--min-nodes-preemptible", + dest="min_nodes_preemptible", + action="store_true", + default=None, + help="if the min nodes are preemptible") + cluster_composition.add_argument("--min-nodes-preemptible-percentage", + dest="min_nodes_preemptible_percentage", + type=int, + default=None, + help="percentage of min nodes that are preemptible") + cluster_composition.add_argument("--autoscaling-nodes-preemptible", + dest="autoscaling_nodes_preemptible", + action="store_true", + default=None, + help="if the autoscaling nodes are preemptible") + cluster_composition.add_argument("--autoscaling-nodes-preemptible-percentage", + dest="autoscaling_nodes_preemptible_percentage", + type=int, + default=None, + help="percentage of autoscaling nodes that are preemptible") diff --git a/tests/test_clusterv2.py b/tests/test_clusterv2.py index ef56ec04..1b33f4e2 100644 --- a/tests/test_clusterv2.py +++ b/tests/test_clusterv2.py @@ -367,6 +367,46 @@ def test_gcp_location_config(self): 'zone': 'yyy'}}, 'cluster_info': {'label': ['test_label']}}) + def test_gcp_cluster_composition(self): + sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'GCP', 'cluster', 'create', '--label', 'test_label', + '--master-preemptible', + '--min-nodes-preemptible', '--min-nodes-preemptible-percentage', '50', + '--autoscaling-nodes-preemptible', '--autoscaling-nodes-preemptible-percentage', '75'] + Qubole.cloud = None + print_command() + Connection._api_call = Mock(return_value={}) + qds.main() + Connection._api_call.assert_called_with('POST', 'clusters', + { + 'cloud_config': { + 'cluster_composition': { + 'master': { + 'preemptible': True + }, + 'min_nodes': { + 'preemptible': True, + 'percentage': 50 + }, + 'autoscaling_nodes': { + 'preemptible': True, + 'percentage': 75 + } + } + }, + 'cluster_info': { + 'label': ['test_label'] + } + }) + + def test_gcp_cluster_composition_invalid(self): + sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'GCP', 'cluster', 'create', '--label', 'test_label', + '--master-preemptible', + '--min-nodes-preemptible', '--min-nodes-preemptible-percentage', 'invalid_value'] + Qubole.cloud = None + print_command() + with self.assertRaises(SystemExit): + qds.main() + def test_presto_engine_config(self): with tempfile.NamedTemporaryFile() as temp: temp.write("config.properties:\na=1\nb=2".encode("utf8"))