Skip to content

Commit

Permalink
SDK-335: GCP creds schema changes (#263)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tarrygo @chattarajoy
tarrygo authored and chattarajoy committed May 14, 2019
1 parent 83d4ad9 commit b1a4ffc
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 109 deletions.
154 changes: 62 additions & 92 deletions qds_sdk/cloud/gcp_cloud.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,16 @@ def __init__(self):
self.storage_config = {}

def set_cloud_config(self,
compute_client_id=None,
compute_project_id=None,
compute_client_email=None,
compute_private_key_id=None,
compute_private_key=None,
qsa_client_id=None,
customer_project_id=None,
qsa_client_email=None,
qsa_private_key_id=None,
qsa_private_key=None,
comp_client_email=None,
inst_client_email=None,
use_account_compute_creds=None,
gcp_region=None,
gcp_zone=None,
storage_client_id=None,
storage_project_id=None,
storage_client_email=None,
storage_private_key_id=None,
storage_private_key=None,
storage_disk_size_in_gb=None,
storage_disk_count=None,
storage_disk_type=None,
Expand All @@ -34,31 +31,25 @@ def set_cloud_config(self,
'''
Args:
compute_client_id: Compute client id for gcp cluster
qsa_client_id: Compute client id for gcp cluster
compute_project_id: Compute project id for gcp cluster
customer_project_id: Compute project id for gcp cluster
compute_client_email: Compute client email for gcp cluster
qsa_client_email: Compute client email for gcp cluster
compute_private_key_id: Compute private key id for gcp cluster
qsa_private_key_id: Compute private key id for gcp cluster
compute_private_key: Compute private key for gcp cluster
qsa_private_key: Compute private key for gcp cluster
comp_client_email: Client compute service account email
inst_client_email: Client storage/instance service account email
use_account_compute_creds: Set it to true to use the account's compute
credentials for all clusters of the account.The default value is false
gcp_region: Region for gcp cluster
storage_client_id: Storage client id for gcp cluster
storage_project_id: Storage project id for gcp cluster
storage_client_email: Storage client email for gcp cluster
storage_private_key_id: Storage private key id for gcp cluster
storage_private_key: Storage private key for gcp cluster
bastion_node_public_dns: public dns name of the bastion node.
Required only if cluster is in a private subnet.
Expand All @@ -67,26 +58,27 @@ def set_cloud_config(self,
subnet_id: Subnet id for gcp cluster
'''

self.set_compute_config(use_account_compute_creds, compute_client_id, compute_project_id, compute_client_email,
compute_private_key_id, compute_private_key)
self.set_compute_config(use_account_compute_creds, qsa_client_id, customer_project_id, qsa_client_email,
qsa_private_key_id, qsa_private_key, comp_client_email)
self.set_location(gcp_region, gcp_zone)
self.set_network_config(bastion_node_public_dns, vpc_id, subnet_id)
self.set_storage_config(storage_client_id, storage_project_id, storage_client_email, storage_private_key_id,
storage_private_key, storage_disk_size_in_gb, storage_disk_count, storage_disk_type)
self.set_storage_config(inst_client_email, storage_disk_size_in_gb, storage_disk_count, storage_disk_type)

def set_compute_config(self,
use_account_compute_creds=None,
compute_client_id=None,
compute_project_id=None,
compute_client_email=None,
compute_private_key_id=None,
compute_private_key=None):
qsa_client_id=None,
customer_project_id=None,
qsa_client_email=None,
qsa_private_key_id=None,
qsa_private_key=None,
comp_client_email=None):
self.compute_config['use_account_compute_creds'] = use_account_compute_creds
self.compute_config['compute_client_id'] = compute_client_id
self.compute_config['compute_project_id'] = compute_project_id
self.compute_config['compute_client_email'] = compute_client_email
self.compute_config['compute_private_key_id'] = compute_private_key_id
self.compute_config['compute_private_key'] = compute_private_key
self.compute_config['qsa_client_id'] = qsa_client_id
self.compute_config['customer_project_id'] = customer_project_id
self.compute_config['qsa_client_email'] = qsa_client_email
self.compute_config['qsa_private_key_id'] = qsa_private_key_id
self.compute_config['qsa_private_key'] = qsa_private_key
self.compute_config['comp_client_email'] = comp_client_email

def set_location(self,
gcp_region=None,
Expand All @@ -104,38 +96,27 @@ def set_network_config(self,
self.network_config['subnet'] = subnet_id

def set_storage_config(self,
storage_client_id=None,
storage_project_id=None,
storage_client_email=None,
storage_private_key_id=None,
storage_private_key=None,
inst_client_email=None,
storage_disk_size_in_gb=None,
storage_disk_count=None,
storage_disk_type=None
):
self.storage_config['storage_client_id'] = storage_client_id
self.storage_config['storage_project_id'] = storage_project_id
self.storage_config['storage_client_email'] = storage_client_email
self.storage_config['storage_private_key_id'] = storage_private_key_id
self.storage_config['storage_private_key'] = storage_private_key
self.storage_config['inst_client_email'] = inst_client_email
self.storage_config['disk_size_in_gb'] = storage_disk_size_in_gb
self.storage_config['disk_count'] = storage_disk_count
self.storage_config['disk_type'] = storage_disk_type

def set_cloud_config_from_arguments(self, arguments):
self.set_cloud_config(compute_client_id=arguments.compute_client_id,
compute_project_id=arguments.compute_project_id,
compute_client_email=arguments.compute_client_email,
compute_private_key_id=arguments.compute_private_key_id,
compute_private_key=arguments.compute_private_key,
self.set_cloud_config(qsa_client_id=arguments.qsa_client_id,
customer_project_id=arguments.customer_project_id,
qsa_client_email=arguments.qsa_client_email,
qsa_private_key_id=arguments.qsa_private_key_id,
qsa_private_key=arguments.qsa_private_key,
inst_client_email=arguments.inst_client_email,
comp_client_email=arguments.comp_client_email,
use_account_compute_creds=arguments.use_account_compute_creds,
gcp_region=arguments.gcp_region,
gcp_zone=arguments.gcp_zone,
storage_client_id=arguments.storage_client_id,
storage_project_id=arguments.storage_project_id,
storage_client_email=arguments.storage_client_email,
storage_private_key_id=arguments.storage_private_key_id,
storage_private_key=arguments.storage_private_key,
storage_disk_size_in_gb=arguments.storage_disk_size_in_gb,
storage_disk_count=arguments.storage_disk_count,
storage_disk_type=arguments.storage_disk_type,
Expand All @@ -157,26 +138,30 @@ def create_parser(self, argparser):
action="store_false",
default=None,
help="to disable account compute credentials")
compute_config.add_argument("--compute-client-id",
dest="compute_client_id",
compute_config.add_argument("--qsa-client-id",
dest="qsa_client_id",
default=None,
help="compute client id for gcp cluster")
compute_config.add_argument("--compute-project-id",
dest="compute_project_id",
help="qsa client id for gcp cluster")
compute_config.add_argument("--customer-project-id",
dest="customer_project_id",
default=None,
help="compute project id for gcp cluster")
compute_config.add_argument("--compute-client-email",
dest="compute_client_email",
help="customer project id for gcp cluster")
compute_config.add_argument("--qsa-client-email",
dest="qsa_client_email",
default=None,
help="compute client email for gcp cluster")
compute_config.add_argument("--compute-private-key-id",
dest="compute_private_key_id",
help="qsa client email for gcp cluster")
compute_config.add_argument("--qsa-private-key-id",
dest="qsa_private_key_id",
default=None,
help="compute private key id for gcp cluster")
compute_config.add_argument("--compute-private-key",
dest="compute_private_key",
help="qsa private key id for gcp cluster")
compute_config.add_argument("--qsa-private-key",
dest="qsa_private_key",
default=None,
help="compute private key for gcp cluster")
help="qsa private key for gcp cluster")
compute_config.add_argument("--compute-client-email",
dest="comp_client_email",
default=None,
help="client compute service account email")

# location settings parser
location_group = argparser.add_argument_group("location config settings")
Expand All @@ -201,26 +186,11 @@ def create_parser(self, argparser):

# storage config settings parser
storage_config = argparser.add_argument_group("storage config settings")
storage_config.add_argument("--storage-client-id",
dest="storage_client_id",
default=None,
help="storage client id for gcp cluster")
storage_config.add_argument("--storage-project-id",
dest="storage_project_id",
default=None,
help="storage project id for gcp cluster")

storage_config.add_argument("--storage-client-email",
dest="storage_client_email",
default=None,
help="storage client email for gcp cluster")
storage_config.add_argument("--storage-private-key-id",
dest="storage_private_key_id",
default=None,
help="storage private key id for gcp cluster")
storage_config.add_argument("--storage-private-key",
dest="storage_private_key",
dest="inst_client_email",
default=None,
help="storage private key for gcp cluster")
help="client storage service account email")
storage_config.add_argument("--storage-disk-size-in-gb",
dest="storage_disk_size_in_gb",
default=None,
Expand Down
29 changes: 12 additions & 17 deletions tests/test_clusterv2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,36 +313,31 @@ def test_oracle_opc_network_config(self):

def test_gcp_compute_config(self):
sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'GCP', 'cluster', 'create', '--label', 'test_label',
'--compute-client-id', 'xxx11', '--compute-project-id', 'www11', '--compute-client-email',
'yyyy11', '--compute-private-key-id', 'zzz22', '--compute-private-key', 'aaa']
'--qsa-client-id', 'xxx11', '--customer-project-id', 'www11', '--qsa-client-email',
'yyyy11', '--qsa-private-key-id', 'zzz22', '--qsa-private-key', 'aaa']
Qubole.cloud = None
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters', {'cloud_config': {'compute_config':
{'compute_private_key_id': 'zzz22',
'compute_private_key': 'aaa',
'compute_client_email': 'yyyy11',
'compute_project_id': 'www11',
'compute_client_id': 'xxx11'}},
{'qsa_private_key_id': 'zzz22',
'qsa_private_key': 'aaa',
'qsa_client_email': 'yyyy11',
'customer_project_id': 'www11',
'qsa_client_id': 'xxx11'}},
'cluster_info': {'label': ['test_label']}})

def test_gcp_storage_config(self):
sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'GCP', 'cluster', 'create', '--label', 'test_label',
'--storage-client-id', 'xxx11', '--storage-project-id', 'yyyy11', '--storage-client-email', 'www11',
'--storage-private-key-id', 'zzz22', '--storage-private-key', 'aaa', '--storage-disk-size-in-gb', 'aaa',
'--storage-client-email', 'aaa', '--storage-disk-size-in-gb', 'aaa',
'--storage-disk-count', 'bbb', '--storage-disk-type', 'ccc' ]
Qubole.cloud = None
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('POST', 'clusters', {'cloud_config':
{'storage_config':
{'storage_private_key_id': 'zzz22',
'storage_private_key': 'aaa',
'storage_client_email': 'www11',
'storage_project_id': 'yyyy11',
'storage_client_id': 'xxx11',
{'inst_client_email': 'aaa',
'disk_size_in_gb': 'aaa',
'disk_count': 'bbb',
'disk_type': 'ccc'}},
Expand Down Expand Up @@ -653,16 +648,16 @@ def test_oracle_opc_cloud_config(self):
def test_gcp_cloud_config(self):
sys.argv = ['qds.py', '--version', 'v2', '--cloud', 'GCP', 'cluster', 'update', '123',
'--gcp-region', 'xxx', '--subnet-id', 'abc-subnet',
'--storage-client-id', 'xxx11', '--compute-client-id', 'yyyy11']
'--storage-client-email', 'xxx11', '--qsa-client-id', 'yyyy11']
Qubole.cloud = None
print_command()
Connection._api_call = Mock(return_value={})
qds.main()
Connection._api_call.assert_called_with('PUT', 'clusters/123', {'cloud_config':
{'network_config':
{'subnet': 'abc-subnet'},
'compute_config': {'compute_client_id': 'yyyy11'},
'storage_config': {'storage_client_id': 'xxx11'},
'compute_config': {'qsa_client_id': 'yyyy11'},
'storage_config': {'inst_client_email': 'xxx11'},
'location': {'region': 'xxx'}
}
})
Expand Down

0 comments on commit b1a4ffc

Please sign in to comment.