Ability to set azure cluster composition (#323)

bin/qds.py

@@ -587,7 +587,7 @@ def main():
                          help="skip verification of server SSL certificate. Insecure: use with caution.")
 
     optparser.add_option("--cloud_name", dest="cloud_name",
-                         default=os.getenv('CLOUD_PROVIDER'),
+                         default=os.getenv('CLOUD_PROVIDER', "AWS"),
                          help="cloud", choices=["AWS", "AZURE", "ORACLE_BMC", "ORACLE_OPC", "GCP"])
 
     optparser.add_option("--base_retry_delay", dest="base_retry_delay",

qds_sdk/cloud/aws_cloud.py

@@ -107,6 +107,147 @@ def set_cloud_config_from_arguments(self, arguments):
                               bastion_node_public_dns=arguments.bastion_node_public_dns,
                               master_elastic_ip=arguments.master_elastic_ip)
 
+    def set_composition_arguments(self, comp_group):
+        comp_group.add_argument(
+            "--master-type",
+            dest="master_type",
+            choices=["ondemand", "spot", "spotblock"],
+            default="ondemand",
+            help="type of master nodes. Valid values are:" +
+                 " ('ondemand', 'spot', 'spotblock')" +
+                 " default: ondemand")
+        comp_group.add_argument(
+            "--master-spot-block-duration",
+            dest="master_spot_block_duration",
+            type=int,
+            default=120,
+            help="spot block duration unit: minutes")
+        comp_group.add_argument(
+            "--master-maximum-bid-price-percentage",
+            dest="master_maximum_bid_price_percentage",
+            type=int,
+            default=100,
+            help="maximum value to bid for master spot instances" +
+                 " expressed as a percentage of the base" +
+                 " price for the master instance types")
+        comp_group.add_argument(
+            "--master-timeout-for-request",
+            dest="master_timeout_for_request",
+            type=int,
+            default=1,
+            help="timeout for a master spot instance request, unit: minutes")
+        comp_group.add_argument(
+            "--master-spot-fallback",
+            dest="master_spot_fallback",
+            choices=["ondemand", None],
+            default=None,
+            help="whether to fallback to on-demand instances for master nodes" +
+                 " if spot instances aren't available")
+        comp_group.add_argument(
+            "--min-ondemand-percentage",
+            dest="min_ondemand_percentage",
+            type=int,
+            help="percentage of ondemand nodes in min config")
+        comp_group.add_argument(
+            "--min-spot-block-percentage",
+            dest="min_spot_block_percentage",
+            type=int,
+            help="percentage of spot block nodes in min config")
+        comp_group.add_argument(
+            "--min-spot-percentage",
+            dest="min_spot_percentage",
+            type=int,
+            help="percentage of spot nodes in min config")
+        comp_group.add_argument(
+            "--min-spot-block-duration",
+            dest="min_spot_block_duration",
+            type=int,
+            default=120,
+            help="spot block duration unit: minutes")
+        comp_group.add_argument(
+            "--min-maximum-bid-price-percentage",
+            dest="min_maximum_bid_price_percentage",
+            type=int,
+            default=100,
+            help="maximum value to bid for min spot instances" +
+                 " expressed as a percentage of the base" +
+                 " price for the master instance types")
+        comp_group.add_argument(
+            "--min-timeout-for-request",
+            dest="min_timeout_for_request",
+            type=int,
+            default=1,
+            help="timeout for a min spot instance request, unit: minutes")
+        comp_group.add_argument(
+            "--min-spot-fallback",
+            dest="min_spot_fallback",
+            choices=["ondemand", None],
+            default=None,
+            help="whether to fallback to on-demand instances for min nodes" +
+                 " if spot instances aren't available")
+        comp_group.add_argument(
+            "--min-spot-allocation-strategy",
+            dest="min_spot_allocation_strategy",
+            choices=["lowestPrice", "capacityOptimized", None],
+            default=None,
+            help="allocation strategy for min spot nodes")
+        comp_group.add_argument(
+            "--autoscaling-ondemand-percentage",
+            dest="autoscaling_ondemand_percentage",
+            type=int,
+            help="percentage of ondemand nodes in autoscaling config")
+        comp_group.add_argument(
+            "--autoscaling-spot-block-percentage",
+            dest="autoscaling_spot_block_percentage",
+            type=int,
+            help="percentage of spot block nodes in autoscaling config")
+        comp_group.add_argument(
+            "--autoscaling-spot-percentage",
+            dest="autoscaling_spot_percentage",
+            type=int,
+            help="percentage of spot nodes in autoscaling config")
+        comp_group.add_argument(
+            "--autoscaling-spot-block-duration",
+            dest="autoscaling_spot_block_duration",
+            type=int,
+            default=120,
+            help="spot block duration unit:  minutes")
+        comp_group.add_argument(
+            "--autoscaling-spot-block-fallback",
+            dest="autoscaling_spot_block_fallback",
+            choices=["ondemand", None],
+            default=None,
+            help="whether to fallback to on-demand instances for autoscaling" +
+                 " nodes if spot block instances aren't available")
+        comp_group.add_argument(
+            "--autoscaling-maximum-bid-price-percentage",
+            dest="autoscaling_maximum_bid_price_percentage",
+            type=int,
+            default=100,
+            help="maximum value to bid for autoscaling spot" +
+                 " instances expressed as a percentage of" +
+                 " the base price for the master instance types")
+        comp_group.add_argument(
+            "--autoscaling-timeout-for-request",
+            dest="autoscaling_timeout_for_request",
+            type=int,
+            default=1,
+            help="timeout for a autoscaling spot instance request, unit: minutes")
+        comp_group.add_argument(
+            "--autoscaling-spot-fallback",
+            dest="autoscaling_spot_fallback",
+            choices=["ondemand", None],
+            default=None,
+            help="whether to fallback to on-demand instances for autoscaling nodes" +
+                 " if spot instances aren't available")
+        comp_group.add_argument(
+            "--autoscaling-spot-allocation-strategy",
+            dest="autoscaling_spot_allocation_strategy",
+            choices=["lowestPrice", "capacityOptimized", None],
+            default=None,
+            help="allocation strategy for autoscaling" +
+                 " spot nodes")
+
     def create_parser(self, argparser):
         # compute settings parser
         compute_config = argparser.add_argument_group("compute config settings")

qds_sdk/cloud/azure_cloud.py

@@ -153,6 +153,89 @@ def set_cloud_config_from_arguments(self, arguments):
                               master_static_public_ip_name=arguments.master_static_public_ip_name,
                               resource_group_name=arguments.resource_group_name)
 
+    def set_composition_arguments(self, comp_group):
+        # composition arguments we want to accept for azure
+        comp_group.add_argument("--min-ondemand-percentage",
+                                dest="min_ondemand_percentage",
+                                type=int, default=0,
+                                help="Percentage of ondemand nodes in min config.")
+        comp_group.add_argument("--min-spot-percentage",
+                                dest="min_spot_percentage",
+                                type=int, default=0,
+                                help="Percentage of spot nodes in min config.")
+        comp_group.add_argument("--max-price-percentage",
+                                dest="max_price_percentage",
+                                type=int, default=100,
+                                help="Percentage of maximum price percentage"
+                                     " for spot nodes.")
+        comp_group.add_argument("--min-spot-fallback",
+                                dest="min_spot_fallback",
+                                choices=["ondemand", None],
+                                default=None,
+                                help="Whether to fallback to on-demand instances" +
+                                     " for min nodes if spot instances" +
+                                     " aren't available.")
+        comp_group.add_argument("--autoscaling-ondemand-percentage",
+                                dest="autoscaling_ondemand_percentage",
+                                type=int, default=0,
+                                help="Percentage of ondemand nodes" +
+                                     "in autoscaling config.")
+        comp_group.add_argument("--autoscaling-spot-percentage",
+                                dest="autoscaling_spot_percentage",
+                                type=int, default=0,
+                                help="Percentage of spot nodes" +
+                                "in autoscaling config.")
+        comp_group.add_argument("--autoscaling-spot-fallback",
+                                dest="autoscaling_spot_fallback",
+                                choices=["ondemand", None],
+                                default=None,
+                                help="Whether to fallback to on-demand instances" +
+                                     " for autoscaling nodes if spot instances" +
+                                     " aren't available.")
+
+    # Ignore other key-value arguments.
+    def get_composition(self,
+                        min_ondemand_percentage=0,
+                        min_spot_percentage=0,
+                        min_spot_fallback=None,
+                        autoscaling_ondemand_percentage=0,
+                        autoscaling_spot_percentage=0,
+                        autoscaling_spot_fallback=None,
+                        max_price_percentage=100,
+                        **kwargs):
+        composition = {}
+        composition["min_nodes"] = {"nodes": []}
+        min_nodes = composition["min_nodes"]["nodes"]
+        if min_ondemand_percentage + min_spot_percentage != 100:
+            raise ValueError("Minimum nodes ondemand+spot percentage"
+                             " should be 100: Ondemand pct: %d Spot pct: %d"
+                             % (min_ondemand_percentage,
+                                min_spot_percentage))
+        if min_ondemand_percentage > 0:
+            min_nodes.append({"type": "ondemand",
+                              "percentage": min_ondemand_percentage})
+        if min_spot_percentage > 0:
+            min_nodes.append({"type": "spot", "percentage": min_spot_percentage,
+                              "fallback": min_spot_fallback,
+                              "max_price_percentage": max_price_percentage})
+
+        composition["autoscaling_nodes"] = {"nodes": []}
+        autoscaling_nodes = composition["autoscaling_nodes"]["nodes"]
+        if autoscaling_ondemand_percentage + autoscaling_spot_percentage != 100:
+            raise ValueError("Autoscaling nodes ondemand+spot percentage" +
+                             " should be 100: Ondemand pct: %d Spot pct: %d"
+                             % (autoscaling_ondemand_percentage,
+                                autoscaling_spot_percentage))
+        if autoscaling_ondemand_percentage > 0:
+            autoscaling_nodes.append({"type": "ondemand",
+                                      "percentage": autoscaling_ondemand_percentage})
+        if autoscaling_spot_percentage > 0:
+            autoscaling_nodes.append({"type": "spot",
+                                      "percentage": autoscaling_spot_percentage,
+                                      "fallback": autoscaling_spot_fallback,
+                                      "max_price_percentage": max_price_percentage})
+        return composition
+
     def create_parser(self, argparser):
         # compute settings parser
         compute_config = argparser.add_argument_group("compute config settings")
@@ -228,4 +311,4 @@ def create_parser(self, argparser):
         storage_config.add_argument("--disk-storage-account-resource-group-name",
                                     dest="disk_storage_account_resource_group_name",
                                     default=None,
-                                    help="disk storage account resource group for azure cluster")
+                                    help="disk storage account resource group for azure cluster")

qds_sdk/cloud/cloud.py

@@ -5,3 +5,9 @@ def create_parser(self, argparser):
 
     def set_cloud_config_from_arguments(self, arguments):
         return NotImplemented
+
+    def set_composition_arguments(self, comp_group):
+        pass
+
+    def get_composition(self, **kwargs):
+        pass