Merge remote-tracking branch 'origin/emr-release-0325' into release-1…

….7.17

* origin/emr-release-0325:
  Updates to emr create-cluster helper text from the name change for CSE
  refactor CSE, SSE, and consistent view examples in man page fix helptext to include descriptions of emrfs options; modify emrfs cse examples to use AMI 3.6.0
  Adding information to EMR command's help about variables that can be set in AWS CLI config file
  Fixing bug in EMR to pick up bootstrap action scripts from the correct region's S3 buckets
  Updating EMR code to incorporate changes in before-building-argument-table-parser even thrown from BasicCommand
  Ensuring that required attributes are displayed correctly when viewing help even if user has those attributes configured in the config file
  Adding support for static configs for EMR commands
  Emiting an event before building arg table parser for custom commands
  Updating calls to assert_params_for_cmd2 to assert_params_for_cmd in test_emrfs_utils.py
  1. Remove changes for list cmd; 2. Update code per cr comments
  Enhance configure set/get/list commands for nested attributes
  Support for client side encryption in --emrfs option of create-cluster command
  remove extraneous impala step; remove invalid -f option
  EMR: Add EMRFS Client-Side Encryption and examples.

awscli/clidriver.py

@@ -240,6 +240,7 @@ def _handle_top_level_args(self, args):
 
 
 class CLICommand(object):
+
     """Interface for a CLI command.
 
     This class represents a top level CLI command
@@ -297,6 +298,7 @@ def arg_table(self):
 
 
 class ServiceCommand(CLICommand):
+
     """A service command for the CLI.
 
     For example, ``aws ec2 ...`` we'd create a ServiceCommand
@@ -409,6 +411,7 @@ def _create_parser(self):
 
 
 class ServiceOperation(object):
+
     """A single operation of a service.
 
     This class represents a single operation for a service, for
@@ -486,7 +489,8 @@ def __call__(self, args, parsed_globals):
         # of a service we can go ahead and load the parameters.
         event = 'before-building-argument-table-parser.%s.%s' % \
             (self._parent_name, self._name)
-        self._emit(event, argument_table=self.arg_table, args=args)
+        self._emit(event, argument_table=self.arg_table, args=args,
+                   session=self._session)
         operation_parser = self._create_operation_parser(self.arg_table)
         self._add_help(operation_parser)
         parsed_args, remaining = operation_parser.parse_known_args(args)
@@ -615,6 +619,7 @@ def _create_operation_parser(self, arg_table):
 
 
 class CLIOperationCaller(object):
+
     """Call an AWS operation and format the response."""
 
     def __init__(self, session):

awscli/customizations/commands.py

@@ -2,25 +2,25 @@
 import os
 
 import bcdoc.docevents
-from botocore.compat import OrderedDict
 from botocore import model
+from botocore.compat import OrderedDict
 from botocore.validate import validate_parameters
 
 import awscli
-from awscli.clidocs import OperationDocumentEventHandler
 from awscli.argparser import ArgTableArgParser
 from awscli.argprocess import unpack_argument, unpack_cli_arg
-from awscli.clidriver import CLICommand
 from awscli.arguments import CustomArgument, create_argument_model_from_schema
+from awscli.clidocs import OperationDocumentEventHandler
+from awscli.clidriver import CLICommand
 from awscli.help import HelpCommand
 from awscli.schema import SchemaTransformer
 
-
 LOG = logging.getLogger(__name__)
 _open = open
 
 
 class _FromFile(object):
+
     def __init__(self, *paths, **kwargs):
         """
         ``**kwargs`` can contain a ``root_module`` argument
@@ -40,6 +40,7 @@ def __init__(self, *paths, **kwargs):
 
 
 class BasicCommand(CLICommand):
+
     """Basic top level command with no subcommands.
 
     If you want to create a new command, subclass this and
@@ -130,6 +131,10 @@ def __call__(self, args, parsed_globals):
         # an arg parser and parse them.
         self._subcommand_table = self._build_subcommand_table()
         self._arg_table = self._build_arg_table()
+        event = 'before-building-argument-table-parser.%s' % \
+            ".".join(self.lineage_names)
+        self._session.emit(event, argument_table=self._arg_table, args=args,
+                           session=self._session)
         parser = ArgTableArgParser(self.arg_table, self.subcommand_table)
         parsed_args, remaining = parser.parse_known_args(args)
 
@@ -189,7 +194,7 @@ def _validate_value_against_schema(self, model, value):
 
     def _should_allow_plugins_override(self, param, value):
         if (param and param.argument_model is not None and
-            value is not None):
+                value is not None):
             return True
         return False
 
@@ -214,7 +219,7 @@ def _build_subcommand_table(self):
                            command_table=subcommand_table,
                            session=self._session,
                            command_object=self)
-        self._add_lineage(subcommand_table)        
+        self._add_lineage(subcommand_table)
         return subcommand_table
 
     def _display_help(self, parsed_args, parsed_globals):
@@ -291,6 +296,7 @@ def lineage(self, value):
 
 
 class BasicHelp(HelpCommand):
+
     def __init__(self, session, command_object, command_table, arg_table,
                  event_handler_class=None):
         super(BasicHelp, self).__init__(session, command_object,
@@ -336,8 +342,8 @@ def _get_doc_contents(self, attr_name):
                 trailing_path = os.path.join(self.name, attr_name + '.rst')
             root_module = value.root_module
             doc_path = os.path.join(
-                os.path.abspath(os.path.dirname(root_module.__file__)), 'examples',
-                trailing_path)
+                os.path.abspath(os.path.dirname(root_module.__file__)),
+                'examples', trailing_path)
             with _open(doc_path) as f:
                 return f.read()
         else:
@@ -355,6 +361,7 @@ def __call__(self, args, parsed_globals):
 
 
 class BasicDocHandler(OperationDocumentEventHandler):
+
     def __init__(self, help_command):
         super(BasicDocHandler, self).__init__(help_command)
         self.doc = help_command.doc

awscli/customizations/configure.py

@@ -24,6 +24,7 @@
 logger = logging.getLogger(__name__)
 NOT_SET = '<not set>'
 
+PREDEFINED_SECTION_NAMES = ('preview', 'plugin')
 
 def register_configure_cmd(cli):
     cli.register('building-command-table.main',
@@ -409,6 +410,16 @@ def _run_main(self, args, parsed_globals):
                 varname = remaining[0]
                 if len(remaining) == 2:
                     value = {remaining[1]: value}
+            elif parts[0] not in PREDEFINED_SECTION_NAMES:
+                if self._session.profile is not None:
+                    section = 'profile %s' % self._session.profile
+                else:
+                    profile_name = self._session.get_config_variable('profile')
+                    if profile_name is not None:
+                        section = profile_name
+                varname = parts[0]
+                if len(parts) == 2:
+                    value = {parts[1]: value}
             elif len(parts) == 2:
                 # Otherwise it's something like "set preview.service true"
                 # of something in the [plugin] section.
@@ -460,38 +471,40 @@ def _run_main(self, args, parsed_globals):
             return 1
 
     def _get_dotted_config_value(self, varname):
-        value = None
+        parts = varname.split('.')
         num_dots = varname.count('.')
-        if num_dots == 1:
+        # Logic to deal with predefined sections like [preview], [plugin] and etc.
+        if num_dots == 1 and parts[0] in PREDEFINED_SECTION_NAMES:
             full_config = self._session.full_config
             section, config_name = varname.split('.')
             value = full_config.get(section, {}).get(config_name)
             if value is None:
                 # Try to retrieve it from the profile config.
                 value = full_config['profiles'].get(
                     section, {}).get(config_name)
-        elif varname.startswith(('default', 'profile')):
-            # We're hard coding logic for profiles here.  Really
-            # we could support any generic format of [section subsection],
-            # but we'd need some botocore.session changes for that,
-            # and nothing would immediately use that feature.
-            parts = varname.split('.')
-            if parts[0] == 'default':
-                profile_name = 'default'
-                config_name = parts[1]
-                remaining = parts[2:]
-            else:
-                # ['profile', 'profname', 'foo', ...]
-                profile_name = parts[1]
-                config_name = parts[2]
-                remaining = parts[3:]
-            self._session.profile = profile_name
-            value = self._session.get_scoped_config().get(config_name)
-            if len(remaining) == 1:
-                try:
-                    value = value.get(remaining[-1])
-                except AttributeError:
-                    value = None
+            return value
+        if parts[0] == 'profile':
+            profile_name = parts[1]
+            config_name = parts[2]
+            remaining = parts[3:]
+        # Check if varname starts with 'default' profile (e.g. default.emr-dev.emr.instance_profile)
+        # If not, go further to check if varname starts with a known profile name
+        elif parts[0] == 'default' or (parts[0] in self._session.full_config['profiles']):
+            profile_name = parts[0]
+            config_name = parts[1]
+            remaining = parts[2:]
+        else:
+            profile_name = self._session.get_config_variable('profile')
+            config_name = parts[0]
+            remaining = parts[1:]
+
+        self._session.profile = profile_name
+        value = self._session.get_scoped_config().get(config_name)
+        if len(remaining) == 1:
+            try:
+                value = value.get(remaining[-1])
+            except AttributeError:
+                value = None
         return value
 
 

awscli/customizations/emr/addinstancegroups.py

@@ -12,14 +12,14 @@
 # language governing permissions and limitations under the License.
 
 
-from awscli.customizations.emr import emrutils
-from awscli.customizations.commands import BasicCommand
 from awscli.customizations.emr import argumentschema
+from awscli.customizations.emr import emrutils
 from awscli.customizations.emr import helptext
 from awscli.customizations.emr import instancegroupsutils
+from awscli.customizations.emr.command import Command
 
 
-class AddInstanceGroups(BasicCommand):
+class AddInstanceGroups(Command):
     NAME = 'add-instance-groups'
     DESCRIPTION = 'Adds an instance group to a running cluster.'
     ARG_TABLE = [
@@ -30,15 +30,15 @@ class AddInstanceGroups(BasicCommand):
          'schema': argumentschema.INSTANCE_GROUPS_SCHEMA}
     ]
 
-    def _run_main(self, parsed_args, parsed_globals):
+    def _run_main_command(self, parsed_args, parsed_globals):
         parameters = {'JobFlowId': parsed_args.cluster_id}
         parameters['InstanceGroups'] = \
             instancegroupsutils.build_instance_groups(
             parsed_args.instance_groups)
 
         add_instance_groups_response = emrutils.call(
             self._session, 'add_instance_groups', parameters,
-            parsed_globals.region, parsed_globals.endpoint_url,
+            self.region, parsed_globals.endpoint_url,
             parsed_globals.verify_ssl)
 
         constructed_result = self._construct_result(
@@ -52,9 +52,9 @@ def _construct_result(self, add_instance_groups_result):
         jobFlowId = None
         instanceGroupIds = None
         if add_instance_groups_result is not None:
-                jobFlowId = add_instance_groups_result.get('JobFlowId')
-                instanceGroupIds = add_instance_groups_result.get(
-                    'InstanceGroupIds')
+            jobFlowId = add_instance_groups_result.get('JobFlowId')
+            instanceGroupIds = add_instance_groups_result.get(
+                'InstanceGroupIds')
 
         if jobFlowId is not None and instanceGroupIds is not None:
             return {'ClusterId': jobFlowId,

awscli/customizations/emr/addsteps.py

@@ -12,13 +12,14 @@
 # language governing permissions and limitations under the License.
 
 from awscli.customizations.commands import BasicCommand
-from awscli.customizations.emr import steputils
 from awscli.customizations.emr import argumentschema
-from awscli.customizations.emr import helptext
 from awscli.customizations.emr import emrutils
+from awscli.customizations.emr import helptext
+from awscli.customizations.emr import steputils
+from awscli.customizations.emr.command import Command
 
 
-class AddSteps(BasicCommand):
+class AddSteps(Command):
     NAME = 'add-steps'
     DESCRIPTION = ('Add a list of steps to a cluster.')
     ARG_TABLE = [
@@ -34,10 +35,10 @@ class AddSteps(BasicCommand):
     ]
     EXAMPLES = BasicCommand.FROM_FILE('emr', 'add-steps.rst')
 
-    def _run_main(self, parsed_args, parsed_globals):
+    def _run_main_command(self, parsed_args, parsed_globals):
         parsed_steps = parsed_args.steps
         step_list = steputils.build_step_config_list(
-            parsed_step_list=parsed_steps, region=parsed_globals.region)
+            parsed_step_list=parsed_steps, region=self.region)
         parameters = {
             'JobFlowId': parsed_args.cluster_id,
             'Steps': step_list

awscli/customizations/emr/applicationutils.py

@@ -16,13 +16,11 @@
 from awscli.customizations.emr import exceptions
 
 
-def build_applications(session,
-                       parsed_applications, parsed_globals, ami_version=None):
+def build_applications(region,
+                       parsed_applications, ami_version=None):
     app_list = []
     step_list = []
     ba_list = []
-    region = parsed_globals.region if parsed_globals.region \
-        else session.get_config_variable('region')
 
     for app_config in parsed_applications:
         app_name = app_config['Name'].lower()

awscli/customizations/emr/argumentschema.py

@@ -11,8 +11,9 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 
-from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME
 from awscli.customizations.emr import helptext
+from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME
+
 
 INSTANCE_GROUPS_SCHEMA = {
     "type": "array",
@@ -201,7 +202,7 @@
                     "A list of command line arguments to pass to the step.",
                 "items": {
                         "type": "string"
-                    }
+                }
             },
             "MainClass": {
                 "type": "string",
@@ -265,6 +266,28 @@
             "items": {
                 "type": "string"
             }
+        },
+        "Encryption": {
+            "type": "string",
+            "description": "EMRFS encryption type.",
+            "enum": ["SERVERSIDE", "CLIENTSIDE"]
+        },
+        "ProviderType": {
+            "type": "string",
+            "description": "EMRFS client-side encryption provider type.",
+            "enum": ["KMS", "CUSTOM"]
+        },
+        "KMSKeyId": {
+            "type": "string",
+            "description": "AWS KMS's customer master key identifier",
+        },
+        "CustomProviderLocation": {
+            "type": "string",
+            "description": "Custom encryption provider JAR location."
+        },
+        "CustomProviderClass": {
+            "type": "string",
+            "description": "Custom encryption provider full class name."
         }
     }
 }