diff --git a/awscli/customizations/commands.py b/awscli/customizations/commands.py index 63a0ca7e4dc6..0e1a23750daa 100644 --- a/awscli/customizations/commands.py +++ b/awscli/customizations/commands.py @@ -120,14 +120,16 @@ class BasicCommand(CLICommand): def __init__(self, session): self._session = session + self._arg_table = None + self._subcommand_table = None def __call__(self, args, parsed_globals): # args is the remaining unparsed args. # We might be able to parse these args so we need to create # an arg parser and parse them. - subcommand_table = self._build_subcommand_table() - arg_table = self.arg_table - parser = ArgTableArgParser(arg_table, subcommand_table) + self._subcommand_table = self._build_subcommand_table() + self._arg_table = self._build_arg_table() + parser = ArgTableArgParser(self.arg_table, self.subcommand_table) parsed_args, remaining = parser.parse_known_args(args) # Unpack arguments @@ -138,8 +140,8 @@ def __call__(self, args, parsed_globals): # as these are how the parameters are stored in the # `arg_table`. xformed = key.replace('_', '-') - if xformed in arg_table: - cli_argument = arg_table[xformed] + if xformed in self.arg_table: + cli_argument = self.arg_table[xformed] value = unpack_argument( self._session, @@ -178,8 +180,8 @@ def __call__(self, args, parsed_globals): raise ValueError("Unknown options: %s" % ','.join(remaining)) return self._run_main(parsed_args, parsed_globals) else: - return subcommand_table[parsed_args.subcommand](remaining, - parsed_globals) + return self.subcommand_table[parsed_args.subcommand](remaining, + parsed_globals) def _validate_value_against_schema(self, model, value): validate_parameters(value, model) @@ -233,9 +235,10 @@ def create_help_command_table(self): commands[command['name']] = command['command_class'](self._session) return commands - @property - def arg_table(self): + def _build_arg_table(self): arg_table = OrderedDict() + self._session.emit('building-arg-table.%s' % self.NAME, + arg_table=self.ARG_TABLE) for arg_data in self.ARG_TABLE: # If a custom schema was passed in, create the argument_model @@ -249,6 +252,18 @@ def arg_table(self): arg_table[arg_data['name']] = custom_argument return arg_table + @property + def arg_table(self): + if self._arg_table is None: + self._arg_table = self._build_arg_table() + return self._arg_table + + @property + def subcommand_table(self): + if self._subcommand_table is None: + self._subcommand_table = self._build_subcommand_table() + return self._subcommand_table + @classmethod def add_command(cls, command_table, session, **kwargs): command_table[cls.NAME] = cls(session) diff --git a/awscli/customizations/s3/comparator.py b/awscli/customizations/s3/comparator.py index 9c86eeee9e96..367a78640571 100644 --- a/awscli/customizations/s3/comparator.py +++ b/awscli/customizations/s3/comparator.py @@ -17,30 +17,17 @@ LOG = logging.getLogger(__name__) -def total_seconds(td): - """ - timedelta's time_seconds() function for python 2.6 users - """ - return (td.microseconds + (td.seconds + td.days * 24 * - 3600) * 10**6) / 10**6 - - class Comparator(object): """ This class performs all of the comparisons behind the sync operation """ - def __init__(self, params=None): - self.delete = False - if 'delete' in params: - self.delete = params['delete'] - - self.compare_on_size_only = False - if 'size_only' in params: - self.compare_on_size_only = params['size_only'] - - self.match_exact_timestamps = False - if 'exact_timestamps' in params: - self.match_exact_timestamps = params['exact_timestamps'] + def __init__(self, file_at_src_and_dest_sync_strategy, + file_not_at_dest_sync_strategy, + file_not_at_src_sync_strategy): + + self._sync_strategy = file_at_src_and_dest_sync_strategy + self._not_at_dest_sync_strategy = file_not_at_dest_sync_strategy + self._not_at_src_sync_strategy = file_not_at_src_sync_strategy def call(self, src_files, dest_files): """ @@ -107,63 +94,39 @@ def call(self, src_files, dest_files): compare_keys = self.compare_comp_key(src_file, dest_file) if compare_keys == 'equal': - same_size = self.compare_size(src_file, dest_file) - same_last_modified_time = self.compare_time(src_file, dest_file) - - if self.compare_on_size_only: - should_sync = not same_size - else: - should_sync = (not same_size) or (not same_last_modified_time) - + should_sync = self._sync_strategy.determine_should_sync( + src_file, dest_file + ) if should_sync: - LOG.debug("syncing: %s -> %s, size_changed: %s, " - "last_modified_time_changed: %s", - src_file.src, src_file.dest, - not same_size, not same_last_modified_time) yield src_file elif compare_keys == 'less_than': src_take = True dest_take = False - LOG.debug("syncing: %s -> %s, file does not exist at destination", - src_file.src, src_file.dest) - yield src_file + should_sync = self._not_at_dest_sync_strategy.determine_should_sync(src_file, None) + if should_sync: + yield src_file elif compare_keys == 'greater_than': src_take = False dest_take = True - dest_file.operation_name = 'delete' - if self.delete: - LOG.debug("syncing: (None) -> %s (remove), file does " - "not exist at source (%s) and delete " - "mode enabled", - dest_file.src, dest_file.dest) + should_sync = self._not_at_src_sync_strategy.determine_should_sync(None, dest_file) + if should_sync: yield dest_file elif (not src_done) and dest_done: src_take = True - LOG.debug("syncing: %s -> %s, file does not exist " - "at destination", - src_file.src, src_file.dest) - yield src_file + should_sync = self._not_at_dest_sync_strategy.determine_should_sync(src_file, None) + if should_sync: + yield src_file elif src_done and (not dest_done): dest_take = True - dest_file.operation_name = 'delete' - if self.delete: - LOG.debug("syncing: (None) -> %s (remove), file does not " - "exist at source (%s) and delete mode enabled", - dest_file.src, dest_file.dest) + should_sync = self._not_at_src_sync_strategy.determine_should_sync(None, dest_file) + if should_sync: yield dest_file else: break - def compare_size(self, src_file, dest_file): - """ - :returns: True if the sizes are the same. - False otherwise. - """ - return src_file.size == dest_file.size - def compare_comp_key(self, src_file, dest_file): """ Determines if the source compare_key is less than, equal to, @@ -180,36 +143,3 @@ def compare_comp_key(self, src_file, dest_file): else: return 'greater_than' - - def compare_time(self, src_file, dest_file): - """ - :returns: True if the file does not need updating based on time of - last modification and type of operation. - False if the file does need updating based on the time of - last modification and type of operation. - """ - src_time = src_file.last_update - dest_time = dest_file.last_update - delta = dest_time - src_time - cmd = src_file.operation_name - if cmd == "upload" or cmd == "copy": - if total_seconds(delta) >= 0: - # Destination is newer than source. - return True - else: - # Destination is older than source, so - # we have a more recently updated file - # at the source location. - return False - elif cmd == "download": - if self.match_exact_timestamps: - # An update is needed unless the - # timestamps match exactly. - return total_seconds(delta) == 0 - - if total_seconds(delta) <= 0: - return True - else: - # delta is positive, so the destination - # is newer than the source. - return False diff --git a/awscli/customizations/s3/s3.py b/awscli/customizations/s3/s3.py index 6ec74cfbf9a4..87bffb627785 100644 --- a/awscli/customizations/s3/s3.py +++ b/awscli/customizations/s3/s3.py @@ -14,6 +14,8 @@ from awscli.customizations.commands import BasicCommand from awscli.customizations.s3.subcommands import ListCommand, WebsiteCommand, \ CpCommand, MvCommand, RmCommand, SyncCommand, MbCommand, RbCommand +from awscli.customizations.s3.syncstrategy.register import \ + register_sync_strategies def awscli_initialize(cli): @@ -24,6 +26,7 @@ def awscli_initialize(cli): file """ cli.register("building-command-table.main", add_s3) + cli.register('building-command-table.sync', register_sync_strategies) def s3_plugin_initialize(event_handlers): diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index 46adc0e9ea97..243f7be7501a 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -28,6 +28,9 @@ from awscli.customizations.s3.s3handler import S3Handler, S3StreamHandler from awscli.customizations.s3.utils import find_bucket_key, uni_print, \ AppendFilter, find_dest_path_comp_key +from awscli.customizations.s3.syncstrategy.base import MissingFileSync, \ + SizeAndLastModifiedSync, NeverSync + RECURSIVE = {'name': 'recursive', 'action': 'store_true', 'dest': 'dir_op', @@ -40,11 +43,6 @@ "Displays the operations that would be performed using the " "specified command without actually running them.")} -DELETE = {'name': 'delete', 'action': 'store_true', - 'help_text': ( - "Files that exist in the destination but not in the source are " - "deleted during sync.")} - QUIET = {'name': 'quiet', 'action': 'store_true', 'help_text': ( "Does not display the operations performed from the specified " @@ -178,19 +176,6 @@ EXPIRES = {'name': 'expires', 'nargs': 1, 'help_text': ("The date and time at " "which the object is no longer cacheable.")} -SIZE_ONLY = {'name': 'size-only', 'action': 'store_true', - 'help_text': ( - 'Makes the size of each key the only criteria used to ' - 'decide whether to sync from source to destination.')} - -EXACT_TIMESTAMPS = {'name': 'exact-timestamps', 'action': 'store_true', - 'help_text': ( - 'When syncing from S3 to local, same-sized ' - 'items will be ignored only when the timestamps ' - 'match exactly. The default behavior is to ignore ' - 'same-sized items unless the local version is newer ' - 'than the S3 version.')} - INDEX_DOCUMENT = {'name': 'index-document', 'help_text': ( 'A suffix that is appended to a request that is for ' @@ -226,8 +211,6 @@ CACHE_CONTROL, CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_LANGUAGE, EXPIRES, SOURCE_REGION, ONLY_SHOW_ERRORS] -SYNC_ARGS = [DELETE, EXACT_TIMESTAMPS, SIZE_ONLY] + TRANSFER_ARGS - def get_endpoint(service, region, endpoint_url, verify): return service.get_endpoint(region_name=region, endpoint_url=endpoint_url, @@ -455,7 +438,7 @@ class SyncCommand(S3TransferCommand): USAGE = " or " \ " or " ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True, - 'synopsis': USAGE}] + SYNC_ARGS + 'synopsis': USAGE}] + TRANSFER_ARGS EXAMPLES = BasicCommand.FROM_FILE('s3/sync.rst') @@ -535,6 +518,33 @@ def needs_filegenerator(self): return False else: return True + + def choose_sync_strategies(self): + """Determines the sync strategy for the command. + + It defaults to the default sync strategies but a customizable sync + strategy can overide the default strategy if it returns the instance + of its self when the event is emitted. + """ + sync_strategies = {} + # Set the default strategies. + sync_strategies['file_at_src_and_dest_sync_strategy'] = \ + SizeAndLastModifiedSync() + sync_strategies['file_not_at_dest_sync_strategy'] = MissingFileSync() + sync_strategies['file_not_at_src_sync_strategy'] = NeverSync() + + # Determine what strategies to overide if any. + responses = self.session.emit( + 'choosing-s3-sync-strategy', params=self.parameters) + if responses is not None: + for response in responses: + override_sync_strategy = response[1] + if override_sync_strategy is not None: + sync_type = override_sync_strategy.sync_type + sync_type += '_sync_strategy' + sync_strategies[sync_type] = override_sync_strategy + + return sync_strategies def run(self): """ @@ -609,6 +619,8 @@ def run(self): s3_stream_handler = S3StreamHandler(self.session, self.parameters, result_queue=result_queue) + sync_strategies = self.choose_sync_strategies() + command_dict = {} if self.cmd == 'sync': command_dict = {'setup': [files, rev_files], @@ -616,7 +628,7 @@ def run(self): rev_generator], 'filters': [create_filter(self.parameters), create_filter(self.parameters)], - 'comparator': [Comparator(self.parameters)], + 'comparator': [Comparator(**sync_strategies)], 'file_info_builder': [file_info_builder], 's3_handler': [s3handler]} elif self.cmd == 'cp' and self.parameters['is_stream']: diff --git a/awscli/customizations/s3/syncstrategy/__init__.py b/awscli/customizations/s3/syncstrategy/__init__.py new file mode 100644 index 000000000000..0b90e827322e --- /dev/null +++ b/awscli/customizations/s3/syncstrategy/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. diff --git a/awscli/customizations/s3/syncstrategy/base.py b/awscli/customizations/s3/syncstrategy/base.py new file mode 100644 index 000000000000..e1e126ae6843 --- /dev/null +++ b/awscli/customizations/s3/syncstrategy/base.py @@ -0,0 +1,255 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import logging + + +LOG = logging.getLogger(__name__) + +VALID_SYNC_TYPES = ['file_at_src_and_dest', 'file_not_at_dest', + 'file_not_at_src'] + + +class BaseSync(object): + """Base sync strategy + + To create a new sync strategy, subclass from this class. + """ + + # This is the argument that will be added to the ``SyncCommand`` arg table. + # This argument will represent the sync strategy when the arguments for + # the sync command are parsed. ``ARGUMENT`` follows the same format as + # a member of ``ARG_TABLE`` in ``BasicCommand`` class as specified in + # ``awscli/customizations/commands.py``. + # + # For example, if I wanted to perform the sync strategy whenever I type + # ``--my-sync-strategy``, I would say: + # + # ARGUMENT = + # {'name': 'my-sync-strategy', 'action': 'store-true', + # 'help_text': 'Performs my sync strategy'} + # + # Typically, the argument's ``action`` should ``store_true`` to + # minimize amount of extra code in making a custom sync strategy. + ARGUMENT = None + + # At this point all that need to be done is implement + # ``determine_should_sync`` method (see method for more information). + + def __init__(self, sync_type='file_at_src_and_dest'): + """ + :type sync_type: string + :param sync_type: This determines where the sync strategy will be + used. There are three strings to choose from: + + 'file_at_src_and_dest': apply sync strategy on a file that + exists both at the source and the destination. + + 'file_not_at_dest': apply sync strategy on a file that + exists at the source but not the destination. + + 'file_not_at_src': apply sync strategy on a file that + exists at the destination but not the source. + """ + self._check_sync_type(sync_type) + self._sync_type = sync_type + + def _check_sync_type(self, sync_type): + if sync_type not in VALID_SYNC_TYPES: + raise ValueError("Unknown sync_type: %s.\n" + "Valid options are %s." % + (sync_type, VALID_SYNC_TYPES)) + + @property + def sync_type(self): + return self._sync_type + + def register_strategy(self, session): + """Registers the sync strategy class to the given session.""" + + session.register('building-arg-table.sync', + self.add_sync_argument) + session.register('choosing-s3-sync-strategy', self.use_sync_strategy) + + def determine_should_sync(self, src_file, dest_file): + """Subclasses should implement this method. + + This function takes two ``FileStat`` objects (one from the source and + one from the destination). Then makes a decision on whether a given + operation (e.g. a upload, copy, download) should be allowed + to take place. + + The function currently raises a ``NotImplementedError``. So this + method must be overwritten when this class is subclassed. Note + that this method must return a Boolean as documented below. + + :type src_file: ``FileStat`` object + :param src_file: A representation of the opertaion that is to be + performed on a specfic file existing in the source. Note if + the file does not exist at the source, ``src_file`` is None. + + :type dest_file: ``FileStat`` object + :param dest_file: A representation of the operation that is to be + performed on a specific file existing in the destination. Note if + the file does not exist at the destination, ``dest_file`` is None. + + :rtype: Boolean + :return: True if an operation based on the ``FileStat`` should be + allowed to occur. + False if if an operation based on the ``FileStat`` should not be + allowed to occur. Note the operation being referred to depends on + the ``sync_type`` of the sync strategy: + + 'file_at_src_and_dest': refers to ``src_file`` + + 'file_not_at_dest': refers to ``src_file`` + + 'file_not_at_src': refers to ``dest_file`` + """ + + raise NotImplementedError("determine_should_sync") + + @property + def arg_name(self): + # Retrieves the ``name`` of the sync strategy's ``ARGUMENT``. + name = None + if self.ARGUMENT is not None: + name = self.ARGUMENT.get('name', None) + return name + + @property + def arg_dest(self): + # Retrieves the ``dest`` of the sync strategy's ``ARGUMENT``. + dest = None + if self.ARGUMENT is not None: + dest = self.ARGUMENT.get('dest', None) + return dest + + def add_sync_argument(self, arg_table, **kwargs): + # This function adds sync strategy's argument to the ``SyncCommand`` + # argument table. + if self.ARGUMENT is not None: + arg_table.append(self.ARGUMENT) + + def use_sync_strategy(self, params, **kwargs): + # This function determines which sync strategy the ``SyncCommand`` will + # use. The sync strategy object must be returned by this method + # if it is to be chosen as the sync strategy to use. + # + # ``params`` is a dictionary that specifies all of the arguments + # the sync command is able to process as well as their values. + # + # Since ``ARGUMENT`` was added to the ``SyncCommand`` arg table, + # the argument will be present in ``params``. + # + # If the argument was included in the actual ``aws s3 sync`` command + # its value will show up as ``True`` in ``params`` otherwise its value + # will be ``False`` in ``params`` assuming the argument's ``action`` + # is ``store_true``. + # + # Note: If the ``action`` of ``ARGUMENT`` was not set to + # ``store_true``, this method will need to be overwritten. + # + name_in_params = None + # Check if a ``dest`` was specified in ``ARGUMENT`` as if it is + # specified, the boolean value will be located at the argument's + # ``dest`` value in the ``params`` dictionary. + if self.arg_dest is not None: + name_in_params = self.arg_dest + # Then check ``name`` of ``ARGUMENT``, the boolean value will be + # located at the argument's ``name`` value in the ``params`` + # dictionary. + elif self.arg_name is not None: + # ``name`` has all ``-`` replaced with ``_`` in ``params``. + name_in_params = self.arg_name.replace('-', '_') + if name_in_params is not None: + if params.get(name_in_params): + # Return the sync strategy object to be used for syncing. + return self + return None + + def total_seconds(self, td): + """ + timedelta's time_seconds() function for python 2.6 users + + :param td: The difference between two datetime objects. + """ + return (td.microseconds + (td.seconds + td.days * 24 * + 3600) * 10**6) / 10**6 + + def compare_size(self, src_file, dest_file): + """ + :returns: True if the sizes are the same. + False otherwise. + """ + return src_file.size == dest_file.size + + def compare_time(self, src_file, dest_file): + """ + :returns: True if the file does not need updating based on time of + last modification and type of operation. + False if the file does need updating based on the time of + last modification and type of operation. + """ + src_time = src_file.last_update + dest_time = dest_file.last_update + delta = dest_time - src_time + cmd = src_file.operation_name + if cmd == "upload" or cmd == "copy": + if self.total_seconds(delta) >= 0: + # Destination is newer than source. + return True + else: + # Destination is older than source, so + # we have a more recently updated file + # at the source location. + return False + elif cmd == "download": + + if self.total_seconds(delta) <= 0: + return True + else: + # delta is positive, so the destination + # is newer than the source. + return False + + +class SizeAndLastModifiedSync(BaseSync): + + def determine_should_sync(self, src_file, dest_file): + same_size = self.compare_size(src_file, dest_file) + same_last_modified_time = self.compare_time(src_file, dest_file) + should_sync = (not same_size) or (not same_last_modified_time) + if should_sync: + LOG.debug("syncing: %s -> %s, size_changed: %s, " + "last_modified_time_changed: %s", + src_file.src, src_file.dest, + not same_size, not same_last_modified_time) + return should_sync + + +class NeverSync(BaseSync): + def __init__(self, sync_type='file_not_at_src'): + super(NeverSync, self).__init__(sync_type) + + def determine_should_sync(self, src_file, dest_file): + return False + + +class MissingFileSync(BaseSync): + def __init__(self, sync_type='file_not_at_dest'): + super(MissingFileSync, self).__init__(sync_type) + + def determine_should_sync(self, src_file, dest_file): + LOG.debug("syncing: %s -> %s, file does not exist at destination", + src_file.src, src_file.dest) + return True diff --git a/awscli/customizations/s3/syncstrategy/delete.py b/awscli/customizations/s3/syncstrategy/delete.py new file mode 100644 index 000000000000..35e088cb2dec --- /dev/null +++ b/awscli/customizations/s3/syncstrategy/delete.py @@ -0,0 +1,36 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import logging + +from awscli.customizations.s3.syncstrategy.base import BaseSync + + +LOG = logging.getLogger(__name__) + + +DELETE = {'name': 'delete', 'action': 'store_true', + 'help_text': ( + "Files that exist in the destination but not in the source are " + "deleted during sync.")} + + +class DeleteSync(BaseSync): + + ARGUMENT = DELETE + + def determine_should_sync(self, src_file, dest_file): + dest_file.operation_name = 'delete' + LOG.debug("syncing: (None) -> %s (remove), file does not " + "exist at source (%s) and delete mode enabled", + dest_file.src, dest_file.dest) + return True diff --git a/awscli/customizations/s3/syncstrategy/exacttimestamps.py b/awscli/customizations/s3/syncstrategy/exacttimestamps.py new file mode 100644 index 000000000000..599341ed14b0 --- /dev/null +++ b/awscli/customizations/s3/syncstrategy/exacttimestamps.py @@ -0,0 +1,54 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import logging + +from awscli.customizations.s3.syncstrategy.base import BaseSync + + +LOG = logging.getLogger(__name__) + + +EXACT_TIMESTAMPS = {'name': 'exact-timestamps', 'action': 'store_true', + 'help_text': ( + 'When syncing from S3 to local, same-sized ' + 'items will be ignored only when the timestamps ' + 'match exactly. The default behavior is to ignore ' + 'same-sized items unless the local version is newer ' + 'than the S3 version.')} + + +class ExactTimestampsSync(BaseSync): + + ARGUMENT = EXACT_TIMESTAMPS + + def determine_should_sync(self, src_file, dest_file): + same_size = self.compare_size(src_file, dest_file) + same_last_modified_time = self.compare_time(src_file, dest_file) + should_sync = (not same_size) or (not same_last_modified_time) + if should_sync: + LOG.debug("syncing: %s -> %s, size_changed: %s, " + "last_modified_time_changed: %s", + src_file.src, src_file.dest, + not same_size, not same_last_modified_time) + return should_sync + + def compare_time(self, src_file, dest_file): + src_time = src_file.last_update + dest_time = dest_file.last_update + delta = dest_time - src_time + cmd = src_file.operation_name + if cmd == 'download': + return self.total_seconds(delta) == 0 + else: + return super(SizeOnlySyncStrategy, self).compare_time(src_file, + dest_file) diff --git a/awscli/customizations/s3/syncstrategy/register.py b/awscli/customizations/s3/syncstrategy/register.py new file mode 100644 index 000000000000..b75674dcb99b --- /dev/null +++ b/awscli/customizations/s3/syncstrategy/register.py @@ -0,0 +1,49 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from awscli.customizations.s3.syncstrategy.sizeonly import SizeOnlySync +from awscli.customizations.s3.syncstrategy.exacttimestamps import \ + ExactTimestampsSync +from awscli.customizations.s3.syncstrategy.delete import DeleteSync + + +def register_sync_strategy(session, strategy_cls, + sync_type='file_at_src_and_dest'): + """Registers a single sync strategy + + :param session: The session that the sync strategy is being registered to. + :param strategy_cls: The class of the sync strategy to be registered. + :param sync_type: A string representing when to perform the sync strategy. + See ``__init__`` method of ``BaseSyncStrategy`` for possible options. + """ + strategy = strategy_cls(sync_type) + strategy.register_strategy(session) + + +def register_sync_strategies(command_table, session, **kwargs): + """Registers the different sync strategies. + + To register a sync strategy add + ``register_sync_strategy(session, YourSyncStrategyClass, sync_type)`` + to the list of registered strategies in this function. + """ + + # Register the size only sync strategy. + register_sync_strategy(session, SizeOnlySync) + + # Register the exact timestamps sync strategy. + register_sync_strategy(session, ExactTimestampsSync) + + # Register the delete sync strategy. + register_sync_strategy(session, DeleteSync, 'file_not_at_src') + + # Register additional sync strategies here... diff --git a/awscli/customizations/s3/syncstrategy/sizeonly.py b/awscli/customizations/s3/syncstrategy/sizeonly.py new file mode 100644 index 000000000000..e83d0fd7be5d --- /dev/null +++ b/awscli/customizations/s3/syncstrategy/sizeonly.py @@ -0,0 +1,37 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import logging + +from awscli.customizations.s3.syncstrategy.base import BaseSync + + +LOG = logging.getLogger(__name__) + + +SIZE_ONLY = {'name': 'size-only', 'action': 'store_true', + 'help_text': ( + 'Makes the size of each key the only criteria used to ' + 'decide whether to sync from source to destination.')} + + +class SizeOnlySync(BaseSync): + + ARGUMENT = SIZE_ONLY + + def determine_should_sync(self, src_file, dest_file): + same_size = self.compare_size(src_file, dest_file) + should_sync = not same_size + if should_sync: + LOG.debug("syncing: %s -> %s, size_changed: %s", + src_file.src, src_file.dest, not same_size) + return should_sync diff --git a/tests/unit/customizations/s3/syncstrategy/__init__.py b/tests/unit/customizations/s3/syncstrategy/__init__.py new file mode 100644 index 000000000000..0b90e827322e --- /dev/null +++ b/tests/unit/customizations/s3/syncstrategy/__init__.py @@ -0,0 +1,12 @@ +# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. diff --git a/tests/unit/customizations/s3/syncstrategy/test_base.py b/tests/unit/customizations/s3/syncstrategy/test_base.py new file mode 100644 index 000000000000..bf5df14201bc --- /dev/null +++ b/tests/unit/customizations/s3/syncstrategy/test_base.py @@ -0,0 +1,296 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import datetime + +from mock import Mock, patch + +from awscli.customizations.s3.filegenerator import FileStat +from awscli.customizations.s3.syncstrategy.base import BaseSync, \ + SizeAndLastModifiedSync, MissingFileSync, NeverSync +from awscli.testutils import unittest + + +class TestBaseSync(unittest.TestCase): + def setUp(self): + self.sync_strategy = BaseSync() + + def test_init(self): + valid_sync_types = ['file_at_src_and_dest', 'file_not_at_dest', + 'file_not_at_src'] + for sync_type in valid_sync_types: + strategy = BaseSync(sync_type) + self.assertEqual(strategy.sync_type, sync_type) + + # Check for invalid ``sync_type`` options. + with self.assertRaises(ValueError): + BaseSync('wrong_sync_type') + + def test_register_strategy(self): + """ + Ensures that the class registers all of the necessary handlers + """ + session = Mock() + self.sync_strategy.register_strategy(session) + register_args = session.register.call_args_list + self.assertEqual(register_args[0][0][0], + 'building-arg-table.sync') + self.assertEqual(register_args[0][0][1], + self.sync_strategy.add_sync_argument) + self.assertEqual(register_args[1][0][0], 'choosing-s3-sync-strategy') + self.assertEqual(register_args[1][0][1], + self.sync_strategy.use_sync_strategy) + + def test_determine_should_sync(self): + """ + Ensure that this class cannot be directly used as the sync strategy. + """ + with self.assertRaises(NotImplementedError): + self.sync_strategy.determine_should_sync(None, None) + + def test_arg_name(self): + """ + Ensure that the ``arg_name`` property works as expected. + """ + self.assertEqual(self.sync_strategy.arg_name, None) + self.sync_strategy.ARGUMENT = {'name': 'my-sync-strategy'} + self.assertEqual(self.sync_strategy.arg_name, 'my-sync-strategy') + + def test_arg_dest(self): + """ + Ensure that the ``arg_dest`` property works as expected. + """ + self.assertEqual(self.sync_strategy.arg_dest, None) + self.sync_strategy.ARGUMENT = {'dest': 'my-dest'} + self.assertEqual(self.sync_strategy.arg_dest, 'my-dest') + + def test_add_sync_argument(self): + """ + Ensures the sync argument is properly added to the + the command's ``arg_table``. + """ + arg_table = [{'name': 'original_argument'}] + self.sync_strategy.ARGUMENT = {'name': 'sync_argument'} + self.sync_strategy.add_sync_argument(arg_table) + self.assertEqual(arg_table, + [{'name': 'original_argument'}, + {'name': 'sync_argument'}]) + + def test_no_add_sync_argument_for_no_argument_specified(self): + """ + Ensures nothing is added to the command's ``arg_table`` if no + ``ARGUMENT`` table is specified. + """ + arg_table = [{'name': 'original_argument'}] + self.sync_strategy.add_sync_argument(arg_table) + self.assertEqual(arg_table, [{'name': 'original_argument'}]) + + def test_no_use_sync_strategy_for_no_argument_specified(self): + """ + Test if that the sync strategy is not returned if it has no argument. + """ + params = {'my_sync_strategy': True} + self.assertEqual(self.sync_strategy.use_sync_strategy(params), None) + + def test_use_sync_strategy_for_name_and_no_dest(self): + """ + Test if sync strategy argument has ``name`` but no ``dest`` and the + strategy was called in ``params``. + """ + self.sync_strategy.ARGUMENT = {'name': 'my-sync-strategy'} + params = {'my_sync_strategy': True} + self.assertEqual(self.sync_strategy.use_sync_strategy(params), + self.sync_strategy) + + def test_no_use_sync_strategy_for_name_and_no_dest(self): + """ + Test if sync strategy argument has ``name`` but no ``dest`` but + the strategy was not called in ``params``. + """ + self.sync_strategy.ARGUMENT = {'name': 'my-sync-strategy'} + params = {'my_sync_strategy': False} + self.assertEqual(self.sync_strategy.use_sync_strategy(params), None) + + def test_no_use_sync_strategy_for_not_in_params(self): + """ + Test if sync strategy argument has a ``name`` but for whatever reason + the strategy is not in ``params``. + """ + self.sync_strategy.ARGUMENT = {'name': 'my-sync-strategy'} + self.assertEqual(self.sync_strategy.use_sync_strategy({}), None) + + def test_use_sync_strategy_for_name_and_dest(self): + """ + Test if sync strategy argument has ``name`` and ``dest`` and the + strategy was called in ``params``. + """ + self.sync_strategy.ARGUMENT = {'name': 'my-sync-strategy', + 'dest': 'my-dest'} + params = {'my-dest': True} + self.assertEqual(self.sync_strategy.use_sync_strategy(params), + self.sync_strategy) + + def test_no_use_sync_strategy_for_name_and_dest(self): + """ + Test if sync strategy argument has ``name`` and ``dest`` but the + the strategy was not called in ``params``. + """ + self.sync_strategy.ARGUMENT = {'name': 'my-sync-strategy', + 'dest': 'my-dest'} + params = {'my-dest': False} + self.assertEqual(self.sync_strategy.use_sync_strategy(params), None) + + def test_no_use_sync_strategy_for_dest_but_only_name_in_params(self): + """ + Test if sync strategy argument has ``name`` and ``dest`` but the + the strategy was not called in ``params`` even though the ``name`` was + called in ``params``. + """ + self.sync_strategy.ARGUMENT = {'name': 'my-sync-strategy', + 'dest': 'my-dest'} + params = {'my-sync-strategy': True} + self.assertEqual(self.sync_strategy.use_sync_strategy(params), None) + + +class TestSizeAndLastModifiedSync(unittest.TestCase): + def setUp(self): + self.sync_strategy = SizeAndLastModifiedSync() + + def test_compare_size(self): + """ + Confirms compare size works. + """ + time = datetime.datetime.now() + src_file = FileStat(src='', dest='', + compare_key='comparator_test.py', size=11, + last_update=time, src_type='local', + dest_type='s3', operation_name='upload') + dest_file = FileStat(src='', dest='', + compare_key='comparator_test.py', size=10, + last_update=time, src_type='s3', + dest_type='local', operation_name='') + should_sync = self.sync_strategy.determine_should_sync( + src_file, dest_file) + self.assertTrue(should_sync) + + def test_compare_lastmod_upload(self): + """ + Confirms compare time works for uploads. + """ + time = datetime.datetime.now() + future_time = time + datetime.timedelta(0, 3) + src_file = FileStat(src='', dest='', + compare_key='comparator_test.py', size=10, + last_update=future_time, src_type='local', + dest_type='s3', operation_name='upload') + dest_file = FileStat(src='', dest='', + compare_key='comparator_test.py', size=10, + last_update=time, src_type='s3', + dest_type='local', operation_name='') + should_sync = self.sync_strategy.determine_should_sync( + src_file, dest_file) + self.assertTrue(should_sync) + + def test_compare_lastmod_copy(self): + """ + Confirms compare time works for copies. + """ + time = datetime.datetime.now() + future_time = time + datetime.timedelta(0, 3) + src_file = FileStat(src='', dest='', + compare_key='comparator_test.py', size=10, + last_update=future_time, src_type='s3', + dest_type='s3', operation_name='copy') + dest_file = FileStat(src='', dest='', + compare_key='comparator_test.py', size=10, + last_update=time, src_type='s3', + dest_type='s3', operation_name='') + should_sync = self.sync_strategy.determine_should_sync( + src_file, dest_file) + self.assertTrue(should_sync) + + def test_compare_lastmod_download(self): + """ + Confirms compare time works for downloads. + """ + time = datetime.datetime.now() + future_time = time + datetime.timedelta(0, 3) + src_file = FileStat(src='', dest='', + compare_key='comparator_test.py', size=10, + last_update=time, src_type='s3', + dest_type='local', operation_name='download') + dest_file = FileStat(src='', dest='', + compare_key='comparator_test.py', size=10, + last_update=future_time, src_type='local', + dest_type='s3', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + src_file, dest_file) + self.assertTrue(should_sync) + + # If the source is newer than the destination do not download. + src_file = FileStat(src='', dest='', + compare_key='comparator_test.py', size=10, + last_update=future_time, src_type='s3', + dest_type='local', operation_name='download') + dest_file = FileStat(src='', dest='', + compare_key='comparator_test.py', size=10, + last_update=time, src_type='local', + dest_type='s3', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + src_file, dest_file) + self.assertFalse(should_sync) + + +class TestNeverSync(unittest.TestCase): + def setUp(self): + self.sync_strategy = NeverSync() + + def test_constructor(self): + self.assertEqual(self.sync_strategy.sync_type, 'file_not_at_src') + + def test_determine_should_sync(self): + time_dst = datetime.datetime.now() + + dst_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_dst, src_type='s3', + dest_type='local', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + None, dst_file) + self.assertFalse(should_sync) + + +class TestMissingFileSync(unittest.TestCase): + def setUp(self): + self.sync_strategy = MissingFileSync() + + def test_constructor(self): + self.assertEqual(self.sync_strategy.sync_type, 'file_not_at_dest') + + def test_determine_should_sync(self): + time_src = datetime.datetime.now() + + src_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_src, src_type='s3', + dest_type='local', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + src_file, None) + self.assertTrue(should_sync) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/customizations/s3/syncstrategy/test_delete.py b/tests/unit/customizations/s3/syncstrategy/test_delete.py new file mode 100644 index 000000000000..9294783cfa98 --- /dev/null +++ b/tests/unit/customizations/s3/syncstrategy/test_delete.py @@ -0,0 +1,40 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import datetime + +from awscli.customizations.s3.filegenerator import FileStat +from awscli.customizations.s3.syncstrategy.delete import DeleteSync + +from awscli.testutils import unittest + + +class TestDeleteSync(unittest.TestCase): + def setUp(self): + self.sync_strategy = DeleteSync() + + def test_determine_should_sync(self): + timenow = datetime.datetime.now() + + dst_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=timenow, src_type='local', + dest_type='s3', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + None, dst_file) + self.assertTrue(should_sync) + self.assertEqual(dst_file.operation_name, 'delete') + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/customizations/s3/syncstrategy/test_exacttimestamps.py b/tests/unit/customizations/s3/syncstrategy/test_exacttimestamps.py new file mode 100644 index 000000000000..480f5de25e91 --- /dev/null +++ b/tests/unit/customizations/s3/syncstrategy/test_exacttimestamps.py @@ -0,0 +1,118 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import datetime + +from awscli.customizations.s3.filegenerator import FileStat +from awscli.customizations.s3.syncstrategy.exacttimestamps import \ + ExactTimestampsSync + +from awscli.testutils import unittest + + +class TestExactTimestampsSync(unittest.TestCase): + def setUp(self): + self.sync_strategy = ExactTimestampsSync() + + def test_compare_exact_timestamps_dest_older(self): + """ + Confirm that same-sized files are synced when + the destination is older than the source and + `exact_timestamps` is set. + """ + time_src = datetime.datetime.now() + time_dst = time_src - datetime.timedelta(days=1) + + src_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_src, src_type='s3', + dest_type='local', operation_name='download') + + dst_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_dst, src_type='local', + dest_type='s3', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + src_file, dst_file) + self.assertTrue(should_sync) + + def test_compare_exact_timestamps_src_older(self): + """ + Confirm that same-sized files are synced when + the source is older than the destination and + `exact_timestamps` is set. + """ + time_src = datetime.datetime.now() - datetime.timedelta(days=1) + time_dst = datetime.datetime.now() + + src_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_src, src_type='s3', + dest_type='local', operation_name='download') + + dst_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_dst, src_type='local', + dest_type='s3', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + src_file, dst_file) + self.assertTrue(should_sync) + + def test_compare_exact_timestamps_same_age_same_size(self): + """ + Confirm that same-sized files are not synced when + the source and destination are the same age and + `exact_timestamps` is set. + """ + time_both = datetime.datetime.now() + + src_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_both, src_type='s3', + dest_type='local', operation_name='download') + + dst_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_both, src_type='local', + dest_type='s3', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + src_file, dst_file) + self.assertFalse(should_sync) + + def test_compare_exact_timestamps_same_age_diff_size(self): + """ + Confirm that files of differing sizes are synced when + the source and destination are the same age and + `exact_timestamps` is set. + """ + time_both = datetime.datetime.now() + + src_file = FileStat(src='', dest='', + compare_key='test.py', size=20, + last_update=time_both, src_type='s3', + dest_type='local', operation_name='download') + + dst_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_both, src_type='local', + dest_type='s3', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + src_file, dst_file) + self.assertTrue(should_sync) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/customizations/s3/syncstrategy/test_register.py b/tests/unit/customizations/s3/syncstrategy/test_register.py new file mode 100644 index 000000000000..2000c08c24c4 --- /dev/null +++ b/tests/unit/customizations/s3/syncstrategy/test_register.py @@ -0,0 +1,53 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from mock import Mock + +from awscli.customizations.s3.syncstrategy.register import \ + register_sync_strategy +from awscli.testutils import unittest + + +class TestRegisterSyncStrategy(unittest.TestCase): + def setUp(self): + self.session = Mock() + self.strategy_cls = Mock() + self.strategy_object = Mock() + self.strategy_cls.return_value = self.strategy_object + + def test_register_sync_strategy(self): + """ + Ensure that registering a single strategy class works as expected + when ``sync_type`` is specified. + """ + register_sync_strategy(self.session, self.strategy_cls, 'sync_type') + # Ensure sync strategy class is instantiated + self.strategy_cls.assert_called_with('sync_type') + # Ensure the sync strategy's ``register_strategy`` method is + # called correctly. + self.strategy_object.register_strategy.assert_called_with(self.session) + + def test_register_sync_strategy_default_sync_type(self): + """ + Ensure that registering a single strategy class works as expected + when the ``sync_type`` is not specified. + """ + register_sync_strategy(self.session, self.strategy_cls) + # Ensure sync strategy class is instantiated + self.strategy_cls.assert_called_with('file_at_src_and_dest') + # Ensure the sync strategy's ``register_strategy`` method is + # called correctly. + self.strategy_object.register_strategy.assert_called_with(self.session) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/customizations/s3/syncstrategy/test_sizeonly.py b/tests/unit/customizations/s3/syncstrategy/test_sizeonly.py new file mode 100644 index 000000000000..21557295aa11 --- /dev/null +++ b/tests/unit/customizations/s3/syncstrategy/test_sizeonly.py @@ -0,0 +1,70 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import datetime + +from awscli.customizations.s3.filegenerator import FileStat +from awscli.customizations.s3.syncstrategy.sizeonly import SizeOnlySync + +from awscli.testutils import unittest + + +class TestSizeOnlySync(unittest.TestCase): + def setUp(self): + self.sync_strategy = SizeOnlySync() + + def test_compare_size_only(self): + """ + Confirm that files are synced when size differs. + """ + time_src = datetime.datetime.now() + time_dst = time_src + datetime.timedelta(days=1) + + src_file = FileStat(src='', dest='', + compare_key='test.py', size=11, + last_update=time_src, src_type='local', + dest_type='s3', operation_name='upload') + + dst_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_dst, src_type='s3', + dest_type='local', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + src_file, dst_file) + self.assertTrue(should_sync) + + def test_compare_size_only_different_update_times(self): + """ + Confirm that files with the same size but different update times + are not synced. + """ + time_src = datetime.datetime.now() + time_dst = time_src + datetime.timedelta(days=1) + + src_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_src, src_type='local', + dest_type='s3', operation_name='upload') + + dst_file = FileStat(src='', dest='', + compare_key='test.py', size=10, + last_update=time_dst, src_type='s3', + dest_type='local', operation_name='') + + should_sync = self.sync_strategy.determine_should_sync( + src_file, dst_file) + self.assertFalse(should_sync) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/customizations/s3/test_comparator.py b/tests/unit/customizations/s3/test_comparator.py index b8909b248e82..7aa404b33548 100644 --- a/tests/unit/customizations/s3/test_comparator.py +++ b/tests/unit/customizations/s3/test_comparator.py @@ -13,18 +13,29 @@ import datetime import unittest +from mock import Mock + from awscli.customizations.s3.comparator import Comparator from awscli.customizations.s3.filegenerator import FileStat class ComparatorTest(unittest.TestCase): def setUp(self): - self.comparator = Comparator({'delete': True}) + self.sync_strategy = Mock() + self.not_at_src_sync_strategy = Mock() + self.not_at_dest_sync_strategy = Mock() + self.comparator = Comparator(self.sync_strategy, + self.not_at_dest_sync_strategy, + self.not_at_src_sync_strategy) - def test_compare_key_equal(self): + def test_compare_key_equal_should_not_sync(self): """ - Confirms checking compare key works. + Confirm the appropriate action is taken when the soruce compare key + is equal to the destination compare key. """ + # Try when the sync strategy says not to sync the file. + self.sync_strategy.determine_should_sync.return_value = False + src_files = [] dest_files = [] ref_list = [] @@ -45,44 +56,35 @@ def test_compare_key_equal(self): result_list.append(filename) self.assertEqual(result_list, ref_list) - def test_compare_size(self): - """ - Confirms compare size works. - """ - src_files = [] - dest_files = [] + # Try when the sync strategy says to sync the file. + self.sync_strategy.determine_should_sync.return_value = True + ref_list = [] result_list = [] - time = datetime.datetime.now() - src_file = FileStat(src='', dest='', - compare_key='comparator_test.py', size=11, - last_update=time, src_type='local', - dest_type='s3', operation_name='upload') - dest_file = FileStat(src='', dest='', - compare_key='comparator_test.py', size=10, - last_update=time, src_type='s3', - dest_type='local', operation_name='') - src_files.append(src_file) - dest_files.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) ref_list.append(src_file) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list) - def test_compare_lastmod_upload(self): + def test_compare_key_less(self): """ - Confirms compare time works for uploads. + Confirm the appropriate action is taken when the soruce compare key + is less than the destination compare key. """ + self.not_at_src_sync_strategy.determine_should_sync.return_value = False + + # Try when the sync strategy says to sync the file. + self.not_at_dest_sync_strategy.determine_should_sync.return_value = True + src_files = [] dest_files = [] ref_list = [] result_list = [] time = datetime.datetime.now() - future_time = time + datetime.timedelta(0, 3) src_file = FileStat(src='', dest='', - compare_key='comparator_test.py', size=10, - last_update=future_time, src_type='local', + compare_key='bomparator_test.py', size=10, + last_update=time, src_type='local', dest_type='s3', operation_name='upload') dest_file = FileStat(src='', dest='', compare_key='comparator_test.py', size=10, @@ -90,95 +92,39 @@ def test_compare_lastmod_upload(self): dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) - files = self.comparator.call(iter(src_files), iter(dest_files)) ref_list.append(src_file) - for filename in files: - result_list.append(filename) - self.assertEqual(result_list, ref_list) - - def test_compare_lastmod_copy(self): - """ - Confirms compare time works for copies - """ - src_files = [] - dest_files = [] - ref_list = [] - result_list = [] - time = datetime.datetime.now() - future_time = time + datetime.timedelta(0, 3) - src_file = FileStat(src='', dest='', - compare_key='comparator_test.py', size=10, - last_update=future_time, src_type='s3', - dest_type='s3', operation_name='copy') - dest_file = FileStat(src='', dest='', - compare_key='comparator_test.py', size=10, - last_update=time, src_type='s3', - dest_type='s3', operation_name='') - src_files.append(src_file) - dest_files.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) - ref_list.append(src_file) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list) - def test_compare_lastmod_download(self): - """ - Confirms compare time works for downloads. - """ - src_files = [] - dest_files = [] - ref_list = [] + # Now try when the sync strategy says not to sync the file. + self.not_at_dest_sync_strategy.determine_should_sync.return_value = False result_list = [] - time = datetime.datetime.now() - future_time = time + datetime.timedelta(0, 3) - src_file = FileStat(src='', dest='', - compare_key='comparator_test.py', size=10, - last_update=time, src_type='s3', - dest_type='local', operation_name='download') - dest_file = FileStat(src='', dest='', - compare_key='comparator_test.py', size=10, - last_update=future_time, src_type='local', - dest_type='s3', operation_name='') - src_files.append(src_file) - dest_files.append(dest_file) + ref_list = [] files = self.comparator.call(iter(src_files), iter(dest_files)) - ref_list.append(src_file) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list) - # If the source is newer than the destination do not download. - src_file = FileStat(src='', dest='', - compare_key='comparator_test.py', size=10, - last_update=future_time, src_type='s3', - dest_type='local', operation_name='download') - dest_file = FileStat(src='', dest='', - compare_key='comparator_test.py', size=10, - last_update=time, src_type='local', - dest_type='s3', operation_name='') - src_files = [] - dest_files = [] - src_files.append(src_file) - dest_files.append(dest_file) - files = self.comparator.call(iter(src_files), iter(dest_files)) - result_list = [] - for filename in files: - result_list.append(filename) - self.assertEqual(result_list, []) - def test_compare_key_less(self): + def test_compare_key_greater(self): """ Confirm the appropriate action is taken when the soruce compare key - is less than the destination compare key. + is greater than the destination compare key. """ + self.not_at_dest_sync_strategy.determine_should_sync.return_value = False + + # Try when the sync strategy says to sync the file. + self.not_at_src_sync_strategy.determine_should_sync.return_value = True + src_files = [] dest_files = [] ref_list = [] result_list = [] time = datetime.datetime.now() src_file = FileStat(src='', dest='', - compare_key='bomparator_test.py', size=10, + compare_key='domparator_test.py', size=10, last_update=time, src_type='local', dest_type='s3', operation_name='upload') dest_file = FileStat(src='', dest='', @@ -187,48 +133,30 @@ def test_compare_key_less(self): dest_type='local', operation_name='') src_files.append(src_file) dest_files.append(dest_file) - dest_file.operation = 'delete' - ref_list.append(src_file) ref_list.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list) - def test_compare_key_greater(self): - """ - Confirm the appropriate action is taken when the soruce compare key - is greater than the destination compare key. - """ - src_files = [] - dest_files = [] - ref_list = [] + # Now try when the sync strategy says not to sync the file. + self.not_at_src_sync_strategy.determine_should_sync.return_value = False result_list = [] - time = datetime.datetime.now() - src_file = FileStat(src='', dest='', - compare_key='domparator_test.py', size=10, - last_update=time, src_type='local', - dest_type='s3', operation_name='upload') - dest_file = FileStat(src='', dest='', - compare_key='comparator_test.py', size=10, - last_update=time, src_type='s3', - dest_type='local', operation_name='') - src_files.append(src_file) - dest_files.append(dest_file) - src_file.operation = 'upload' - dest_file.operation = 'delete' - ref_list.append(dest_file) - ref_list.append(src_file) + ref_list = [] files = self.comparator.call(iter(src_files), iter(dest_files)) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list) + def test_empty_src(self): """ Confirm the appropriate action is taken when there are no more source files to take. """ + # Try when the sync strategy says to sync the file. + self.not_at_src_sync_strategy.determine_should_sync.return_value = True + src_files = [] dest_files = [] ref_list = [] @@ -239,18 +167,29 @@ def test_empty_src(self): last_update=time, src_type='s3', dest_type='local', operation_name='') dest_files.append(dest_file) - dest_file.operation = 'delete' ref_list.append(dest_file) files = self.comparator.call(iter(src_files), iter(dest_files)) for filename in files: result_list.append(filename) self.assertEqual(result_list, ref_list) + # Now try when the sync strategy says not to sync the file. + self.not_at_src_sync_strategy.determine_should_sync.return_value = False + result_list = [] + ref_list = [] + files = self.comparator.call(iter(src_files), iter(dest_files)) + for filename in files: + result_list.append(filename) + self.assertEqual(result_list, ref_list) + def test_empty_dest(self): """ Confirm the appropriate action is taken when there are no more dest files to take. """ + # Try when the sync strategy says to sync the file. + self.not_at_dest_sync_strategy.determine_should_sync.return_value = True + src_files = [] dest_files = [] ref_list = [] @@ -267,6 +206,16 @@ def test_empty_dest(self): result_list.append(filename) self.assertEqual(result_list, ref_list) + # Now try when the sync strategy says not to sync the file. + self.not_at_dest_sync_strategy.determine_should_sync.return_value = False + result_list = [] + ref_list = [] + files = self.comparator.call(iter(src_files), iter(dest_files)) + for filename in files: + result_list.append(filename) + self.assertEqual(result_list, ref_list) + + def test_empty_src_dest(self): """ Confirm the appropriate action is taken when there are no more @@ -282,143 +231,5 @@ def test_empty_src_dest(self): self.assertEqual(result_list, ref_list) -class ComparatorSizeOnlyTest(unittest.TestCase): - def setUp(self): - self.comparator = Comparator({'delete': True, 'size_only': True}) - - def test_compare_size_only_dest_older_than_src(self): - """ - Confirm that files with the same size but different update times are not - synced when `size_only` is set. - """ - time_src = datetime.datetime.now() - time_dst = time_src + datetime.timedelta(days=1) - - src_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_src, src_type='local', - dest_type='s3', operation_name='upload') - - dst_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_dst, src_type='s3', - dest_type='local', operation_name='') - - files = self.comparator.call(iter([src_file]), iter([dst_file])) - self.assertEqual(sum(1 for _ in files), 0) - - def test_compare_size_only_src_older_than_dest(self): - """ - Confirm that files with the same size but different update times are not - synced when `size_only` is set. - """ - time_dst = datetime.datetime.now() - time_src = time_dst + datetime.timedelta(days=1) - - src_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_src, src_type='local', - dest_type='s3', operation_name='upload') - - dst_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_dst, src_type='s3', - dest_type='local', operation_name='') - - files = self.comparator.call(iter([src_file]), iter([dst_file])) - self.assertEqual(sum(1 for _ in files), 0) - - -class ComparatorExactTimestampsTest(unittest.TestCase): - def setUp(self): - self.comparator = Comparator({'exact_timestamps': True}) - - def test_compare_exact_timestamps_dest_older(self): - """ - Confirm that same-sized files are synced when - the destination is older than the source and - `exact_timestamps` is set. - """ - time_src = datetime.datetime.now() - time_dst = time_src - datetime.timedelta(days=1) - - src_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_src, src_type='s3', - dest_type='local', operation_name='download') - - dst_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_dst, src_type='local', - dest_type='s3', operation_name='') - - files = self.comparator.call(iter([src_file]), iter([dst_file])) - self.assertEqual(sum(1 for _ in files), 1) - - def test_compare_exact_timestamps_src_older(self): - """ - Confirm that same-sized files are synced when - the source is older than the destination and - `exact_timestamps` is set. - """ - time_src = datetime.datetime.now() - datetime.timedelta(days=1) - time_dst = datetime.datetime.now() - - src_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_src, src_type='s3', - dest_type='local', operation_name='download') - - dst_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_dst, src_type='local', - dest_type='s3', operation_name='') - - files = self.comparator.call(iter([src_file]), iter([dst_file])) - self.assertEqual(sum(1 for _ in files), 1) - - def test_compare_exact_timestamps_same_age_same_size(self): - """ - Confirm that same-sized files are not synced when - the source and destination are the same age and - `exact_timestamps` is set. - """ - time_both = datetime.datetime.now() - - src_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_both, src_type='s3', - dest_type='local', operation_name='download') - - dst_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_both, src_type='local', - dest_type='s3', operation_name='') - - files = self.comparator.call(iter([src_file]), iter([dst_file])) - self.assertEqual(sum(1 for _ in files), 0) - - def test_compare_exact_timestamps_same_age_diff_size(self): - """ - Confirm that files of differing sizes are synced when - the source and destination are the same age and - `exact_timestamps` is set. - """ - time_both = datetime.datetime.now() - - src_file = FileStat(src='', dest='', - compare_key='test.py', size=20, - last_update=time_both, src_type='s3', - dest_type='local', operation_name='download') - - dst_file = FileStat(src='', dest='', - compare_key='test.py', size=10, - last_update=time_both, src_type='local', - dest_type='s3', operation_name='') - - files = self.comparator.call(iter([src_file]), iter([dst_file])) - self.assertEqual(sum(1 for _ in files), 1) - - if __name__ == "__main__": unittest.main() diff --git a/tests/unit/customizations/s3/test_s3.py b/tests/unit/customizations/s3/test_s3.py index 31915835b1b5..2dbdd7aae3c4 100644 --- a/tests/unit/customizations/s3/test_s3.py +++ b/tests/unit/customizations/s3/test_s3.py @@ -28,6 +28,7 @@ def test_initialize(self): awscli_initialize(self.cli) reference = [] reference.append("building-command-table.main") + reference.append("building-command-table.sync") for arg in self.cli.register.call_args_list: self.assertIn(arg[0][0], reference) diff --git a/tests/unit/customizations/s3/test_subcommands.py b/tests/unit/customizations/s3/test_subcommands.py index fab4076b2752..872c9bdc1183 100644 --- a/tests/unit/customizations/s3/test_subcommands.py +++ b/tests/unit/customizations/s3/test_subcommands.py @@ -16,12 +16,14 @@ import sys import mock -from mock import patch, MagicMock +from mock import patch, Mock, MagicMock import botocore.session from awscli.customizations.s3.s3 import S3 from awscli.customizations.s3.subcommands import CommandParameters, \ CommandArchitecture, CpCommand, SyncCommand, ListCommand, get_endpoint +from awscli.customizations.s3.syncstrategy.base import \ + SizeAndLastModifiedSync, NeverSync, MissingFileSync from awscli.testutils import unittest, BaseAWSHelpOutputTest from tests.unit.customizations.s3 import make_loc_files, clean_loc_files, \ make_s3_files, s3_cleanup, S3HandlerBaseTest @@ -191,6 +193,65 @@ def test_create_instructions(self): 'file_info_builder', 's3_handler']) + def test_choose_sync_strategy_default(self): + session = Mock() + cmd_arc = CommandArchitecture(session, 'sync', + {'region': 'us-east-1', + 'endpoint_url': None, + 'verify_ssl': None}) + # Check if no plugins return their sync strategy. Should + # result in the default strategies + session.emit.return_value = None + sync_strategies = cmd_arc.choose_sync_strategies() + self.assertEqual( + sync_strategies['file_at_src_and_dest_sync_strategy'].__class__, + SizeAndLastModifiedSync + ) + self.assertEqual( + sync_strategies['file_not_at_dest_sync_strategy'].__class__, + MissingFileSync + ) + self.assertEqual( + sync_strategies['file_not_at_src_sync_strategy'].__class__, + NeverSync + ) + + def test_choose_sync_strategy_overwrite(self): + session = Mock() + cmd_arc = CommandArchitecture(session, 'sync', + {'region': 'us-east-1', + 'endpoint_url': None, + 'verify_ssl': None}) + # Check that the default sync strategy is overwritted if a plugin + # returns its sync strategy. + mock_strategy = Mock() + mock_strategy.sync_type = 'file_at_src_and_dest' + + mock_not_at_dest_sync_strategy = Mock() + mock_not_at_dest_sync_strategy.sync_type = 'file_not_at_dest' + + mock_not_at_src_sync_strategy = Mock() + mock_not_at_src_sync_strategy.sync_type = 'file_not_at_src' + + responses = [(None, mock_strategy), + (None, mock_not_at_dest_sync_strategy), + (None, mock_not_at_src_sync_strategy)] + + session.emit.return_value = responses + sync_strategies = cmd_arc.choose_sync_strategies() + self.assertEqual( + sync_strategies['file_at_src_and_dest_sync_strategy'], + mock_strategy + ) + self.assertEqual( + sync_strategies['file_not_at_dest_sync_strategy'], + mock_not_at_dest_sync_strategy + ) + self.assertEqual( + sync_strategies['file_not_at_src_sync_strategy'], + mock_not_at_src_sync_strategy + ) + def test_run_cp_put(self): # This ensures that the architecture sets up correctly for a ``cp`` put # command. It is just just a dry run, but all of the components need @@ -538,6 +599,7 @@ def test_s3command_help(self): # parts. Note the examples are not included because # the event was not registered. s3command = CpCommand(self.session) + s3command._arg_table = s3command._build_arg_table() parser = argparse.ArgumentParser() parser.add_argument('--paginate', action='store_true') parsed_global = parser.parse_args(['--paginate']) diff --git a/tests/unit/customizations/test_commands.py b/tests/unit/customizations/test_commands.py index 837c86ac6898..a8248c5fa6df 100644 --- a/tests/unit/customizations/test_commands.py +++ b/tests/unit/customizations/test_commands.py @@ -29,3 +29,29 @@ def test_basic_help_with_contents(self): mock_open.return_value.__enter__.return_value.read.return_value = \ 'fake description' self.assertEqual(help_command.description, 'fake description') + + +class TestBasicCommand(unittest.TestCase): + def setUp(self): + self.session = mock.Mock() + self.command = BasicCommand(self.session) + + def test_load_arg_table_property(self): + # Ensure ``_build_arg_table()`` is called if it has not been + # built via the ``arg_table`` property. It should be an empty + # dictionary. + orig_arg_table = self.command.arg_table + self.assertEqual(orig_arg_table, {}) + # Ensure the ``arg_table`` is not built again if + # ``arg_table`` property is called again. + self.assertIs(orig_arg_table, self.command.arg_table) + + def test_load_subcommand_table_property(self): + # Ensure ``_build_subcommand_table()`` is called if it has not + # been built via the ``subcommand_table`` property. It should be + # an empty dictionary. + orig_subcommand_table = self.command.subcommand_table + self.assertEqual(orig_subcommand_table, {}) + # Ensure the ``subcommand_table`` is not built again if + # ``subcommand_table`` property is called again. + self.assertIs(orig_subcommand_table, self.command.subcommand_table)