Merge pull request #172 from jedwards4b/update_manage_externals

bring in latest manage_externals

manage_externals/.gitignore

@@ -12,3 +12,6 @@ components/
 
 # generated python files
 *.pyc
+
+# test tmp file
+test/tmp

manage_externals/.travis.yml

@@ -1,7 +1,6 @@
 language: python
 os: linux
-python: 
-  - "2.7"
+python:
   - "3.4"
   - "3.5"
   - "3.6"

manage_externals/manic/checkout.py

@@ -279,6 +279,9 @@ def commandline_arguments(args=None):
                         help='The externals description filename. '
                         'Default: %(default)s.')
 
+    parser.add_argument('-x', '--exclude', nargs='*',
+                        help='Component(s) listed in the externals file which should be ignored.')
+
     parser.add_argument('-o', '--optional', action='store_true', default=False,
                         help='By default only the required externals '
                         'are checked out. This flag will also checkout the '
@@ -362,7 +365,7 @@ def main(args):
     root_dir = os.path.abspath(os.getcwd())
     external_data = read_externals_description_file(root_dir, args.externals)
     external = create_externals_description(
-        external_data, components=args.components)
+        external_data, components=args.components, exclude=args.exclude)
 
     for comp in args.components:
         if comp not in external.keys():
@@ -377,29 +380,37 @@ def main(args):
 
     if args.status:
         # user requested status-only
-        for comp in sorted(tree_status.keys()):
+        for comp in sorted(tree_status):
             tree_status[comp].log_status_message(args.verbose)
     else:
         # checkout / update the external repositories.
         safe_to_update = check_safe_to_update_repos(tree_status)
         if not safe_to_update:
             # print status
-            for comp in sorted(tree_status.keys()):
+            for comp in sorted(tree_status):
                 tree_status[comp].log_status_message(args.verbose)
             # exit gracefully
             msg = """The external repositories labeled with 'M' above are not in a clean state.
 
-The following are two options for how to proceed:
+The following are three options for how to proceed:
 
-(1) Go into each external that is not in a clean state and issue either
-    an 'svn status' or a 'git status' command. Either revert or commit
-    your changes so that all externals are in a clean state. (Note,
-    though, that it is okay to have untracked files in your working
+(1) Go into each external that is not in a clean state and issue either a 'git status' or
+    an 'svn status' command (depending on whether the external is managed by git or
+    svn). Either revert or commit your changes so that all externals are in a clean
+    state. (To revert changes in git, follow the instructions given when you run 'git
+    status'.) (Note, though, that it is okay to have untracked files in your working
     directory.) Then rerun {program_name}.
 
-(2) Alternatively, you do not have to rely on {program_name}. Instead, you
-    can manually update out-of-sync externals (labeled with 's' above)
-    as described in the configuration file {config_file}.
+(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually
+    update out-of-sync externals (labeled with 's' above) as described in the
+    configuration file {config_file}. (For example, run 'git fetch' and 'git checkout'
+    commands to checkout the appropriate tags for each external, as given in
+    {config_file}.)
+
+(3) You can also use {program_name} to manage most, but not all externals: You can specify
+    one or more externals to ignore using the '-x' or '--exclude' argument to
+    {program_name}. Excluding externals labeled with 'M' will allow {program_name} to
+    update the other, non-excluded externals.
 
 
 The external repositories labeled with '?' above are not under version

manage_externals/manic/externals_description.py

@@ -180,6 +180,9 @@ def parse_submodules_desc_section(section_items, file_path):
             path = item[1].strip()
         elif name == 'url':
             url = item[1].strip()
+        elif name == 'branch':
+            # We do not care about branch since we have a hash - silently ignore
+            pass
         else:
             msg = 'WARNING: Ignoring unknown {} property, in {}'
             msg = msg.format(item[0], file_path) # fool pylint
@@ -250,9 +253,21 @@ def read_gitmodules_file(root_dir, file_name):
                                           ExternalsDescription.REPO_URL, url)
                 externals_description.set(sec_name,
                                           ExternalsDescription.REQUIRED, 'True')
-                git_hash = submods[sec_name]['hash']
-                externals_description.set(sec_name,
-                                          ExternalsDescription.HASH, git_hash)
+                if sec_name in submods:
+                    submod_name = sec_name
+                else:
+                    # The section name does not have to match the path
+                    submod_name = path
+
+                if submod_name in submods:
+                    git_hash = submods[submod_name]['hash']
+                    externals_description.set(sec_name,
+                                              ExternalsDescription.HASH,
+                                              git_hash)
+                else:
+                    emsg = "submodule status has no section, '{}'"
+                    emsg += "\nCheck section names in externals config file"
+                    fatal_error(emsg.format(submod_name))
 
         # Required items
         externals_description.add_section(DESCRIPTION_SECTION)
@@ -261,18 +276,18 @@ def read_gitmodules_file(root_dir, file_name):
     return externals_description
 
 def create_externals_description(
-        model_data, model_format='cfg', components=None, parent_repo=None):
+        model_data, model_format='cfg', components=None, exclude=None, parent_repo=None):
     """Create the a externals description object from the provided data
     """
     externals_description = None
     if model_format == 'dict':
         externals_description = ExternalsDescriptionDict(
-            model_data, components=components)
+            model_data, components=components, exclude=exclude)
     elif model_format == 'cfg':
         major, _, _ = get_cfg_schema_version(model_data)
         if major == 1:
             externals_description = ExternalsDescriptionConfigV1(
-                model_data, components=components, parent_repo=parent_repo)
+                model_data, components=components, exclude=exclude, parent_repo=parent_repo)
         else:
             msg = ('Externals description file has unsupported schema '
                    'version "{0}".'.format(major))
@@ -707,7 +722,7 @@ class ExternalsDescriptionDict(ExternalsDescription):
 
     """
 
-    def __init__(self, model_data, components=None):
+    def __init__(self, model_data, components=None, exclude=None):
         """Parse a native dictionary into a externals description.
         """
         ExternalsDescription.__init__(self)
@@ -719,10 +734,15 @@ def __init__(self, model_data, components=None):
         self._input_patch = 0
         self._verify_schema_version()
         if components:
-            for key in model_data.items():
+            for key in list(model_data.keys()):
                 if key not in components:
                     del model_data[key]
 
+        if exclude:
+            for key in list(model_data.keys()):
+                if key in exclude:
+                    del model_data[key]
+
         self.update(model_data)
         self._check_user_input()
 
@@ -733,7 +753,7 @@ class ExternalsDescriptionConfigV1(ExternalsDescription):
 
     """
 
-    def __init__(self, model_data, components=None, parent_repo=None):
+    def __init__(self, model_data, components=None, exclude=None, parent_repo=None):
         """Convert the config data into a standardized dict that can be used to
         construct the source objects
 
@@ -746,7 +766,7 @@ def __init__(self, model_data, components=None, parent_repo=None):
             get_cfg_schema_version(model_data)
         self._verify_schema_version()
         self._remove_metadata(model_data)
-        self._parse_cfg(model_data, components=components)
+        self._parse_cfg(model_data, components=components, exclude=exclude)
         self._check_user_input()
 
     @staticmethod
@@ -758,7 +778,7 @@ def _remove_metadata(model_data):
         """
         model_data.remove_section(DESCRIPTION_SECTION)
 
-    def _parse_cfg(self, cfg_data, components=None):
+    def _parse_cfg(self, cfg_data, components=None, exclude=None):
         """Parse a config_parser object into a externals description.
         """
         def list_to_dict(input_list, convert_to_lower_case=True):
@@ -775,7 +795,7 @@ def list_to_dict(input_list, convert_to_lower_case=True):
 
         for section in cfg_data.sections():
             name = config_string_cleaner(section.lower().strip())
-            if components and name not in components:
+            if (components and name not in components) or (exclude and name in exclude):
                 continue
             self[name] = {}
             self[name].update(list_to_dict(cfg_data.items(section)))

manage_externals/manic/sourcetree.py

@@ -331,12 +331,14 @@ def checkout(self, verbosity, load_all, load_comp=None):
             printlog('Checking out externals: ', end='')
 
         if load_all:
-            load_comps = self._all_components.keys()
+            tmp_comps = self._all_components.keys()
         elif load_comp is not None:
-            load_comps = [load_comp]
+            tmp_comps = [load_comp]
         else:
-            load_comps = self._required_compnames
-
+            tmp_comps = self._required_compnames
+        # Sort by path so that if paths are nested the
+        # parent repo is checked out first.
+        load_comps = sorted(tmp_comps, key=lambda comp: self._all_components[comp].get_local_path())
         # checkout the primary externals
         for comp in load_comps:
             if verbosity < VERBOSITY_VERBOSE:
@@ -346,8 +348,6 @@ def checkout(self, verbosity, load_all, load_comp=None):
                 # output a newline
                 printlog(EMPTY_STR)
             self._all_components[comp].checkout(verbosity, load_all)
-        printlog('')
-
-        # now give each external an opportunitity to checkout it's externals.
-        for comp in load_comps:
+            # now give each external an opportunitity to checkout it's externals.
             self._all_components[comp].checkout_externals(verbosity, load_all)
+        printlog('')