Add support for git sparse checkout

Added a new, optional external property, sparse. sparse points to a file (relative to external repository local directory) with git sparse checkout patterns (e.g., directory names to include).
If the sparse keyword is included, enable sparse checkout for that repository and use git read-tree to perform the sparse checkout.

Added a new test for sparse checkout. The test is a repo with two externals which are both tag2 of the simple_ext repo with one of them configured for a sparse checkout. The test makes sure that both externals have the correct files.

Updated the documentation for the externals configuration file.

User interface changes?: Yes
New sparse keyword is optional with a blank default value which preserves current functionality.

Fixes: #120

Testing:
test removed: None
unit tests: All passed
system tests: All passed, including new test for sparse feature
manual testing: Several manual tests of NASA test cases.

README.md

@@ -85,7 +85,7 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     description file:
 
         $ cd ${SRC_ROOT}
-        $ ./manage_externals/checkout_externals --excernals my-externals.cfg
+        $ ./manage_externals/checkout_externals --externals my-externals.cfg
 
   * Status summary of the repositories managed by checkout_externals:
 
@@ -202,6 +202,21 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     Then the main 'externals' field in the top level repo should point to
     'sub-externals.cfg'.
 
+  * from_submodule (True / False) : used to pull the repo_url, local_path,
+    and hash properties for this external from the .gitmodules file in
+    this repository. Note that the section name (the entry in square
+    brackets) must match the name in the .gitmodules file.
+    If from_submodule is True, the protocol must be git and no repo_url,
+    local_path, hash, branch, or tag entries are allowed.
+    Default: False
+
+  * sparse (string) : used to control a sparse checkout. This optional
+    entry should point to a filename (path relative to local_path) that
+    contains instructions on which repository paths to include (or
+    exclude) from the working tree.
+    See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree
+    Default: sparse checkout is disabled
+
   * Lines begining with '#' or ';' are comments and will be ignored.
 
 # Obtaining this tool, reporting issues, etc.

manic/checkout.py

@@ -227,6 +227,21 @@ def commandline_arguments(args=None):
     Now, %(prog)s will process Externals.cfg and also process
     Externals_LIBX.cfg as if it was a sub-external.
 
+  * from_submodule (True / False) : used to pull the repo_url, local_path,
+    and hash properties for this external from the .gitmodules file in
+    this repository. Note that the section name (the entry in square
+    brackets) must match the name in the .gitmodules file.
+    If from_submodule is True, the protocol must be git and no repo_url,
+    local_path, hash, branch, or tag entries are allowed.
+    Default: False
+
+  * sparse (string) : used to control a sparse checkout. This optional
+    entry should point to a filename (path relative to local_path) that
+    contains instructions on which repository paths to include (or
+    exclude) from the working tree.
+    See the "SPARSE CHECKOUT" section of https://git-scm.com/docs/git-read-tree
+    Default: sparse checkout is disabled
+
   * Lines beginning with '#' or ';' are comments and will be ignored.
 
 # Obtaining this tool, reporting issues, etc.

manic/externals_description.py

@@ -351,6 +351,7 @@ class ExternalsDescription(dict):
     REPO_URL = 'repo_url'
     REQUIRED = 'required'
     TAG = 'tag'
+    SPARSE = 'sparse'
 
     PROTOCOL_EXTERNALS_ONLY = 'externals_only'
     PROTOCOL_GIT = 'git'
@@ -374,6 +375,7 @@ class ExternalsDescription(dict):
                              TAG: 'string',
                              BRANCH: 'string',
                              HASH: 'string',
+                             SPARSE: 'string',
                             }
                      }
 
@@ -562,6 +564,8 @@ def _check_optional(self):
                 self[field][self.REPO][self.HASH] = EMPTY_STR
             if self.REPO_URL not in self[field][self.REPO]:
                 self[field][self.REPO][self.REPO_URL] = EMPTY_STR
+            if self.SPARSE not in self[field][self.REPO]:
+                self[field][self.REPO][self.SPARSE] = EMPTY_STR
 
             # from_submodule has a complex relationship with other fields
             if self.SUBMODULE in self[field]:

manic/repository.py

@@ -21,6 +21,7 @@ def __init__(self, component_name, repo):
         self._branch = repo[ExternalsDescription.BRANCH]
         self._hash = repo[ExternalsDescription.HASH]
         self._url = repo[ExternalsDescription.REPO_URL]
+        self._sparse = repo[ExternalsDescription.SPARSE]
 
         if self._url is EMPTY_STR:
             fatal_error('repo must have a URL')

manic/repository_git.py

@@ -316,8 +316,11 @@ def _checkout_ref(self, repo_dir, verbosity, submodules):
         else:
             self._checkout_external_ref(verbosity, submodules)
 
+        if self._sparse:
+            self._sparse_checkout(repo_dir, verbosity)
         os.chdir(cwd)
 
+
     def _checkout_local_ref(self, verbosity, submodules):
         """Checkout the reference considering the local repo only. Do not
         fetch any additional remotes or specify the remote when
@@ -362,6 +365,20 @@ def _checkout_external_ref(self, verbosity, submodules):
             ref = '{0}/{1}'.format(remote_name, ref)
         self._git_checkout_ref(ref, verbosity, submodules)
 
+    def _sparse_checkout(self, repo_dir, verbosity):
+        """Use git read-tree to thin the working tree."""
+        cwd = os.getcwd()
+
+        cmd = ['cp', self._sparse, os.path.join(repo_dir,
+                                                '.git/info/sparse-checkout')]
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+        os.chdir(repo_dir)
+        self._git_sparse_checkout(verbosity)
+
+        os.chdir(cwd)
+
     def _check_for_valid_ref(self, ref, remote_name=None):
         """Try some basic sanity checks on the user supplied reference so we
         can provide a more useful error message than calledprocess
@@ -776,6 +793,18 @@ def _git_checkout_ref(ref, verbosity, submodules):
         if submodules:
             GitRepository._git_update_submodules(verbosity)
 
+    @staticmethod
+    def _git_sparse_checkout(verbosity):
+        """Configure repo via read-tree."""
+        cmd = ['git', 'config', 'core.sparsecheckout', 'true']
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+        cmd = ['git', 'read-tree', '-mu', 'HEAD']
+        if verbosity >= VERBOSITY_VERBOSE:
+            printlog('    {0}'.format(' '.join(cmd)))
+        execute_subprocess(cmd)
+
     @staticmethod
     def _git_update_submodules(verbosity):
         """Run git submodule update for the side effect of updating this

manic/sourcetree.py

@@ -45,6 +45,7 @@ def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry):
         self._externals = EMPTY_STR
         self._externals_sourcetree = None
         self._stat = ExternalStatus()
+        self._sparse = None
         # Parse the sub-elements
 
         # _path : local path relative to the containing source tree

test/repos/simple-ext.git/refs/heads/master

@@ -1 +1 @@
-9b75494003deca69527bb64bcaa352e801611dd2
+607ec299c17dd285c029edc41a0109e49d441380

test/repos/simple-ext.git/refs/tags/tag2

@@ -0,0 +1 @@
+b7692b6d391899680da7b9b6fd8af4c413f06fe7

test/test_sys_checkout.py

@@ -88,6 +88,8 @@
 
 SVN_TEST_REPO = 'https://github.com/escomp/cesm'
 
+# Disable too-many-public-methods error
+# pylint: disable=R0904
 
 def setUpModule():  # pylint: disable=C0103
     """Setup for all tests in this module. It is called once per module!
@@ -183,6 +185,25 @@ def container_simple_svn(self, dest_dir):
 
         self.write_config(dest_dir)
 
+    def container_sparse(self, dest_dir):
+        """Create a container with a full external and a sparse external
+
+        """
+        # Create a file for a sparse pattern match
+        sparse_filename = 'sparse_checkout'
+        with open(os.path.join(dest_dir, sparse_filename), 'w') as sfile:
+            sfile.write('readme.txt')
+
+        self.create_config()
+        self.create_section(SIMPLE_REPO_NAME, 'simp_tag',
+                            tag='tag2')
+
+        sparse_relpath = '../../{}'.format(sparse_filename)
+        self.create_section(SIMPLE_REPO_NAME, 'simp_sparse',
+                            tag='tag2', sparse=sparse_relpath)
+
+        self.write_config(dest_dir)
+
     def mixed_simple_base(self, dest_dir):
         """Create a mixed-use base externals file with only simple externals.
 
@@ -239,7 +260,8 @@ def create_metadata(self):
 
     def create_section(self, repo_type, name, tag='', branch='',
                        ref_hash='', required=True, path=EXTERNALS_NAME,
-                       externals='', repo_path=None, from_submodule=False):
+                       externals='', repo_path=None, from_submodule=False,
+                       sparse=''):
         # pylint: disable=too-many-branches
         """Create a config section with autofilling some items and handling
         optional items.
@@ -287,6 +309,9 @@ def create_section(self, repo_type, name, tag='', branch='',
         if externals:
             self._config.set(name, ExternalsDescription.EXTERNALS, externals)
 
+        if sparse:
+            self._config.set(name, ExternalsDescription.SPARSE, sparse)
+
         if from_submodule:
             self._config.set(name, ExternalsDescription.SUBMODULE, "True")
 
@@ -710,6 +735,14 @@ def _check_mixed_ext_branch_modified(self, tree, directory=EXTERNALS_NAME):
         name = './{0}/mixed_req'.format(directory)
         self._check_generic_modified_ok_required(tree, name)
 
+    def _check_simple_sparse_empty(self, tree, directory=EXTERNALS_NAME):
+        name = './{0}/simp_sparse'.format(directory)
+        self._check_generic_empty_default_required(tree, name)
+
+    def _check_simple_sparse_ok(self, tree, directory=EXTERNALS_NAME):
+        name = './{0}/simp_sparse'.format(directory)
+        self._check_generic_ok_clean_required(tree, name)
+
     # ----------------------------------------------------------------
     #
     # Check results for groups of externals under specific conditions
@@ -870,6 +903,23 @@ def _check_mixed_cont_simple_required_post_checkout(self, overall, tree):
         self._check_simple_branch_ok(tree, directory=EXTERNALS_NAME)
         self._check_simple_branch_ok(tree, directory=SUB_EXTERNALS_PATH)
 
+    def _check_container_sparse_pre_checkout(self, overall, tree):
+        self.assertEqual(overall, 0)
+        self._check_simple_tag_empty(tree)
+        self._check_simple_sparse_empty(tree)
+
+    def _check_container_sparse_post_checkout(self, overall, tree):
+        self.assertEqual(overall, 0)
+        self._check_simple_tag_ok(tree)
+        self._check_simple_sparse_ok(tree)
+
+    def _check_file_exists(self, repo_dir, pathname):
+        "Check that <pathname> exists in <repo_dir>"
+        self.assertTrue(os.path.exists(os.path.join(repo_dir, pathname)))
+
+    def _check_file_absent(self, repo_dir, pathname):
+        "Check that <pathname> does not exist in <repo_dir>"
+        self.assertFalse(os.path.exists(os.path.join(repo_dir, pathname)))
 
 class TestSysCheckout(BaseTestSysCheckout):
     """Run systems level tests of checkout_externals
@@ -1234,6 +1284,14 @@ def test_container_full(self):
                                                 self.status_args)
         self._check_container_full_post_checkout(overall, tree)
 
+        # Check existance of some files
+        subrepo_path = os.path.join('externals', 'simp_tag')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_absent(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+
         # update the mixed-use repo to point to different branch
         self._generator.update_branch(under_test_dir, 'mixed_req',
                                       'new-feature', MIXED_REPO_NAME)
@@ -1314,6 +1372,40 @@ def test_mixed_simple(self):
                                                 self.status_args)
         self._check_mixed_cont_simple_required_post_checkout(overall, tree)
 
+    def test_container_sparse(self):
+        """Verify that 'full' container with simple subrepo
+        can run a sparse checkout and generate the correct initial status.
+
+        """
+        # create the test repository
+        under_test_dir = self.setup_test_repo(CONTAINER_REPO_NAME)
+
+        # create the top level externals file
+        self._generator.container_sparse(under_test_dir)
+
+        # inital checkout
+        overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                self.checkout_args)
+        self._check_container_sparse_pre_checkout(overall, tree)
+
+        overall, tree = self.execute_cmd_in_dir(under_test_dir,
+                                                self.status_args)
+        self._check_container_sparse_post_checkout(overall, tree)
+
+        # Check existance of some files
+        subrepo_path = os.path.join('externals', 'simp_tag')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_exists(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+        subrepo_path = os.path.join('externals', 'simp_sparse')
+        self._check_file_exists(under_test_dir,
+                                os.path.join(subrepo_path, 'readme.txt'))
+        self._check_file_absent(under_test_dir, os.path.join(subrepo_path,
+                                                             'simple_subdir',
+                                                             'subdir_file.txt'))
+
 
 class TestSysCheckoutSVN(BaseTestSysCheckout):
     """Run systems level tests of checkout_externals accessing svn repositories

test/test_unit_repository.py

@@ -36,7 +36,8 @@ def setUp(self):
                       ExternalsDescription.REPO_URL: 'junk_root',
                       ExternalsDescription.TAG: 'junk_tag',
                       ExternalsDescription.BRANCH: EMPTY_STR,
-                      ExternalsDescription.HASH: EMPTY_STR, }
+                      ExternalsDescription.HASH: EMPTY_STR,
+                      ExternalsDescription.SPARSE: EMPTY_STR, }
 
     def test_create_repo_git(self):
         """Verify that several possible names for the 'git' protocol
@@ -95,7 +96,8 @@ def test_tag(self):
                      ExternalsDescription.REPO_URL: url,
                      ExternalsDescription.TAG: tag,
                      ExternalsDescription.BRANCH: EMPTY_STR,
-                     ExternalsDescription.HASH: EMPTY_STR, }
+                     ExternalsDescription.HASH: EMPTY_STR,
+                     ExternalsDescription.SPARSE: EMPTY_STR, }
         repo = Repository(name, repo_info)
         print(repo.__dict__)
         self.assertEqual(repo.tag(), tag)
@@ -112,7 +114,8 @@ def test_branch(self):
                      ExternalsDescription.REPO_URL: url,
                      ExternalsDescription.BRANCH: branch,
                      ExternalsDescription.TAG: EMPTY_STR,
-                     ExternalsDescription.HASH: EMPTY_STR, }
+                     ExternalsDescription.HASH: EMPTY_STR,
+                     ExternalsDescription.SPARSE: EMPTY_STR, }
         repo = Repository(name, repo_info)
         print(repo.__dict__)
         self.assertEqual(repo.branch(), branch)
@@ -125,11 +128,13 @@ def test_hash(self):
         protocol = 'test_protocol'
         url = 'test_url'
         ref = 'deadc0de'
+        sparse = EMPTY_STR
         repo_info = {ExternalsDescription.PROTOCOL: protocol,
                      ExternalsDescription.REPO_URL: url,
                      ExternalsDescription.BRANCH: EMPTY_STR,
                      ExternalsDescription.TAG: EMPTY_STR,
-                     ExternalsDescription.HASH: ref, }
+                     ExternalsDescription.HASH: ref,
+                     ExternalsDescription.SPARSE: sparse, }
         repo = Repository(name, repo_info)
         print(repo.__dict__)
         self.assertEqual(repo.hash(), ref)
@@ -146,11 +151,13 @@ def test_tag_branch(self):
         branch = 'test_branch'
         tag = 'test_tag'
         ref = EMPTY_STR
+        sparse = EMPTY_STR
         repo_info = {ExternalsDescription.PROTOCOL: protocol,
                      ExternalsDescription.REPO_URL: url,
                      ExternalsDescription.BRANCH: branch,
                      ExternalsDescription.TAG: tag,
-                     ExternalsDescription.HASH: ref, }
+                     ExternalsDescription.HASH: ref,
+                     ExternalsDescription.SPARSE: sparse, }
         with self.assertRaises(RuntimeError):
             Repository(name, repo_info)
 
@@ -165,11 +172,13 @@ def test_tag_branch_hash(self):
         branch = 'test_branch'
         tag = 'test_tag'
         ref = 'deadc0de'
+        sparse = EMPTY_STR
         repo_info = {ExternalsDescription.PROTOCOL: protocol,
                      ExternalsDescription.REPO_URL: url,
                      ExternalsDescription.BRANCH: branch,
                      ExternalsDescription.TAG: tag,
-                     ExternalsDescription.HASH: ref, }
+                     ExternalsDescription.HASH: ref,
+                     ExternalsDescription.SPARSE: sparse, }
         with self.assertRaises(RuntimeError):
             Repository(name, repo_info)
 
@@ -184,11 +193,13 @@ def test_no_tag_no_branch(self):
         branch = EMPTY_STR
         tag = EMPTY_STR
         ref = EMPTY_STR
+        sparse = EMPTY_STR
         repo_info = {ExternalsDescription.PROTOCOL: protocol,
                      ExternalsDescription.REPO_URL: url,
                      ExternalsDescription.BRANCH: branch,
                      ExternalsDescription.TAG: tag,
-                     ExternalsDescription.HASH: ref, }
+                     ExternalsDescription.HASH: ref,
+                     ExternalsDescription.SPARSE: sparse, }
         with self.assertRaises(RuntimeError):
             Repository(name, repo_info)