From da8224fc171b1e2d46093f6315470759db87aea3 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Fri, 3 Mar 2023 16:09:23 -0700 Subject: [PATCH] Squashed 'manage_externals/' changes from 82d3b247f..7b6d92ef6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 7b6d92ef6 Merge pull request #198 from johnpaulalex/gitdir 927ce3a98 Merge pull request #197 from johnpaulalex/testpath a04f1148f Merge pull request #196 from johnpaulalex/readmod d9c14bf25 Change the rest of the methods to use -C. Still some usage of getcwd in test_unit_repository_git. 332b10640 Fix incorrect logged path of checkout_externals in test_sys_checkout: it was basically the parent of the current directory, which varies throughout the test. (it called abspath with '{0}/../../', which adds arbitrary and not-interpolated subdir '{0}' to the path, then removes it and removes one more level). 932a7499b Remove printlog from read_gitmodules_file since read_externals_description_file() already has a nearly-the-same printlog (but add it to the other caller). 5d13719ed Merge pull request #195 from johnpaulalex/check_repo 423395449 Update utest to mock _git_remote_verbose in a new way, since it is now called via the GitRepository class rather than on the specific GitRepository instance. d7a42ae96 Check that desired repo was actually checked out. 71596bbc1 Merge pull request #194 from johnpaulalex/manic2 4c96e824e Make the MANIC_TEST_BARE_REPO_ROOT env var special - give it a constant for easy tracking, and automatically tear it down after each test. 259bfc04d test_sys_checkout: use actual paths in on-the-fly configs rather than MANIC_TEST_BARE_REPO_ROOT env var. This will make it easier to test (in the near future) that checkout_externals actually checked out the desired repo dir. 557bbd6eb Merge pull request #193 from johnpaulalex/manic 5314eede1 Remove MANIC_TEST_TMP_REPO_ROOT environment variable in favor of module-level variable. 345fc1e14 Merge pull request #191 from johnpaulalex/test_doc12 2117b843c test_sys_checkout: verify that basic by-tag/branch/hash tests actually take us to the correct git tag/branch/hash. 94d6e5f2b Merge pull request #190 from johnpaulalex/test_doc11 3ff33a6a8 Inline local-path-creation methods 47dea7f64 Merge pull request #189 from johnpaulalex/test_doc10 9ea75cbf8 Grab-bag of renamings: Remove redundant _NAME from repo constants, and consistently add _REPO suffix (This causes the majority of diffs). c0c847ec8 Merge pull request #188 from johnpaulalex/test_doc9 2dd5ce0f7 test_sys_checkout.py: only check for correct 'required' or 'optional' state in the test that exercises required vs optional behavior. Removed a lot of boilerplate. eb3085984 Merge pull request #187 from johnpaulalex/test_doc8 1832e1f84 test_sys_checkout: Simplify many tests to only use a single external. 8689d61ec Merge pull request #186 from johnpaulalex/test_doc7 fbee4253e Grab bag of test_sys_checkout cleanups: Doc inside of each test more clearly/consistently. TestSysCheckoutSVN didn’t get the inlining-of-helper-methods treatment, now it has that. Move various standalone repo helper methods (like create_branch) into a RepoUtils class. README.md was missing newlines when rendered as markdown. Doc the return value of checkout.main Fix test_container_exclude_component - it was looking for the wrong key (which is never present); now it looks for the correct key. f0ed44a6e Merge pull request #185 from johnpaulalex/test_doc6 a3d59f5f2 Merge pull request #184 from johnpaulalex/test_doc5 5329c8ba7 test_sys_checkout: Inline config generation functions that are only called once. 464f2c7a7 test_sys_checkout: Inline another layer (per-config-file checks). Rename the 4 methods that are used multiple times, to reflect what they do rather than what they're called. 8872c0df6 Merge pull request #183 from johnpaulalex/doc_test4 c045335f6 Merge pull request #182 from johnpaulalex/doc_test3 c583b956e Merge pull request #181 from johnpaulalex/doc_test2 e01cfe278 test_sys_checkout: less confusing handling of return values from checkout_externals. Specifically, when doing a checkout, don't return tree_status from _before_ the checkout. Make a new wrapper to call checkout_externals a second time, to calculate the new status after a checkout (very frequent pattern). 23286818c test_sys_checkout: Remove another layer (which generates test component names) c3717b6bc Merge pull request #180 from johnpaulalex/doc_test 36d7a4434 test_sys_checkout.py: remove one layer of functions (that check for local status enums). No-op. 2c4584bf7 More documentation about tests: * contents of test repositories (n a new README.md) * various constants in test_sys_checkout.py that point to those contents, and terminology like container/simple/mixed. * in each test method, the scenarios being tested. * The coupling between test methods. 55e74bd0a Merge pull request #179 from johnpaulalex/circ 66be84290 Remove circular dependency by making _External stop doing tricky things with sourcetrees. git-subtree-dir: manage_externals git-subtree-split: 7b6d92ef689e2f65733e27f8635ab91fb341356b --- manic/checkout.py | 8 +- manic/externals_description.py | 20 +- manic/externals_status.py | 4 +- manic/repository_git.py | 315 +++-- manic/sourcetree.py | 249 ++-- test/repos/README.md | 33 + test/test_sys_checkout.py | 2182 ++++++++++++++---------------- test/test_sys_repository_git.py | 58 +- test/test_unit_repository_git.py | 133 +- 9 files changed, 1458 insertions(+), 1544 deletions(-) create mode 100644 test/repos/README.md diff --git a/manic/checkout.py b/manic/checkout.py index ac30f3a5d2..3f5537adce 100755 --- a/manic/checkout.py +++ b/manic/checkout.py @@ -378,9 +378,9 @@ def main(args): the --all option is passed. Returns a tuple (overall_status, tree_status). overall_status is 0 - on success, non-zero on failure. tree_status gives the full status - *before* executing the checkout command - i.e., the status that it - used to determine if it's safe to proceed with the checkout. + on success, non-zero on failure. tree_status is a dict mapping local path + to ExternalStatus -- if no checkout is happening. If checkout is happening, tree_status + is None. """ if args.do_logging: logging.basicConfig(filename=LOG_FILE_NAME, @@ -438,6 +438,8 @@ def main(args): for comp in args.components: source_tree.checkout(args.verbose, load_all, load_comp=comp) printlog('') + # New tree status is unknown, don't return anything. + tree_status = None logging.info('%s completed without exceptions.', program_name) # NOTE(bja, 2017-11) tree status is used by the systems tests diff --git a/manic/externals_description.py b/manic/externals_description.py index f5615b6730..546e7fdcb4 100644 --- a/manic/externals_description.py +++ b/manic/externals_description.py @@ -88,7 +88,7 @@ def read_externals_description_file(root_dir, file_name): externals_description = None if file_name == ExternalsDescription.GIT_SUBMODULES_FILENAME: - externals_description = read_gitmodules_file(root_dir, file_name) + externals_description = _read_gitmodules_file(root_dir, file_name) else: try: config = config_parser() @@ -151,9 +151,8 @@ def git_submodule_status(repo_dir): """Run the git submodule status command to obtain submodule hashes. """ # This function is here instead of GitRepository to avoid a dependency loop - cwd = os.getcwd() - os.chdir(repo_dir) - cmd = ['git', 'submodule', 'status'] + cmd = 'git -C {repo_dir} submodule status'.format( + repo_dir=repo_dir).split() git_output = execute_subprocess(cmd, output_to_caller=True) submodules = {} submods = git_output.split('\n') @@ -168,7 +167,6 @@ def git_submodule_status(repo_dir): submodules[items[1]] = {'hash':items[0], 'status':status, 'tag':tag} - os.chdir(cwd) return submodules def parse_submodules_desc_section(section_items, file_path): @@ -191,7 +189,7 @@ def parse_submodules_desc_section(section_items, file_path): return path, url -def read_gitmodules_file(root_dir, file_name): +def _read_gitmodules_file(root_dir, file_name): # pylint: disable=deprecated-method # Disabling this check because the method is only used for python2 # pylint: disable=too-many-locals @@ -203,12 +201,11 @@ def read_gitmodules_file(root_dir, file_name): root_dir = os.path.abspath(root_dir) msg = 'In directory : {0}'.format(root_dir) logging.info(msg) - printlog('Processing submodules description file : {0}'.format(file_name)) file_path = os.path.join(root_dir, file_name) if not os.path.exists(file_name): msg = ('ERROR: submodules description file, "{0}", does not ' - 'exist at path:\n {1}'.format(file_name, file_path)) + 'exist in dir:\n {1}'.format(file_name, root_dir)) fatal_error(msg) submodules_description = None @@ -640,8 +637,11 @@ def _repo_config_from_submodule(self, field, submod_desc): ' Parent repo, "{1}" does not have submodules') fatal_error(msg.format(field, self._parent_repo.name())) - submod_file = read_gitmodules_file(repo_path, submod_file) - submod_desc = create_externals_description(submod_file) + printlog( + 'Processing submodules description file : {0} ({1})'.format( + submod_file, repo_path)) + submod_model_data= _read_gitmodules_file(repo_path, submod_file) + submod_desc = create_externals_description(submod_model_data) # Can we find our external? repo_url = None diff --git a/manic/externals_status.py b/manic/externals_status.py index 4900e41254..6bc29e9732 100644 --- a/manic/externals_status.py +++ b/manic/externals_status.py @@ -30,12 +30,12 @@ class ExternalStatus(object): """ # sync_state and clean_state can be one of the following: - DEFAULT = '-' # aka not set yet. + DEFAULT = '-' # not set yet (sync_state). clean_state can be this if sync_state is EMPTY. UNKNOWN = '?' EMPTY = 'e' MODEL_MODIFIED = 's' # repo version != externals (sync_state only) DIRTY = 'M' # repo is dirty (clean_state only) - STATUS_OK = ' ' # repo is clean/matches externals. + STATUS_OK = ' ' # repo is clean (clean_state) or matches externals version (sync_state) STATUS_ERROR = '!' # source_type can be one of the following: diff --git a/manic/repository_git.py b/manic/repository_git.py index 3a6a0f1716..adc666cc57 100644 --- a/manic/repository_git.py +++ b/manic/repository_git.py @@ -25,7 +25,7 @@ class GitRepository(Repository): * be isolated in separate functions with no application logic * of the form: - - cmd = ['git', ...] + - cmd = 'git -C {dirname} ...'.format(dirname=dirname).split() - value = execute_subprocess(cmd, output_to_caller={T|F}, status_to_caller={T|F}) - return value @@ -39,7 +39,7 @@ class GitRepository(Repository): def __init__(self, component_name, repo): """ - Parse repo (a XML element). + repo: ExternalsDescription. """ Repository.__init__(self, component_name, repo) self._gitmodules = None @@ -99,15 +99,13 @@ def submodules_file(self, repo_path=None): # # ---------------------------------------------------------------- def _clone_repo(self, base_dir_path, repo_dir_name, verbosity): - """Prepare to execute the clone by managing directory location + """Clones repo_dir_name into base_dir_path. """ - cwd = os.getcwd() - os.chdir(base_dir_path) - self._git_clone(self._url, repo_dir_name, verbosity) - os.chdir(cwd) + self._git_clone(self._url, os.path.join(base_dir_path, repo_dir_name), + verbosity=verbosity) - def _current_ref(self): - """Determine the *name* associated with HEAD. + def _current_ref(self, dirname): + """Determine the *name* associated with HEAD at dirname. If we're on a tag, then returns the tag name; otherwise, returns the current hash. Returns an empty string if no reference can be @@ -119,21 +117,21 @@ def _current_ref(self): ref_found = False # If we're exactly at a tag, use that as the current ref - tag_found, tag_name = self._git_current_tag() + tag_found, tag_name = self._git_current_tag(dirname) if tag_found: current_ref = tag_name ref_found = True if not ref_found: # Otherwise, use current hash as the current ref - hash_found, hash_name = self._git_current_hash() + hash_found, hash_name = self._git_current_hash(dirname) if hash_found: current_ref = hash_name ref_found = True if ref_found: # If we're on a branch, include branch name in current ref - branch_found, branch_name = self._git_current_branch() + branch_found, branch_name = self._git_current_branch(dirname) if branch_found: current_ref = "{} (branch {})".format(current_ref, branch_name) else: @@ -184,17 +182,15 @@ def compare_refs(current_ref, expected_ref): status = ExternalStatus.MODEL_MODIFIED return status - cwd = os.getcwd() - os.chdir(repo_dir_path) - # get the full hash of the current commit - _, current_ref = self._git_current_hash() + _, current_ref = self._git_current_hash(repo_dir_path) if self._branch: if self._url == LOCAL_PATH_INDICATOR: expected_ref = self._branch else: - remote_name = self._determine_remote_name() + remote_name = self._remote_name_for_url(self._url, + repo_dir_path) if not remote_name: # git doesn't know about this remote. by definition # this is a modified state. @@ -211,7 +207,7 @@ def compare_refs(current_ref, expected_ref): fatal_error(msg) # record the *names* of the current and expected branches - stat.current_version = self._current_ref() + stat.current_version = self._current_ref(repo_dir_path) stat.expected_version = copy.deepcopy(expected_ref) if current_ref == EMPTY_STR: @@ -219,7 +215,7 @@ def compare_refs(current_ref, expected_ref): else: # get the underlying hash of the expected ref revparse_status, expected_ref_hash = self._git_revparse_commit( - expected_ref) + expected_ref, repo_dir_path) if revparse_status: # We failed to get the hash associated with # expected_ref. Maybe we should assign this to some special @@ -230,18 +226,13 @@ def compare_refs(current_ref, expected_ref): # compare the underlying hashes stat.sync_state = compare_refs(current_ref, expected_ref_hash) - os.chdir(cwd) - - def _determine_remote_name(self): - """Return the remote name. - - Note that this is for the *future* repo url and branch, not - the current working copy! + @classmethod + def _remote_name_for_url(cls, remote_url, dirname): + """Return the remote name matching remote_url (or None) """ - git_output = self._git_remote_verbose() + git_output = cls._git_remote_verbose(dirname) git_output = git_output.splitlines() - remote_name = '' for line in git_output: data = line.strip() if not data: @@ -249,10 +240,9 @@ def _determine_remote_name(self): data = data.split() name = data[0].strip() url = data[1].strip() - if self._url == url: - remote_name = name - break - return remote_name + if remote_url == url: + return name + return None def _create_remote_name(self): """The url specified in the externals description file was not known @@ -308,19 +298,16 @@ def _checkout_ref(self, repo_dir, verbosity, submodules): the repo's submodules """ # import pdb; pdb.set_trace() - cwd = os.getcwd() - os.chdir(repo_dir) if self._url.strip() == LOCAL_PATH_INDICATOR: - self._checkout_local_ref(verbosity, submodules) + self._checkout_local_ref(verbosity, submodules, repo_dir) else: - self._checkout_external_ref(verbosity, submodules) + self._checkout_external_ref(verbosity, submodules, repo_dir) if self._sparse: self._sparse_checkout(repo_dir, verbosity) - os.chdir(cwd) - def _checkout_local_ref(self, verbosity, submodules): + def _checkout_local_ref(self, verbosity, submodules, dirname): """Checkout the reference considering the local repo only. Do not fetch any additional remotes or specify the remote when checkout out the ref. @@ -334,13 +321,18 @@ def _checkout_local_ref(self, verbosity, submodules): else: ref = self._hash - self._check_for_valid_ref(ref) - self._git_checkout_ref(ref, verbosity, submodules) + self._check_for_valid_ref(ref, remote_name=None, + dirname=dirname) + self._git_checkout_ref(ref, verbosity, submodules, dirname) - def _checkout_external_ref(self, verbosity, submodules): - """Checkout the reference from a remote repository + def _checkout_external_ref(self, verbosity, submodules, dirname): + """Checkout the reference from a remote repository into dirname. if is True, recursively initialize and update - the repo's submodules + the repo's submodules. + Note that this results in a 'detached HEAD' state if checking out + a branch, because we check out the remote branch rather than the + local. See https://github.com/ESMCI/manage_externals/issues/34 for + more discussion. """ if self._tag: ref = self._tag @@ -349,44 +341,45 @@ def _checkout_external_ref(self, verbosity, submodules): else: ref = self._hash - remote_name = self._determine_remote_name() + remote_name = self._remote_name_for_url(self._url, dirname) if not remote_name: remote_name = self._create_remote_name() - self._git_remote_add(remote_name, self._url) - self._git_fetch(remote_name) + self._git_remote_add(remote_name, self._url, dirname) + self._git_fetch(remote_name, dirname) # NOTE(bja, 2018-03) we need to send separate ref and remote # name to check_for_vaild_ref, but the combined name to # checkout_ref! - self._check_for_valid_ref(ref, remote_name) + self._check_for_valid_ref(ref, remote_name, dirname) if self._branch: + # Prepend remote name to branch. This means we avoid various + # special cases if the local branch is not tracking the remote or + # cannot be trivially fast-forwarded to match; but, it also + # means we end up in a 'detached HEAD' state. ref = '{0}/{1}'.format(remote_name, ref) - self._git_checkout_ref(ref, verbosity, submodules) + self._git_checkout_ref(ref, verbosity, submodules, dirname) def _sparse_checkout(self, repo_dir, verbosity): """Use git read-tree to thin the working tree.""" - cwd = os.getcwd() - - cmd = ['cp', self._sparse, os.path.join(repo_dir, - '.git/info/sparse-checkout')] + cmd = ['cp', os.path.join(repo_dir, self._sparse), + os.path.join(repo_dir, + '.git/info/sparse-checkout')] if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) - os.chdir(repo_dir) - self._git_sparse_checkout(verbosity) + self._git_sparse_checkout(verbosity, repo_dir) - os.chdir(cwd) - - def _check_for_valid_ref(self, ref, remote_name=None): + def _check_for_valid_ref(self, ref, remote_name, dirname): """Try some basic sanity checks on the user supplied reference so we can provide a more useful error message than calledprocess error... + remote_name can be NOne """ - is_tag = self._ref_is_tag(ref) - is_branch = self._ref_is_branch(ref, remote_name) - is_hash = self._ref_is_hash(ref) + is_tag = self._ref_is_tag(ref, dirname) + is_branch = self._ref_is_branch(ref, remote_name, dirname) + is_hash = self._ref_is_hash(ref, dirname) is_valid = is_tag or is_branch or is_hash if not is_valid: @@ -397,7 +390,8 @@ def _check_for_valid_ref(self, ref, remote_name=None): fatal_error(msg) if is_tag: - is_unique_tag, msg = self._is_unique_tag(ref, remote_name) + is_unique_tag, msg = self._is_unique_tag(ref, remote_name, + dirname) if not is_unique_tag: msg = ('In repo "{0}": tag "{1}" {2}'.format( self._name, self._tag, msg)) @@ -405,7 +399,7 @@ def _check_for_valid_ref(self, ref, remote_name=None): return is_valid - def _is_unique_tag(self, ref, remote_name): + def _is_unique_tag(self, ref, remote_name, dirname): """Verify that a reference is a valid tag and is unique (not a branch) Tags may be tag names, or SHA id's. It is also possible that a @@ -416,9 +410,9 @@ def _is_unique_tag(self, ref, remote_name): error! """ - is_tag = self._ref_is_tag(ref) - is_branch = self._ref_is_branch(ref, remote_name) - is_hash = self._ref_is_hash(ref) + is_tag = self._ref_is_tag(ref, dirname) + is_branch = self._ref_is_branch(ref, remote_name, dirname) + is_hash = self._ref_is_hash(ref, dirname) msg = '' is_unique_tag = False @@ -449,7 +443,7 @@ def _is_unique_tag(self, ref, remote_name): return is_unique_tag, msg - def _ref_is_tag(self, ref): + def _ref_is_tag(self, ref, dirname): """Verify that a reference is a valid tag according to git. Note: values returned by git_showref_* and git_revparse are @@ -457,28 +451,30 @@ def _ref_is_tag(self, ref): error! """ is_tag = False - value = self._git_showref_tag(ref) + value = self._git_showref_tag(ref, dirname) if value == 0: is_tag = True return is_tag - def _ref_is_branch(self, ref, remote_name=None): + def _ref_is_branch(self, ref, remote_name, dirname): """Verify if a ref is any kind of branch (local, tracked remote, untracked remote). + remote_name can be None. """ local_branch = False remote_branch = False if remote_name: - remote_branch = self._ref_is_remote_branch(ref, remote_name) - local_branch = self._ref_is_local_branch(ref) + remote_branch = self._ref_is_remote_branch(ref, remote_name, + dirname) + local_branch = self._ref_is_local_branch(ref, dirname) is_branch = False if local_branch or remote_branch: is_branch = True return is_branch - def _ref_is_local_branch(self, ref): + def _ref_is_local_branch(self, ref, dirname): """Verify that a reference is a valid branch according to git. show-ref branch returns local branches that have been @@ -491,12 +487,12 @@ def _ref_is_local_branch(self, ref): """ is_branch = False - value = self._git_showref_branch(ref) + value = self._git_showref_branch(ref, dirname) if value == 0: is_branch = True return is_branch - def _ref_is_remote_branch(self, ref, remote_name): + def _ref_is_remote_branch(self, ref, remote_name, dirname): """Verify that a reference is a valid branch according to git. show-ref branch returns local branches that have been @@ -509,12 +505,12 @@ def _ref_is_remote_branch(self, ref, remote_name): """ is_branch = False - value = self._git_lsremote_branch(ref, remote_name) + value = self._git_lsremote_branch(ref, remote_name, dirname) if value == 0: is_branch = True return is_branch - def _ref_is_commit(self, ref): + def _ref_is_commit(self, ref, dirname): """Verify that a reference is a valid commit according to git. This could be a tag, branch, sha1 id, HEAD and potentially others... @@ -524,12 +520,12 @@ def _ref_is_commit(self, ref): error! """ is_commit = False - value, _ = self._git_revparse_commit(ref) + value, _ = self._git_revparse_commit(ref, dirname) if value == 0: is_commit = True return is_commit - def _ref_is_hash(self, ref): + def _ref_is_hash(self, ref, dirname): """Verify that a reference is a valid hash according to git. Git doesn't seem to provide an exact way to determine if user @@ -544,7 +540,7 @@ def _ref_is_hash(self, ref): """ is_hash = False - status, git_output = self._git_revparse_commit(ref) + status, git_output = self._git_revparse_commit(ref, dirname) if status == 0: if git_output.strip().startswith(ref): is_hash = True @@ -554,9 +550,7 @@ def _status_summary(self, stat, repo_dir_path): """Determine the clean/dirty status of a git repository """ - cwd = os.getcwd() - os.chdir(repo_dir_path) - git_output = self._git_status_porcelain_v1z() + git_output = self._git_status_porcelain_v1z(repo_dir_path) is_dirty = self._status_v1z_is_dirty(git_output) if is_dirty: stat.clean_state = ExternalStatus.DIRTY @@ -565,8 +559,7 @@ def _status_summary(self, stat, repo_dir_path): # Now save the verbose status output incase the user wants to # see it. - stat.status_output = self._git_status_verbose() - os.chdir(cwd) + stat.status_output = self._git_status_verbose(repo_dir_path) @staticmethod def _status_v1z_is_dirty(git_output): @@ -601,7 +594,7 @@ def _status_v1z_is_dirty(git_output): # # ---------------------------------------------------------------- @staticmethod - def _git_current_hash(): + def _git_current_hash(dirname): """Return the full hash of the currently checked-out version. Returns a tuple, (hash_found, hash), where hash_found is a @@ -609,21 +602,51 @@ def _git_current_hash(): could mean we're not in a git repository at all). (If hash_found is False, then hash is ''.) """ - status, git_output = GitRepository._git_revparse_commit("HEAD") + status, git_output = GitRepository._git_revparse_commit("HEAD", + dirname) hash_found = not status if not hash_found: git_output = '' return hash_found, git_output @staticmethod - def _git_current_branch(): - """Determines the name of the current branch. + def _git_current_remote_branch(dirname): + """Determines the name of the current remote branch, if any. + + if dir is None, uses the cwd. Returns a tuple, (branch_found, branch_name), where branch_found - is a logical specifying whether a branch name was found for + is a bool specifying whether a branch name was found for + HEAD. (If branch_found is False, then branch_name is ''). + branch_name is in the format '$remote/$branch', e.g. 'origin/foo'. + """ + branch_found = False + branch_name = '' + + cmd = 'git -C {dirname} log -n 1 --pretty=%d HEAD'.format( + dirname=dirname).split() + status, git_output = execute_subprocess(cmd, + output_to_caller=True, + status_to_caller=True) + branch_found = 'HEAD,' in git_output + if branch_found: + # git_output is of the form " (HEAD, origin/blah)" + branch_name = git_output.split(',')[1].strip()[:-1] + return branch_found, branch_name + + @staticmethod + def _git_current_branch(dirname): + """Determines the name of the current local branch. + + Returns a tuple, (branch_found, branch_name), where branch_found + is a bool specifying whether a branch name was found for HEAD. (If branch_found is False, then branch_name is ''.) + Note that currently we check out the remote branch rather than + the local, so this command does not return the just-checked-out + branch. See _git_current_remote_branch. """ - cmd = ['git', 'symbolic-ref', '--short', '-q', 'HEAD'] + cmd = 'git -C {dirname} symbolic-ref --short -q HEAD'.format( + dirname=dirname).split() status, git_output = execute_subprocess(cmd, output_to_caller=True, status_to_caller=True) @@ -635,15 +658,17 @@ def _git_current_branch(): return branch_found, git_output @staticmethod - def _git_current_tag(): + def _git_current_tag(dirname): """Determines the name tag corresponding to HEAD (if any). + if dirname is None, uses the cwd. + Returns a tuple, (tag_found, tag_name), where tag_found is a - logical specifying whether we found a tag name corresponding to + bool specifying whether we found a tag name corresponding to HEAD. (If tag_found is False, then tag_name is ''.) """ - # git describe --exact-match --tags HEAD - cmd = ['git', 'describe', '--exact-match', '--tags', 'HEAD'] + cmd = 'git -C {dirname} describe --exact-match --tags HEAD'.format( + dirname=dirname).split() status, git_output = execute_subprocess(cmd, output_to_caller=True, status_to_caller=True) @@ -655,53 +680,54 @@ def _git_current_tag(): return tag_found, git_output @staticmethod - def _git_showref_tag(ref): + def _git_showref_tag(ref, dirname): """Run git show-ref check if the user supplied ref is a tag. could also use git rev-parse --quiet --verify tagname^{tag} """ - cmd = ['git', 'show-ref', '--quiet', '--verify', - 'refs/tags/{0}'.format(ref), ] + cmd = ('git -C {dirname} show-ref --quiet --verify refs/tags/{ref}' + .format(dirname=dirname, ref=ref).split()) status = execute_subprocess(cmd, status_to_caller=True) return status @staticmethod - def _git_showref_branch(ref): + def _git_showref_branch(ref, dirname): """Run git show-ref check if the user supplied ref is a local or tracked remote branch. """ - cmd = ['git', 'show-ref', '--quiet', '--verify', - 'refs/heads/{0}'.format(ref), ] + cmd = ('git -C {dirname} show-ref --quiet --verify refs/heads/{ref}' + .format(dirname=dirname, ref=ref).split()) status = execute_subprocess(cmd, status_to_caller=True) return status @staticmethod - def _git_lsremote_branch(ref, remote_name): + def _git_lsremote_branch(ref, remote_name, dirname): """Run git ls-remote to check if the user supplied ref is a remote branch that is not being tracked """ - cmd = ['git', 'ls-remote', '--exit-code', '--heads', - remote_name, ref, ] + cmd = ('git -C {dirname} ls-remote --exit-code --heads ' + '{remote_name} {ref}').format( + dirname=dirname, remote_name=remote_name, ref=ref).split() status = execute_subprocess(cmd, status_to_caller=True) return status @staticmethod - def _git_revparse_commit(ref): + def _git_revparse_commit(ref, dirname): """Run git rev-parse to detect if a reference is a SHA, HEAD or other valid commit. """ - cmd = ['git', 'rev-parse', '--quiet', '--verify', - '{0}^{1}'.format(ref, '{commit}'), ] + cmd = ('git -C {dirname} rev-parse --quiet --verify {ref}^{commit}' + .format(dirname=dirname, ref=ref, commit='{commit}').split()) status, git_output = execute_subprocess(cmd, status_to_caller=True, output_to_caller=True) git_output = git_output.strip() return status, git_output @staticmethod - def _git_status_porcelain_v1z(): + def _git_status_porcelain_v1z(dirname): """Run git status to obtain repository information. This is run with '--untracked=no' to ignore untracked files. @@ -710,36 +736,38 @@ def _git_status_porcelain_v1z(): between git versions or *user configuration*. """ - cmd = ['git', 'status', '--untracked-files=no', '--porcelain', '-z'] + cmd = ('git -C {dirname} status --untracked-files=no --porcelain -z' + .format(dirname=dirname)).split() git_output = execute_subprocess(cmd, output_to_caller=True) return git_output @staticmethod - def _git_status_verbose(): + def _git_status_verbose(dirname): """Run the git status command to obtain repository information. """ - cmd = ['git', 'status'] + cmd = 'git -C {dirname} status'.format(dirname=dirname).split() git_output = execute_subprocess(cmd, output_to_caller=True) return git_output @staticmethod - def _git_remote_verbose(): + def _git_remote_verbose(dirname): """Run the git remote command to obtain repository information. + + Returned string is of the form: + myfork git@github.com:johnpaulalex/manage_externals_jp.git (fetch) + myfork git@github.com:johnpaulalex/manage_externals_jp.git (push) """ - cmd = ['git', 'remote', '--verbose'] - git_output = execute_subprocess(cmd, output_to_caller=True) - return git_output + cmd = 'git -C {dirname} remote --verbose'.format( + dirname=dirname).split() + return execute_subprocess(cmd, output_to_caller=True) @staticmethod - def has_submodules(repo_dir_path=None): - """Return True iff the repository at (or the current - directory if is None) has a '.gitmodules' file + def has_submodules(repo_dir_path): + """Return True iff the repository at has a + '.gitmodules' file """ - if repo_dir_path is None: - fname = ExternalsDescription.GIT_SUBMODULES_FILENAME - else: - fname = os.path.join(repo_dir_path, - ExternalsDescription.GIT_SUBMODULES_FILENAME) + fname = os.path.join(repo_dir_path, + ExternalsDescription.GIT_SUBMODULES_FILENAME) return os.path.exists(fname) @@ -750,68 +778,71 @@ def has_submodules(repo_dir_path=None): # ---------------------------------------------------------------- @staticmethod def _git_clone(url, repo_dir_name, verbosity): - """Run git clone for the side effect of creating a repository. + """Clones url into repo_dir_name. """ - cmd = ['git', 'clone', '--quiet'] - subcmd = None - - cmd.extend([url, repo_dir_name]) + cmd = 'git clone --quiet {url} {repo_dir_name}'.format( + url=url, repo_dir_name=repo_dir_name).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) - if subcmd is not None: - os.chdir(repo_dir_name) - execute_subprocess(subcmd) @staticmethod - def _git_remote_add(name, url): + def _git_remote_add(name, url, dirname): """Run the git remote command for the side effect of adding a remote """ - cmd = ['git', 'remote', 'add', name, url] + cmd = 'git -C {dirname} remote add {name} {url}'.format( + dirname=dirname, name=name, url=url).split() execute_subprocess(cmd) @staticmethod - def _git_fetch(remote_name): + def _git_fetch(remote_name, dirname): """Run the git fetch command for the side effect of updating the repo """ - cmd = ['git', 'fetch', '--quiet', '--tags', remote_name] + cmd = 'git -C {dirname} fetch --quiet --tags {remote_name}'.format( + dirname=dirname, remote_name=remote_name).split() execute_subprocess(cmd) @staticmethod - def _git_checkout_ref(ref, verbosity, submodules): + def _git_checkout_ref(ref, verbosity, submodules, dirname): """Run the git checkout command for the side effect of updating the repo Param: ref is a reference to a local or remote object in the form 'origin/my_feature', or 'tag1'. """ - cmd = ['git', 'checkout', '--quiet', ref] + cmd = 'git -C {dirname} checkout --quiet {ref}'.format( + dirname=dirname, ref=ref).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) if submodules: - GitRepository._git_update_submodules(verbosity) + GitRepository._git_update_submodules(verbosity, dirname) @staticmethod - def _git_sparse_checkout(verbosity): + def _git_sparse_checkout(verbosity, dirname): """Configure repo via read-tree.""" - cmd = ['git', 'config', 'core.sparsecheckout', 'true'] + cmd = 'git -C {dirname} config core.sparsecheckout true'.format( + dirname=dirname).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) - cmd = ['git', 'read-tree', '-mu', 'HEAD'] + cmd = 'git -C {dirname} read-tree -mu HEAD'.format( + dirname=dirname).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) execute_subprocess(cmd) @staticmethod - def _git_update_submodules(verbosity): + def _git_update_submodules(verbosity, dirname): """Run git submodule update for the side effect of updating this repo's submodules. """ # First, verify that we have a .gitmodules file - if os.path.exists(ExternalsDescription.GIT_SUBMODULES_FILENAME): - cmd = ['git', 'submodule', 'update', '--init', '--recursive'] + if os.path.exists( + os.path.join(dirname, + ExternalsDescription.GIT_SUBMODULES_FILENAME)): + cmd = ('git -C {dirname} submodule update --init --recursive' + .format(dirname=dirname)).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) diff --git a/manic/sourcetree.py b/manic/sourcetree.py index 218b5febb4..cf2a5b7569 100644 --- a/manic/sourcetree.py +++ b/manic/sourcetree.py @@ -1,6 +1,6 @@ """ - -FIXME(bja, 2017-11) External and SourceTree have a circular dependancy! +Classes to represent an externals config file (SourceTree) and the components +within it (_External). """ import errno @@ -20,71 +20,53 @@ class _External(object): """ A single component hosted in an external repository (and any children). - """ + The component may or may not be checked-out upon construction. + """ # pylint: disable=R0902 - def __init__(self, root_dir, name, ext_description, svn_ignore_ancestry): - """Parse an external description file into a dictionary of externals. + def __init__(self, root_dir, name, local_path, required, subexternals_path, + repo, svn_ignore_ancestry, subexternal_sourcetree): + """Create a single external component (checked out or not). Input: + root_dir : string - the (checked-out) parent repo's root dir. + local_path : string - this external's (checked-out) subdir relative + to root_dir, e.g. "components/mom" + repo: Repository - the repo object for this external. Can be None (e.g. if this external just refers to another external file). - root_dir : string - the root directory path where - 'local_path' is relative to. - - name : string - name of the ext_description object. may or may not - correspond to something in the path. + name : string - name of this external (as named by the parent + reference). May or may not correspond to something in the path. ext_description : dict - source ExternalsDescription object svn_ignore_ancestry : bool - use --ignore-externals with svn switch + subexternals_path: string - path to sub-externals config file, if any. Relative to local_path, or special value 'none'. + subexternal_sourcetree: SourceTree - corresponding to subexternals_path, if subexternals_path exists (it might not, if it is not checked out yet). """ self._name = name - self._repo = None # Repository object. - - # Subcomponent externals file and data object, if any. - self._externals_path = EMPTY_STR # Can also be "none" - self._externals_sourcetree = None + self._required = required self._stat = None # Populated in status() - self._sparse = None - # Parse the sub-elements - # _local_path : local path relative to the containing source tree, e.g. - # "components/mom" - self._local_path = ext_description[ExternalsDescription.PATH] + self._local_path = local_path # _repo_dir_path : full repository directory, e.g. # "/components/mom" - repo_dir = os.path.join(root_dir, self._local_path) + repo_dir = os.path.join(root_dir, local_path) self._repo_dir_path = os.path.abspath(repo_dir) # _base_dir_path : base directory *containing* the repository, e.g. # "/components" self._base_dir_path = os.path.dirname(self._repo_dir_path) - # _repo_dir_name : base_dir_path + repo_dir_name = rep_dir_path + # _repo_dir_name : base_dir_path + repo_dir_name = repo_dir_path # e.g., "mom" self._repo_dir_name = os.path.basename(self._repo_dir_path) - assert(os.path.join(self._base_dir_path, self._repo_dir_name) - == self._repo_dir_path) - - self._required = ext_description[ExternalsDescription.REQUIRED] + self._repo = repo # Does this component have subcomponents aka an externals config? - self._externals_path = ext_description[ExternalsDescription.EXTERNALS] - # Treat a .gitmodules file as a backup externals config - if not self._externals_path: - if GitRepository.has_submodules(self._repo_dir_path): - self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME - - repo = create_repository( - name, ext_description[ExternalsDescription.REPO], - svn_ignore_ancestry=svn_ignore_ancestry) - if repo: - self._repo = repo - - # Recurse into subcomponents, if any. - if self._externals_path and (self._externals_path.lower() != 'none'): - self._create_externals_sourcetree() + self._subexternals_path = subexternals_path + self._subexternal_sourcetree = subexternal_sourcetree + def get_name(self): """ @@ -98,6 +80,15 @@ def get_local_path(self): """ return self._local_path + def get_repo_dir_path(self): + return self._repo_dir_path + + def get_subexternals_path(self): + return self._subexternals_path + + def get_repo(self): + return self._repo + def status(self, force=False, print_progress=False): """ Returns status of this component and all subcomponents. @@ -148,12 +139,12 @@ def status(self, force=False, print_progress=False): self._repo.status(self._stat, self._repo_dir_path) # Status of subcomponents, if any. - if self._externals_path and self._externals_sourcetree: + if self._subexternals_path and self._subexternal_sourcetree: cwd = os.getcwd() # SourceTree.status() expects to be called from the correct # root directory. os.chdir(self._repo_dir_path) - subcomponent_stats = self._externals_sourcetree.status(self._local_path, force=force, print_progress=print_progress) + subcomponent_stats = self._subexternal_sourcetree.status(self._local_path, force=force, print_progress=print_progress) os.chdir(cwd) # Merge our status + subcomponent statuses into one return dict keyed @@ -174,10 +165,10 @@ def status(self, force=False, print_progress=False): def checkout(self, verbosity): """ If the repo destination directory exists, ensure it is correct (from - correct URL, correct branch or tag), and possibly update the external. + correct URL, correct branch or tag), and possibly updateit. If the repo destination directory does not exist, checkout the correct branch or tag. - Does not check out sub-externals, see checkout_subexternals(). + Does not check out sub-externals, see SourceTree.checkout(). """ # Make sure we are in correct location if not os.path.exists(self._repo_dir_path): @@ -215,98 +206,117 @@ def checkout(self, verbosity): self._repo.checkout(self._base_dir_path, self._repo_dir_name, checkout_verbosity, self.clone_recursive()) - def checkout_subexternals(self, verbosity, load_all): - """Recursively checkout the sub-externals for this component, if any. - - See load_all documentation in SourceTree.checkout(). - """ - if self.load_externals(): - if self._externals_sourcetree: - # NOTE(bja, 2018-02): the subtree externals objects - # were created during initial status check. Updating - # the external may have changed which sub-externals - # are needed. We need to delete those objects and - # re-read the potentially modified externals - # description file. - self._externals_sourcetree = None - self._create_externals_sourcetree() - self._externals_sourcetree.checkout(verbosity, load_all) - - def load_externals(self): - 'Return True iff an externals file exists (and therefore should be loaded)' - load_ex = False - if os.path.exists(self._repo_dir_path): - if self._externals_path: - if self._externals_path.lower() != 'none': - load_ex = os.path.exists(os.path.join(self._repo_dir_path, - self._externals_path)) - - return load_ex + def replace_subexternal_sourcetree(self, sourcetree): + self._subexternal_sourcetree = sourcetree def clone_recursive(self): 'Return True iff any .gitmodules files should be processed' # Try recursive .gitmodules unless there is an externals entry - recursive = not self._externals_path + recursive = not self._subexternals_path return recursive - def _create_externals_sourcetree(self): - """ - Note this only creates an object, it doesn't write to the file system. + +class SourceTree(object): + """ + SourceTree represents a group of managed externals. + + Those externals may not be checked out locally yet, they might only + have Repository objects pointing to their respective repositories. + """ + + @classmethod + def from_externals_file(cls, parent_repo_dir_path, parent_repo, + externals_path): + """Creates a SourceTree representing the given externals file. + + Looks up a git submodules file as an optional backup if there is no + externals file specified. + + Returns None if there is no externals file (i.e. it's None or 'none'), + or if the externals file hasn't been checked out yet. + + parent_repo_dir_path: parent repo root dir + parent_repo: parent repo. + externals_path: path to externals file, relative to parent_repo_dir_path. """ - if not os.path.exists(self._repo_dir_path): + if not os.path.exists(parent_repo_dir_path): # NOTE(bja, 2017-10) repository has not been checked out # yet, can't process the externals file. Assume we are # checking status before code is checkoud out and this # will be handled correctly later. - return + return None - cwd = os.getcwd() - os.chdir(self._repo_dir_path) - if self._externals_path.lower() == 'none': - msg = ('Internal: Attempt to create source tree for ' - 'externals = none in {}'.format(self._repo_dir_path)) - fatal_error(msg) + if externals_path.lower() == 'none': + # With explicit 'none', do not look for git submodules file. + return None - if not os.path.exists(self._externals_path): - if GitRepository.has_submodules(): - self._externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME + cwd = os.getcwd() + os.chdir(parent_repo_dir_path) + + if not externals_path: + if GitRepository.has_submodules(parent_repo_dir_path): + externals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME + else: + return None - if not os.path.exists(self._externals_path): - # NOTE(bja, 2017-10) this check is redundent with the one + if not os.path.exists(externals_path): + # NOTE(bja, 2017-10) this check is redundant with the one # in read_externals_description_file! - msg = ('External externals description file "{0}" ' + msg = ('Externals description file "{0}" ' 'does not exist! In directory: {1}'.format( - self._externals_path, self._repo_dir_path)) + externals_path, parent_repo_dir_path)) fatal_error(msg) - externals_root = self._repo_dir_path + externals_root = parent_repo_dir_path # model_data is a dict-like object which mirrors the file format. model_data = read_externals_description_file(externals_root, - self._externals_path) + externals_path) # ext_description is another dict-like object (see ExternalsDescription) ext_description = create_externals_description(model_data, - parent_repo=self._repo) - self._externals_sourcetree = SourceTree(externals_root, ext_description) + parent_repo=parent_repo) + externals_sourcetree = SourceTree(externals_root, ext_description) os.chdir(cwd) - -class SourceTree(object): - """ - SourceTree represents a group of managed externals - """ - + return externals_sourcetree + def __init__(self, root_dir, ext_description, svn_ignore_ancestry=False): """ Build a SourceTree object from an ExternalDescription. + + root_dir: the (checked-out) parent repo root dir. """ self._root_dir = os.path.abspath(root_dir) self._all_components = {} # component_name -> _External self._required_compnames = [] - for comp in ext_description: - src = _External(self._root_dir, comp, ext_description[comp], - svn_ignore_ancestry) + for comp, desc in ext_description.items(): + local_path = desc[ExternalsDescription.PATH] + required = desc[ExternalsDescription.REQUIRED] + repo_info = desc[ExternalsDescription.REPO] + subexternals_path = desc[ExternalsDescription.EXTERNALS] + + repo = create_repository(comp, + repo_info, + svn_ignore_ancestry=svn_ignore_ancestry) + + sourcetree = None + # Treat a .gitmodules file as a backup externals config + if not subexternals_path: + parent_repo_dir_path = os.path.abspath(os.path.join(root_dir, + local_path)) + if GitRepository.has_submodules(parent_repo_dir_path): + subexternals_path = ExternalsDescription.GIT_SUBMODULES_FILENAME + + # Might return None (if the subexternal isn't checked out yet, or subexternal is None or 'none') + subexternal_sourcetree = SourceTree.from_externals_file( + os.path.join(self._root_dir, local_path), + repo, + subexternals_path) + src = _External(self._root_dir, comp, local_path, required, + subexternals_path, repo, svn_ignore_ancestry, + subexternal_sourcetree) + self._all_components[comp] = src - if ext_description[comp][ExternalsDescription.REQUIRED]: + if required: self._required_compnames.append(comp) def status(self, relative_path_base=LOCAL_PATH_INDICATOR, @@ -353,8 +363,11 @@ def _find_installed_optional_components(self): continue # Note that in practice we expect this status to be cached. path_to_stat = ext.status() - if any(stat.sync_state != ExternalStatus.EMPTY - for stat in path_to_stat.values()): + + # If any part of this component exists locally, consider it + # installed and therefore eligible for updating. + if any(s.sync_state != ExternalStatus.EMPTY + for s in path_to_stat.values()): installed_comps.append(comp_name) return installed_comps @@ -376,6 +389,9 @@ def checkout(self, verbosity, load_all, load_comp=None): if local_optional_compnames: printlog('Found locally installed optional components: ' + ', '.join(local_optional_compnames)) + bad_compnames = set(local_optional_compnames) - set(self._all_components.keys()) + if bad_compnames: + printlog('Internal error: found locally installed components that are not in the global list of all components: ' + ','.join(bad_compnames)) if verbosity >= VERBOSITY_VERBOSE: printlog('Checking out externals: ') @@ -387,16 +403,23 @@ def checkout(self, verbosity, load_all, load_comp=None): load_comps = sorted(tmp_comps, key=lambda comp: self._all_components[comp].get_local_path()) # checkout. - for comp in load_comps: + for comp_name in load_comps: if verbosity < VERBOSITY_VERBOSE: - printlog('{0}, '.format(comp), end='') + printlog('{0}, '.format(comp_name), end='') else: # verbose output handled by the _External object, just # output a newline printlog(EMPTY_STR) + c = self._all_components[comp_name] # Does not recurse. - self._all_components[comp].checkout(verbosity) - # Recursively check out subexternals, if any. - self._all_components[comp].checkout_subexternals(verbosity, - load_all) + c.checkout(verbosity) + # Recursively check out subexternals, if any. Returns None + # if there's no subexternals path. + component_subexternal_sourcetree = SourceTree.from_externals_file( + c.get_repo_dir_path(), + c.get_repo(), + c.get_subexternals_path()) + c.replace_subexternal_sourcetree(component_subexternal_sourcetree) + if component_subexternal_sourcetree: + component_subexternal_sourcetree.checkout(verbosity, load_all) printlog('') diff --git a/test/repos/README.md b/test/repos/README.md new file mode 100644 index 0000000000..8a3502c35f --- /dev/null +++ b/test/repos/README.md @@ -0,0 +1,33 @@ +Git repositories for testing git-related behavior. For usage and terminology notes, see test/test_sys_checkout.py. + +To list files and view file contents at HEAD: +``` +cd +git ls-tree --full-tree -r --name-only HEAD +git cat-file -p HEAD: +``` + +File contents at a glance: +``` +container.git/ + readme.txt + +simple-ext.git/ + (has branches: feature2, feature3) + (has tags: tag1, tag2) + readme.txt + simple_subdir/subdir_file.txt + +simple-ext-fork.git/ + (has tags: abandoned-feature, forked-feature-v1, tag1) + (has branch: feature2) + readme.txt + +mixed-cont-ext.git/ + (has branch: new-feature) + readme.txt + sub-externals.cfg ('simp_branch' section refers to 'feature2' branch in simple-ext.git/ repo) + +error/ + (no git repo here, just a readme.txt in the clear) +``` diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py index 27b649bea4..ab4f77e88f 100644 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -2,6 +2,14 @@ """Unit test driver for checkout_externals +Terminology: + * 'container': a repo that has externals + * 'simple': a repo that has no externals, but is referenced as an external by another repo. + * 'mixed': a repo that both has externals and is referenced as an external by another repo. + + * 'clean': the local repo matches the version in the externals and has no local modifications. + * 'empty': the external isn't checked out at all. + Note: this script assume the path to the manic and checkout_externals module is already in the python path. This is usually handled by the makefile. If you call it directly, you may need @@ -21,7 +29,6 @@ * Erase any existing repos at the begining of the module in setUpModule. - """ # NOTE(bja, 2017-11) pylint complains that the module is too big, but @@ -68,39 +75,50 @@ # Module-wide root directory for all the per-test subdirs we'll create on -# the fly. -MANIC_TEST_TMP_REPO_ROOT = 'MANIC_TEST_TMP_REPO_ROOT' # env var name +# the fly (which are placed under wherever $CWD is when the test runs). +# Set by setupModule(). +module_tmp_root_dir = None TMP_REPO_DIR_NAME = 'tmp' # subdir under $CWD -# We clone these checked-in repositories on a per-test basis. The -# 'bare repo root' is the test/repos/ subdir that holds them all. -MANIC_TEST_BARE_REPO_ROOT = 'MANIC_TEST_BARE_REPO_ROOT' # env var name -BARE_REPO_ROOT_NAME = 'repos' # subdir name - -# Subdirs under bare repo root, each holding a repository. -CONTAINER_REPO_NAME = 'container.git' -MIXED_REPO_NAME = 'mixed-cont-ext.git' -SIMPLE_REPO_NAME = 'simple-ext.git' -SIMPLE_FORK_NAME = 'simple-ext-fork.git' - - -# In each of the test repos, all externals live under this subdir. -EXTERNALS_NAME = 'externals' +# subdir under test/ that holds all of our checked-in repositories (which we +# will clone for these tests). +BARE_REPO_ROOT_NAME = 'repos' + +# Environment var referenced by checked-in externals file in mixed-cont-ext.git, +# which should be pointed to the fully-resolved BARE_REPO_ROOT_NAME directory. +# We explicitly clear this after every test, via tearDown(). +MIXED_CONT_EXT_ROOT_ENV_VAR = 'MANIC_TEST_BARE_REPO_ROOT' + +# Subdirs under bare repo root, each holding a repository. For more info +# on the contents of these repositories, see test/repos/README.md. In these +# tests the 'parent' repos are cloned as a starting point, whereas the 'child' +# repos are checked out when the tests run checkout_externals. +CONTAINER_REPO = 'container.git' # Parent repo +SIMPLE_REPO = 'simple-ext.git' # Child repo +SIMPLE_FORK_REPO = 'simple-ext-fork.git' # Child repo +MIXED_REPO = 'mixed-cont-ext.git' # Both parent and child + +# Standard (arbitrary) external names for test configs +TAG_SECTION = 'simp_tag' +BRANCH_SECTION = 'simp_branch' +HASH_SECTION = 'simp_hash' + +# All the configs we construct check out their externals into these local paths. +EXTERNALS_PATH = 'externals' SUB_EXTERNALS_PATH = 'src' # For mixed test repos, # For testing behavior with '.' instead of an explicit paths. SIMPLE_LOCAL_ONLY_NAME = '.' -# Filenames -CFG_NAME = 'externals.cfg' -CFG_SUB_NAME = 'sub-externals.cfg' -README_NAME = 'readme.txt' # Arbitrary file to check in. +# Externals files. +CFG_NAME = 'externals.cfg' # We construct this on a per-test basis. +CFG_SUB_NAME = 'sub-externals.cfg' # Already exists in mixed-cont-ext repo. -# Arbirary test branch name -REMOTE_BRANCH_FEATURE2 = 'feature2' +# Arbitrary text file in all the test repos. +README_NAME = 'readme.txt' -# Section names in container_nested_* test config. -NESTED_NAME = ['./fred', './fred/wilma', './fred/wilma/barney'] +# Branch that exists in both the simple and simple-fork repos. +REMOTE_BRANCH_FEATURE2 = 'feature2' SVN_TEST_REPO = 'https://github.com/escomp/cesm' @@ -123,152 +141,110 @@ def setUpModule(): # pylint: disable=C0103 pass # create clean dir for this run os.mkdir(repo_root) - # set into the environment so var will be expanded in externals - # files when executables are run - os.environ[MANIC_TEST_TMP_REPO_ROOT] = repo_root - - -class GenerateExternalsDescriptionCfgV1(object): - """Building blocks to create ExternalsDescriptionCfgV1 files. - - Basic usage: create_config() multiple create_*(), then write_config(). - Optionally after that: write_with_*(). - - Includes methods (like container_*()) to create predefined configs for the - checked-in repositories in test/repos/. - """ - - def __init__(self): - self._schema_version = '1.1.0' - self._config = None - - def container_full(self, dest_dir): - """Create the full container config file with simple and mixed use - externals - - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2) - - self.create_section(SIMPLE_REPO_NAME, 'simp_opt', - tag='tag1', required=False) - self.create_section(MIXED_REPO_NAME, 'mixed_req', - branch='master', externals=CFG_SUB_NAME) + # Make available to all tests in this file. + global module_tmp_root_dir + assert module_tmp_root_dir == None, module_tmp_root_dir + module_tmp_root_dir = repo_root - self.write_config(dest_dir) - - def container_simple_required(self, dest_dir): - """Create a container externals file with only simple externals. - - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2) - - self.create_section(SIMPLE_REPO_NAME, 'simp_hash', - ref_hash='60b1cc1a38d63') - - self.write_config(dest_dir) - - def container_nested_required(self, dest_dir, order): - """Create a container externals file with only simple externals. - - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', nested=True, - tag='tag1', path=NESTED_NAME[order[0]]) - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', nested=True, - branch=REMOTE_BRANCH_FEATURE2, path=NESTED_NAME[order[1]]) - - self.create_section(SIMPLE_REPO_NAME, 'simp_hash', nested=True, - ref_hash='60b1cc1a38d63', path=NESTED_NAME[order[2]]) - - self.write_config(dest_dir) - - - def container_simple_optional(self, dest_dir): - """Create a container externals file with optional simple externals +class RepoUtils(object): + """Convenience methods for interacting with git repos.""" + @staticmethod + def create_branch(repo_base_dir, external_name, branch, with_commit=False): + """Create branch and optionally (with_commit) add a single commit. """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_req', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_opt', - tag='tag1', required=False) - - self.write_config(dest_dir) - - def container_simple_svn(self, dest_dir): - """Create a container externals file with only simple externals. + # pylint: disable=R0913 + cwd = os.getcwd() + repo_root = os.path.join(repo_base_dir, EXTERNALS_PATH, external_name) + os.chdir(repo_root) + cmd = ['git', 'checkout', '-b', branch, ] + execute_subprocess(cmd) + if with_commit: + msg = 'start work on {0}'.format(branch) + with open(README_NAME, 'a') as handle: + handle.write(msg) + cmd = ['git', 'add', README_NAME, ] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-m', msg, ] + execute_subprocess(cmd) + os.chdir(cwd) + @staticmethod + def create_commit(repo_base_dir, external_name): + """Make a commit to the given external. + + This is used to test sync state changes from local commits on + detached heads and tracking branches. """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', tag='tag1') - - self.create_svn_external('svn_branch', branch='trunk') - self.create_svn_external('svn_tag', tag='tags/cesm2.0.beta07') + cwd = os.getcwd() + repo_root = os.path.join(repo_base_dir, EXTERNALS_PATH, external_name) + os.chdir(repo_root) - self.write_config(dest_dir) + msg = 'work on great new feature!' + with open(README_NAME, 'a') as handle: + handle.write(msg) + cmd = ['git', 'add', README_NAME, ] + execute_subprocess(cmd) + cmd = ['git', 'commit', '-m', msg, ] + execute_subprocess(cmd) + os.chdir(cwd) - def container_sparse(self, dest_dir): - """Create a container with a full external and a sparse external + @staticmethod + def clone_test_repo(bare_root, test_id, parent_repo_name, dest_dir_in): + """Clone repo at / into dest_dir_in or local per-test-subdir. + Returns output dir. """ - # Create a file for a sparse pattern match - sparse_filename = 'sparse_checkout' - with open(os.path.join(dest_dir, sparse_filename), 'w') as sfile: - sfile.write('readme.txt') - - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag2') - - sparse_relpath = '../../{}'.format(sparse_filename) - self.create_section(SIMPLE_REPO_NAME, 'simp_sparse', - tag='tag2', sparse=sparse_relpath) + parent_repo_dir = os.path.join(bare_root, parent_repo_name) + if dest_dir_in is None: + # create unique subdir for this test + test_dir_name = test_id + print("Test repository name: {0}".format(test_dir_name)) + dest_dir = os.path.join(module_tmp_root_dir, test_dir_name) + else: + dest_dir = dest_dir_in - self.write_config(dest_dir) + # pylint: disable=W0212 + GitRepository._git_clone(parent_repo_dir, dest_dir, VERBOSITY_DEFAULT) + return dest_dir - def mixed_simple_base(self, dest_dir): - """Create a mixed-use base externals file with only simple externals. + @staticmethod + def add_file_to_repo(under_test_dir, filename, tracked): + """Add a file to the repository so we can put it into a dirty state """ - self.create_config() - self.create_section_ext_only('mixed_base') - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1') - - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2) + cwd = os.getcwd() + os.chdir(under_test_dir) + with open(filename, 'w') as tmp: + tmp.write('Hello, world!') - self.create_section(SIMPLE_REPO_NAME, 'simp_hash', - ref_hash='60b1cc1a38d63') + if tracked: + # NOTE(bja, 2018-01) brittle hack to obtain repo dir and + # file name + path_data = filename.split('/') + repo_dir = os.path.join(path_data[0], path_data[1]) + os.chdir(repo_dir) + tracked_file = path_data[2] + cmd = ['git', 'add', tracked_file] + execute_subprocess(cmd) - self.write_config(dest_dir) + os.chdir(cwd) - def mixed_simple_sub(self, dest_dir): - """Create a mixed-use sub externals file with only simple externals. +class GenerateExternalsDescriptionCfgV1(object): + """Building blocks to create ExternalsDescriptionCfgV1 files. - """ - self.create_config() - self.create_section(SIMPLE_REPO_NAME, 'simp_tag', - tag='tag1', path=SUB_EXTERNALS_PATH) + Basic usage: create_config() multiple create_*(), then write_config(). + Optionally after that: write_with_*(). + """ - self.create_section(SIMPLE_REPO_NAME, 'simp_branch', - branch=REMOTE_BRANCH_FEATURE2, - path=SUB_EXTERNALS_PATH) + def __init__(self, bare_root): + self._schema_version = '1.1.0' + self._config = None - self.write_config(dest_dir, filename=CFG_SUB_NAME) + # directory where we have test repositories (which we will clone for + # tests) + self._bare_root = bare_root def write_config(self, dest_dir, filename=CFG_NAME): """Write self._config to disk @@ -293,9 +269,15 @@ def create_metadata(self): self._config.set(DESCRIPTION_SECTION, VERSION_ITEM, self._schema_version) + def url_for_repo_path(self, repo_path, repo_path_abs=None): + if repo_path_abs is not None: + return repo_path_abs + else: + return os.path.join(self._bare_root, repo_path) + def create_section(self, repo_path, name, tag='', branch='', - ref_hash='', required=True, path=EXTERNALS_NAME, - externals='', repo_path_abs=None, from_submodule=False, + ref_hash='', required=True, path=EXTERNALS_PATH, + sub_externals='', repo_path_abs=None, from_submodule=False, sparse='', nested=False): # pylint: disable=too-many-branches """Create a config ExternalsDescription section with the given name. @@ -303,7 +285,7 @@ def create_section(self, repo_path, name, tag='', branch='', Autofills some items and handles some optional items. repo_path_abs overrides repo_path (which is relative to the bare repo) - path is a subdir under repo_path. + path is a subdir under repo_path to check out to. """ # pylint: disable=R0913 self._config.add_section(name) @@ -328,10 +310,7 @@ def create_section(self, repo_path, name, tag='', branch='', ref_hash = '' branch = '' - if repo_path_abs is not None: - repo_url = repo_path_abs - else: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) + repo_url = self.url_for_repo_path(repo_path, repo_path_abs) if not from_submodule: self._config.set(name, ExternalsDescription.REPO_URL, repo_url) @@ -347,8 +326,9 @@ def create_section(self, repo_path, name, tag='', branch='', if ref_hash: self._config.set(name, ExternalsDescription.HASH, ref_hash) - if externals: - self._config.set(name, ExternalsDescription.EXTERNALS, externals) + if sub_externals: + self._config.set(name, ExternalsDescription.EXTERNALS, + sub_externals) if sparse: self._config.set(name, ExternalsDescription.SPARSE, sparse) @@ -356,10 +336,8 @@ def create_section(self, repo_path, name, tag='', branch='', if from_submodule: self._config.set(name, ExternalsDescription.SUBMODULE, "True") - def create_section_ext_only(self, name, - required=True, externals=CFG_SUB_NAME): - """Create a config section with autofilling some items and handling - optional items. + def create_section_reference_to_subexternal(self, name): + """Just a reference to another externals file. """ # pylint: disable=R0913 @@ -372,10 +350,9 @@ def create_section_ext_only(self, name, self._config.set(name, ExternalsDescription.REPO_URL, LOCAL_PATH_INDICATOR) - self._config.set(name, ExternalsDescription.REQUIRED, str(required)) + self._config.set(name, ExternalsDescription.REQUIRED, str(True)) - if externals: - self._config.set(name, ExternalsDescription.EXTERNALS, externals) + self._config.set(name, ExternalsDescription.EXTERNALS, CFG_SUB_NAME) def create_svn_external(self, name, tag='', branch=''): """Create a config section for an svn repository. @@ -383,7 +360,7 @@ def create_svn_external(self, name, tag='', branch=''): """ self._config.add_section(name) self._config.set(name, ExternalsDescription.PATH, - os.path.join(EXTERNALS_NAME, name)) + os.path.join(EXTERNALS_PATH, name)) self._config.set(name, ExternalsDescription.PROTOCOL, ExternalsDescription.PROTOCOL_SVN) @@ -398,54 +375,7 @@ def create_svn_external(self, name, tag='', branch=''): if branch: self._config.set(name, ExternalsDescription.BRANCH, branch) - @staticmethod - def create_branch(dest_dir, repo_name, branch, with_commit=False): - """Create branch and optionally (with_commit) add a single commit. - """ - # pylint: disable=R0913 - cwd = os.getcwd() - repo_root = os.path.join(dest_dir, EXTERNALS_NAME, repo_name) - os.chdir(repo_root) - cmd = ['git', 'checkout', '-b', branch, ] - execute_subprocess(cmd) - if with_commit: - msg = 'start work on {0}'.format(branch) - with open(README_NAME, 'a') as handle: - handle.write(msg) - cmd = ['git', 'add', README_NAME, ] - execute_subprocess(cmd) - cmd = ['git', 'commit', '-m', msg, ] - execute_subprocess(cmd) - os.chdir(cwd) - - @staticmethod - def create_commit(dest_dir, repo_name, local_tracking_branch=None): - """Make a commit on whatever is currently checked out. - - This is used to test sync state changes from local commits on - detached heads and tracking branches. - - NOTE(jpalex, 2023-01) This has nothing to do with self._config, - should probably live elsewhere. - """ - cwd = os.getcwd() - repo_root = os.path.join(dest_dir, EXTERNALS_NAME, repo_name) - os.chdir(repo_root) - if local_tracking_branch: - cmd = ['git', 'checkout', '-b', local_tracking_branch, ] - execute_subprocess(cmd) - - msg = 'work on great new feature!' - with open(README_NAME, 'a') as handle: - handle.write(msg) - cmd = ['git', 'add', README_NAME, ] - execute_subprocess(cmd) - cmd = ['git', 'commit', '-m', msg, ] - execute_subprocess(cmd) - os.chdir(cwd) - - def write_with_git_branch(self, dest_dir, name, branch, repo_path=None, - filename=CFG_NAME): + def write_with_git_branch(self, dest_dir, name, branch, new_remote_repo_path=None): """Update fields in our config and write it to disk. name is the key of the ExternalsDescription in self._config to update. @@ -453,12 +383,11 @@ def write_with_git_branch(self, dest_dir, name, branch, repo_path=None, # pylint: disable=R0913 self._config.set(name, ExternalsDescription.BRANCH, branch) - if repo_path: - if repo_path == SIMPLE_LOCAL_ONLY_NAME: + if new_remote_repo_path: + if new_remote_repo_path == SIMPLE_LOCAL_ONLY_NAME: repo_url = SIMPLE_LOCAL_ONLY_NAME else: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', - repo_path) + repo_url = os.path.join(self._bare_root, new_remote_repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) try: @@ -467,9 +396,9 @@ def write_with_git_branch(self, dest_dir, name, branch, repo_path=None, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def write_with_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): + def write_with_svn_branch(self, dest_dir, name, branch): """Update a repository branch, and potentially the remote. """ # pylint: disable=R0913 @@ -481,11 +410,11 @@ def write_with_svn_branch(self, dest_dir, name, branch, filename=CFG_NAME): except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def write_with_tag(self, dest_dir, name, tag, repo_path=None, - filename=CFG_NAME, remove_branch=True): - """Update a repository tag, and potentially the remote + def write_with_tag_and_remote_repo(self, dest_dir, name, tag, new_remote_repo_path, + remove_branch=True): + """Update a repository tag and the remote. NOTE(bja, 2017-11) remove_branch=False should result in an overspecified external with both a branch and tag. This is @@ -495,8 +424,8 @@ def write_with_tag(self, dest_dir, name, tag, repo_path=None, # pylint: disable=R0913 self._config.set(name, ExternalsDescription.TAG, tag) - if repo_path: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) + if new_remote_repo_path: + repo_url = os.path.join(self._bare_root, new_remote_repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) try: @@ -506,10 +435,9 @@ def write_with_tag(self, dest_dir, name, tag, repo_path=None, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def write_without_branch_tag(self, dest_dir, name, - filename=CFG_NAME): + def write_without_branch_tag(self, dest_dir, name): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 @@ -525,10 +453,9 @@ def write_without_branch_tag(self, dest_dir, name, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def write_without_repo_url(self, dest_dir, name, - filename=CFG_NAME): + def write_without_repo_url(self, dest_dir, name): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 @@ -538,22 +465,59 @@ def write_without_repo_url(self, dest_dir, name, except BaseException: pass - self.write_config(dest_dir, filename) + self.write_config(dest_dir) - def write_with_protocol(self, dest_dir, name, protocol, repo_path=None, - filename=CFG_NAME): + def write_with_protocol(self, dest_dir, name, protocol, repo_path=None): """Update a repository protocol, and potentially the remote """ # pylint: disable=R0913 self._config.set(name, ExternalsDescription.PROTOCOL, protocol) if repo_path: - repo_url = os.path.join('${MANIC_TEST_BARE_REPO_ROOT}', repo_path) + repo_url = os.path.join(self._bare_root, repo_path) self._config.set(name, ExternalsDescription.REPO_URL, repo_url) - self.write_config(dest_dir, filename) + self.write_config(dest_dir) + + +def _execute_checkout_in_dir(dirname, args, debug_env=''): + """Execute the checkout command in the appropriate repo dir with the + specified additional args. + args should be a list of strings. + debug_env shuld be a string of the form 'FOO=bar' or the empty string. + Note that we are calling the command line processing and main + routines and not using a subprocess call so that we get code + coverage results! Note this means that environment variables are passed + to checkout_externals via os.environ; debug_env is just used to aid + manual reproducibility of a given call. + + Returns (overall_status, tree_status) + where overall_status is 0 for success, nonzero otherwise. + and tree_status is set if --status was passed in, None otherwise. + + Note this command executes the checkout command, it doesn't + necessarily do any checking out (e.g. if --status is passed in). + """ + cwd = os.getcwd() + + # Construct a command line for reproducibility; this command is not + # actually executed in the test. + os.chdir(dirname) + cmdline = ['--externals', CFG_NAME, ] + cmdline += args + manual_cmd = ('Running equivalent of:\n' + 'pushd {dirname}; ' + '{debug_env} /path/to/checkout_externals {args}'.format( + dirname=dirname, debug_env=debug_env, + args=' '.join(cmdline))) + printlog(manual_cmd) + options = checkout.commandline_arguments(cmdline) + overall_status, tree_status = checkout.main(options) + os.chdir(cwd) + return overall_status, tree_status + class BaseTestSysCheckout(unittest.TestCase): """Base class of reusable systems level test setup for checkout_externals @@ -596,425 +560,57 @@ def setUp(self): # path to the executable self._checkout = os.path.join(root_dir, 'checkout_externals') - # directory where we have test repositories + # directory where we have test repositories (which we will clone for + # tests) self._bare_root = os.path.abspath( os.path.join(root_dir, 'test', BARE_REPO_ROOT_NAME)) - # set into the environment so var will be expanded in externals files - os.environ[MANIC_TEST_BARE_REPO_ROOT] = self._bare_root - # set the input file generator - self._generator = GenerateExternalsDescriptionCfgV1() + self._generator = GenerateExternalsDescriptionCfgV1(self._bare_root) # set the input file generator for secondary externals - self._sub_generator = GenerateExternalsDescriptionCfgV1() + self._sub_generator = GenerateExternalsDescriptionCfgV1(self._bare_root) def tearDown(self): """Tear down for individual tests """ - # remove the env var we added in setup - del os.environ[MANIC_TEST_BARE_REPO_ROOT] - # return to our common starting point os.chdir(self._return_dir) + + # (in case this was set) Don't pollute environment of other tests. + os.environ.pop(MIXED_CONT_EXT_ROOT_ENV_VAR, + None) # Don't care if key wasn't set. def clone_test_repo(self, parent_repo_name, dest_dir_in=None): - """Clone repo under bare_root into dest_dir_in or local per-test-subdir. - - Returns output dir. - """ - parent_repo_dir = os.path.join(self._bare_root, parent_repo_name) - if dest_dir_in is None: - # unique repo for this test - test_dir_name = self._test_id - print("Test repository name: {0}".format(test_dir_name)) - dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], - test_dir_name) - else: - dest_dir = dest_dir_in - - # pylint: disable=W0212 - GitRepository._git_clone(parent_repo_dir, dest_dir, VERBOSITY_DEFAULT) - return dest_dir - - @staticmethod - def _add_file_to_repo(under_test_dir, filename, tracked): - """Add a file to the repository so we can put it into a dirty state - - """ - cwd = os.getcwd() - os.chdir(under_test_dir) - with open(filename, 'w') as tmp: - tmp.write('Hello, world!') - - if tracked: - # NOTE(bja, 2018-01) brittle hack to obtain repo dir and - # file name - path_data = filename.split('/') - repo_dir = os.path.join(path_data[0], path_data[1]) - os.chdir(repo_dir) - tracked_file = path_data[2] - cmd = ['git', 'add', tracked_file] - execute_subprocess(cmd) - - os.chdir(cwd) + """Clones repo under self._bare_root""" + return RepoUtils.clone_test_repo(self._bare_root, self._test_id, + parent_repo_name, dest_dir_in) + + def execute_checkout_in_dir(self, dirname, args, debug_env=''): + overall_status, tree_status = _execute_checkout_in_dir(dirname, args, + debug_env=debug_env) + self.assertEqual(overall_status, 0) + return tree_status + + def execute_checkout_with_status(self, dirname, args, debug_env=''): + """Calls checkout a second time to get status if needed.""" + tree_status = self.execute_checkout_in_dir( + dirname, args, debug_env=debug_env) + if tree_status is None: + tree_status = self.execute_checkout_in_dir(dirname, + self.status_args, + debug_env=debug_env) + self.assertNotEqual(tree_status, None) + return tree_status + + def _check_sync_clean(self, ext_status, expected_sync_state, + expected_clean_state): + self.assertEqual(ext_status.sync_state, expected_sync_state) + self.assertEqual(ext_status.clean_state, expected_clean_state) @staticmethod - def execute_checkout_in_dir(dirname, args): - """Extecute the checkout command in the appropriate repo dir with the - specified additional args - - Note that we are calling the command line processing and main - routines and not using a subprocess call so that we get code - coverage results! - - """ - cwd = os.getcwd() - checkout_path = os.path.abspath('{0}/../../checkout_externals') - os.chdir(dirname) - cmdline = ['--externals', CFG_NAME, ] - cmdline += args - repo_root = 'MANIC_TEST_BARE_REPO_ROOT={root}'.format( - root=os.environ[MANIC_TEST_BARE_REPO_ROOT]) - manual_cmd = ('Test cmd:\npushd {cwd}; {env} {checkout} {args}'.format( - cwd=dirname, env=repo_root, checkout=checkout_path, - args=' '.join(cmdline))) - printlog(manual_cmd) - options = checkout.commandline_arguments(cmdline) - overall_status, tree_status = checkout.main(options) - os.chdir(cwd) - return overall_status, tree_status - - # ---------------------------------------------------------------- - # - # Check results for generic perturbation of states - # - # 'tree' is a dict with ExternalStatus as values. - # 'name' is a key into that dict. - # ---------------------------------------------------------------- - def _check_generic_empty_default_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.EMPTY) - self.assertEqual(tree[name].clean_state, ExternalStatus.DEFAULT) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_ok_clean_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].clean_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_ok_dirty_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].clean_state, ExternalStatus.DIRTY) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_modified_ok_required(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.MODEL_MODIFIED) - self.assertEqual(tree[name].clean_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].source_type, ExternalStatus.MANAGED) - - def _check_generic_empty_default_optional(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.EMPTY) - self.assertEqual(tree[name].clean_state, ExternalStatus.DEFAULT) - self.assertEqual(tree[name].source_type, ExternalStatus.OPTIONAL) - - def _check_generic_ok_clean_optional(self, tree, name): - self.assertEqual(tree[name].sync_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].clean_state, ExternalStatus.STATUS_OK) - self.assertEqual(tree[name].source_type, ExternalStatus.OPTIONAL) - - # ---------------------------------------------------------------- - # - # Check results for individual named externals - # - # 'tree' is a dict of string to ExternalStatus. - # ---------------------------------------------------------------- - def _check_simple_tag_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_nested_tag_empty(self, tree, name=EXTERNALS_NAME): - self._check_generic_empty_default_required(tree, name) - - def _check_simple_tag_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_nested_tag_ok(self, tree, name=EXTERNALS_NAME): - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_tag_dirty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_ok_dirty_required(tree, name) - - def _check_simple_tag_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_tag'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_branch_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_branch'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_nested_branch_empty(self, tree, name=EXTERNALS_NAME): - self._check_generic_empty_default_required(tree, name) - - def _check_simple_branch_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_branch'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_nested_branch_ok(self, tree, name=EXTERNALS_NAME): - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_branch_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_branch'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_hash_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_hash'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_nested_hash_empty(self, tree, name=EXTERNALS_NAME): - self._check_generic_empty_default_required(tree, name) - - def _check_simple_hash_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_hash'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_nested_hash_ok(self, tree, name=EXTERNALS_NAME): - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_hash_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_hash'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_req_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_req'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_simple_req_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_req'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_simple_opt_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_opt'.format(directory) - self._check_generic_empty_default_optional(tree, name) - - def _check_simple_opt_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_opt'.format(directory) - self._check_generic_ok_clean_optional(tree, name) - - def _check_mixed_ext_branch_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/mixed_req'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_mixed_ext_branch_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/mixed_req'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_mixed_ext_branch_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/mixed_req'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_simple_sparse_empty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_sparse'.format(directory) - self._check_generic_empty_default_required(tree, name) - - def _check_simple_sparse_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/simp_sparse'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - # ---------------------------------------------------------------- - # - # Check results for groups of externals under specific conditions - # - # ---------------------------------------------------------------- - def _check_container_simple_required_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - self._check_simple_hash_empty(tree) - - def _check_container_nested_required_pre_checkout(self, overall, tree, order): - self.assertEqual(overall, 0) - self._check_nested_tag_empty(tree, name=NESTED_NAME[order[0]]) - self._check_nested_branch_empty(tree, name=NESTED_NAME[order[1]]) - self._check_nested_hash_empty(tree, name=NESTED_NAME[order[2]]) - - def _check_container_simple_required_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - self._check_simple_hash_empty(tree) - - def _check_container_nested_required_checkout(self, overall, tree, order): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_nested_tag_empty(tree, name=NESTED_NAME[order[0]]) - self._check_nested_branch_empty(tree, name=NESTED_NAME[order[1]]) - self._check_nested_hash_empty(tree, name=NESTED_NAME[order[2]]) - - def _check_container_simple_required_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_hash_ok(tree) - - def _check_container_nested_required_post_checkout(self, overall, tree, order): - self.assertEqual(overall, 0) - self._check_nested_tag_ok(tree, name=NESTED_NAME[order[0]]) - self._check_nested_branch_ok(tree, name=NESTED_NAME[order[1]]) - self._check_nested_hash_ok(tree, name=NESTED_NAME[order[2]]) - - def _check_container_simple_required_out_of_sync(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_modified(tree) - self._check_simple_branch_modified(tree) - self._check_simple_hash_modified(tree) - - def _check_container_simple_optional_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_empty(tree) - self._check_simple_opt_empty(tree) - - def _check_container_simple_optional_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_empty(tree) - self._check_simple_opt_empty(tree) - - def _check_container_simple_optional_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_ok(tree) - self._check_simple_opt_empty(tree) - - def _check_container_simple_optional_post_optional(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_req_ok(tree) - self._check_simple_opt_ok(tree) - - def _check_container_simple_required_sb_modified(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_modified(tree) - self._check_simple_hash_ok(tree) - - def _check_container_simple_optional_st_dirty(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_dirty(tree) - self._check_simple_branch_ok(tree) - - def _check_container_full_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_pre_checkout(overall, tree) - - def _check_container_component_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_opt_ok(tree) - self._check_simple_tag_empty(tree) - self._check_simple_branch_empty(tree) - - def _check_container_component_post_checkout2(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_branch_ok(tree) - - def _check_container_component_post_checkout3(self, overall, tree): - self.assertEqual(overall, 0) - self.assertFalse("simp_opt" in tree) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - - def _check_container_full_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_post_checkout(overall, tree) - - def _check_container_full_pre_checkout_ext_change(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_pre_checkout_ext_change( - overall, tree) - - def _check_container_full_post_checkout_subext_modified( - self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_branch_ok(tree) - self._check_simple_opt_empty(tree) - self._check_mixed_ext_branch_required_post_checkout_subext_modified( - overall, tree) - - def _check_mixed_ext_branch_required_pre_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_empty(tree, directory=EXTERNALS_NAME) - # NOTE: externals/mixed_req/src should not exist in the tree - # since this is the status before checkout of mixed_req. - - def _check_mixed_ext_branch_required_post_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_ok(tree, directory=EXTERNALS_NAME) - check_dir = "{0}/{1}/{2}".format(EXTERNALS_NAME, "mixed_req", - SUB_EXTERNALS_PATH) - self._check_simple_branch_ok(tree, directory=check_dir) - - def _check_mixed_ext_branch_required_pre_checkout_ext_change( - self, overall, tree): - # Note, this is the internal tree status just after change the - # externals description file, but before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_modified(tree, directory=EXTERNALS_NAME) - check_dir = "{0}/{1}/{2}".format(EXTERNALS_NAME, "mixed_req", - SUB_EXTERNALS_PATH) - self._check_simple_branch_ok(tree, directory=check_dir) - - def _check_mixed_ext_branch_required_post_checkout_subext_modified( - self, overall, tree): - # Note, this is the internal tree status just after change the - # externals description file, but before checkout - self.assertEqual(overall, 0) - self._check_mixed_ext_branch_ok(tree, directory=EXTERNALS_NAME) - check_dir = "{0}/{1}/{2}".format(EXTERNALS_NAME, "mixed_req", - SUB_EXTERNALS_PATH) - self._check_simple_branch_modified(tree, directory=check_dir) - - def _check_mixed_cont_simple_required_pre_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=SUB_EXTERNALS_PATH) - - def _check_mixed_cont_simple_required_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_empty(tree, directory=SUB_EXTERNALS_PATH) - - def _check_mixed_cont_simple_required_post_checkout(self, overall, tree): - # Note, this is the internal tree status just before checkout - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_ok(tree, directory=EXTERNALS_NAME) - self._check_simple_branch_ok(tree, directory=SUB_EXTERNALS_PATH) - - def _check_container_sparse_pre_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_empty(tree) - self._check_simple_sparse_empty(tree) - - def _check_container_sparse_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_simple_sparse_ok(tree) - + def _external_path(section_name, base_path=EXTERNALS_PATH): + return './{0}/{1}'.format(base_path, section_name) + def _check_file_exists(self, repo_dir, pathname): "Check that exists in " self.assertTrue(os.path.exists(os.path.join(repo_dir, pathname))) @@ -1023,9 +619,9 @@ def _check_file_absent(self, repo_dir, pathname): "Check that does not exist in " self.assertFalse(os.path.exists(os.path.join(repo_dir, pathname))) + class TestSysCheckout(BaseTestSysCheckout): """Run systems level tests of checkout_externals - """ # NOTE(bja, 2017-11) pylint complains about long method names, but # it is hard to differentiate tests without making them more @@ -1037,246 +633,431 @@ class TestSysCheckout(BaseTestSysCheckout): # Run systems tests # # ---------------------------------------------------------------- - def test_container_simple_required(self): - """Verify that a container with simple subrepos - generates the correct initial status. - - """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # status of empty repo - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_pre_checkout(overall, tree) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # status clean checked out - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) - + def test_required_bytag(self): + """Check out a required external pointing to a git tag.""" + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + local_path_rel = self._external_path(TAG_SECTION) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + local_path_abs = os.path.join(cloned_repo_dir, local_path_rel) + self.assertFalse(os.path.exists(local_path_abs)) + + # after checkout, the external is 'clean' aka at the correct version. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # Actually checked out the desired repo. + self.assertEqual('origin', GitRepository._remote_name_for_url( + # Which url to look up + self._generator.url_for_repo_path(SIMPLE_REPO), + # Which directory has the local checked-out repo. + dirname=local_path_abs)) + + # Actually checked out the desired tag. + (tag_found, tag_name) = GitRepository._git_current_tag(local_path_abs) + self.assertEqual(tag_name, 'tag1') + + # Check existence of some simp_tag files + tag_path = os.path.join('externals', TAG_SECTION) + self._check_file_exists(cloned_repo_dir, + os.path.join(tag_path, README_NAME)) + # Subrepo should not exist (not referenced by configs). + self._check_file_absent(cloned_repo_dir, os.path.join(tag_path, + 'simple_subdir', + 'subdir_file.txt')) + + def test_required_bybranch(self): + """Check out a required external pointing to a git branch.""" + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + local_path_rel = self._external_path(BRANCH_SECTION) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + local_path_abs = os.path.join(cloned_repo_dir, local_path_rel) + self.assertFalse(os.path.exists(local_path_abs)) + + # after checkout, the external is 'clean' aka at the correct version. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertTrue(os.path.exists(local_path_abs)) + + # Actually checked out the desired repo. + self.assertEqual('origin', GitRepository._remote_name_for_url( + # Which url to look up + self._generator.url_for_repo_path(SIMPLE_REPO), + # Which directory has the local checked-out repo. + dirname=local_path_abs)) + + # Actually checked out the desired branch. + (branch_found, branch_name) = GitRepository._git_current_remote_branch( + local_path_abs) + self.assertEquals(branch_name, 'origin/' + REMOTE_BRANCH_FEATURE2) + + def test_required_byhash(self): + """Check out a required external pointing to a git hash.""" + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, HASH_SECTION, + ref_hash='60b1cc1a38d63') + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + local_path_rel = self._external_path(HASH_SECTION) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + local_path_abs = os.path.join(cloned_repo_dir, local_path_rel) + self.assertFalse(os.path.exists(local_path_abs)) + + # after checkout, the externals are 'clean' aka at their correct version. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[local_path_rel], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # Actually checked out the desired repo. + self.assertEqual('origin', GitRepository._remote_name_for_url( + # Which url to look up + self._generator.url_for_repo_path(SIMPLE_REPO), + # Which directory has the local checked-out repo. + dirname=local_path_abs)) + + # Actually checked out the desired hash. + (hash_found, hash_name) = GitRepository._git_current_hash( + local_path_abs) + self.assertTrue(hash_name.startswith('60b1cc1a38d63'), + msg=hash_name) + def test_container_nested_required(self): - """Verify that a container with nested subrepos - generates the correct initial status. + """Verify that a container with nested subrepos generates the correct initial status. Tests over all possible permutations """ + # Output subdirs for each of the externals, to test that one external can be + # checked out in a subdir of another. + NESTED_SUBDIR = ['./fred', './fred/wilma', './fred/wilma/barney'] + # Assert that each type of external (e.g. tag vs branch) can be at any parent level + # (e.g. child/parent/grandparent). orders = [[0, 1, 2], [1, 2, 0], [2, 0, 1], [0, 2, 1], [2, 1, 0], [1, 0, 2]] for n, order in enumerate(orders): - # create repo - dest_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], - self._test_id, "test"+str(n)) - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME, - dest_dir_in=dest_dir) - self._generator.container_nested_required(cloned_repo_dir, order) - - # status of empty repo - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_nested_required_pre_checkout(overall, tree, order) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_nested_required_checkout(overall, tree, order) - - # status clean checked out - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_nested_required_post_checkout(overall, tree, order) - + dest_dir = os.path.join(module_tmp_root_dir, self._test_id, + "test"+str(n)) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO, + dest_dir_in=dest_dir) + self._generator.create_config() + # We happen to check out each section via a different reference (tag/branch/hash) but + # those don't really matter, we just need to check out three repos into a nested set of + # directories. + self._generator.create_section( + SIMPLE_REPO, TAG_SECTION, nested=True, + tag='tag1', path=NESTED_SUBDIR[order[0]]) + self._generator.create_section( + SIMPLE_REPO, BRANCH_SECTION, nested=True, + branch=REMOTE_BRANCH_FEATURE2, path=NESTED_SUBDIR[order[1]]) + self._generator.create_section( + SIMPLE_REPO, HASH_SECTION, nested=True, + ref_hash='60b1cc1a38d63', path=NESTED_SUBDIR[order[2]]) + self._generator.write_config(cloned_repo_dir) + + # all externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + self._check_sync_clean(tree[NESTED_SUBDIR[order[0]]], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[NESTED_SUBDIR[order[1]]], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[NESTED_SUBDIR[order[2]]], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + + # after checkout, all the repos are 'clean'. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[NESTED_SUBDIR[order[0]]], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[NESTED_SUBDIR[order[1]]], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[NESTED_SUBDIR[order[2]]], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + def test_container_simple_optional(self): - """Verify that container with an optional simple subrepos - generates the correct initial status. + """Verify that container with an optional simple subrepos generates + the correct initial status. """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_optional(cloned_repo_dir) - - # check status of empty repo - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_optional_pre_checkout(overall, tree) - - # checkout required - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_optional_checkout(overall, tree) - - # status - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_optional_post_checkout(overall, tree) + # create repo and externals config. + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, 'simp_req', + tag='tag1') - # checkout optional - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.optional_args) - self._check_container_simple_optional_post_checkout(overall, tree) + self._generator.create_section(SIMPLE_REPO, 'simp_opt', + tag='tag1', required=False) - # status - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_optional_post_optional(overall, tree) + self._generator.write_config(cloned_repo_dir) + + # all externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + req_status = tree[self._external_path('simp_req')] + self._check_sync_clean(req_status, + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self.assertEqual(req_status.source_type, ExternalStatus.MANAGED) + + opt_status = tree[self._external_path('simp_opt')] + self._check_sync_clean(opt_status, + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self.assertEqual(opt_status.source_type, ExternalStatus.OPTIONAL) + + # after checkout, required external is clean, optional is still empty. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + req_status = tree[self._external_path('simp_req')] + self._check_sync_clean(req_status, + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertEqual(req_status.source_type, ExternalStatus.MANAGED) + + opt_status = tree[self._external_path('simp_opt')] + self._check_sync_clean(opt_status, + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self.assertEqual(opt_status.source_type, ExternalStatus.OPTIONAL) + + # after checking out optionals, the optional external is also clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.optional_args) + req_status = tree[self._external_path('simp_req')] + self._check_sync_clean(req_status, + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertEqual(req_status.source_type, ExternalStatus.MANAGED) + + opt_status = tree[self._external_path('simp_opt')] + self._check_sync_clean(opt_status, + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self.assertEqual(opt_status.source_type, ExternalStatus.OPTIONAL) def test_container_simple_verbose(self): - """Verify that container with simple subrepos runs with verbose status - output and generates the correct initial status. - + """Verify that verbose status matches non-verbose. """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # check verbose status - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.verbose_args) - self._check_container_simple_required_post_checkout(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # after checkout, all externals should be 'clean'. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # 'Verbose' status should tell the same story. + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.verbose_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_simple_dirty(self): - """Verify that a container with simple subrepos - and a dirty status exits gracefully. - + """Verify that a container with a new tracked file is marked dirty. """ - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # add a file to the repo - tracked = True - self._add_file_to_repo(cloned_repo_dir, 'externals/simp_tag/tmp.txt', - tracked) - - # checkout: pre-checkout status should be dirty, did not - # modify working copy. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_optional_st_dirty(overall, tree) - - # verify status is still dirty - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_optional_st_dirty(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # checkout, should start out clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # add a tracked file to the simp_tag external, should be dirty. + RepoUtils.add_file_to_repo(cloned_repo_dir, + 'externals/{0}/tmp.txt'.format(TAG_SECTION), + tracked=True) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.DIRTY) + + # Re-checkout; simp_tag should still be dirty. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.DIRTY) def test_container_simple_untracked(self): """Verify that a container with simple subrepos and a untracked files is not considered 'dirty' and will attempt an update. """ - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # add a file to the repo - tracked = False - self._add_file_to_repo(cloned_repo_dir, 'externals/simp_tag/tmp.txt', - tracked) - - # checkout: pre-checkout status should be clean, ignoring the - # untracked file. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_post_checkout(overall, tree) - - # verify status is still clean - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + self._generator.write_config(cloned_repo_dir) + + # checkout, should start out clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # add an untracked file to the simp_tag external, should stay clean. + RepoUtils.add_file_to_repo(cloned_repo_dir, + 'externals/{0}/tmp.txt'.format(TAG_SECTION), + tracked=False) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # After checkout, the external should still be 'clean'. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_simple_detached_sync(self): """Verify that a container with simple subrepos generates the correct out of sync status when making commits from a detached head - state. + state. + For more info about 'detached head' state: https://www.cloudbees.com/blog/git-detached-head """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # status of empty repo - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_pre_checkout(overall, tree) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # make a commit on the detached head of the tag and hash externals - self._generator.create_commit(cloned_repo_dir, 'simp_tag') - self._generator.create_commit(cloned_repo_dir, 'simp_hash') - self._generator.create_commit(cloned_repo_dir, 'simp_branch') - - # status of repo, branch, tag and hash should all be out of sync! - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_out_of_sync(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + + self._generator.create_section(SIMPLE_REPO, 'simp_hash', + ref_hash='60b1cc1a38d63') + + self._generator.write_config(cloned_repo_dir) + + # externals start out 'empty' aka not checked out. + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - # same pre-checkout out of sync status - self._check_container_simple_required_out_of_sync(overall, tree) - - # now status should be in-sync - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) + + # Commit on top of the tag and hash (creating the detached head state in those two + # externals' repos) + # The branch commit does not create the detached head state, but here for completeness. + RepoUtils.create_commit(cloned_repo_dir, TAG_SECTION) + RepoUtils.create_commit(cloned_repo_dir, HASH_SECTION) + RepoUtils.create_commit(cloned_repo_dir, BRANCH_SECTION) + + # sync status of all three should be 'modified' (uncommitted changes) + # clean status is 'ok' (matches externals version) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # after checkout, all externals should be totally clean (no uncommitted changes, + # and matches externals version). + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_remote_branch(self): """Verify that a container with remote branch change works """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) - - # update the config file to point to a different remote with - # the same branch - self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', - branch=REMOTE_BRANCH_FEATURE2, - repo_path=SIMPLE_FORK_NAME) - - # status of simp_branch should be out of sync - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # checkout new externals - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # status should be synced - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) + + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) + + # update the branch external to point to a different remote with the same branch, + # then simp_branch should be out of sync + self._generator.write_with_git_branch(cloned_repo_dir, + name=BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2, + new_remote_repo_path=SIMPLE_FORK_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # checkout new externals, now simp_branch should be clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_remote_tag_same_branch(self): """Verify that a container with remote tag change works. The new tag @@ -1285,276 +1066,323 @@ def test_container_remote_tag_same_branch(self): the branch. """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update the config file to point to a different remote with - # the tag instead of branch. Tag MUST NOT be in the original - # repo! - self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', - 'forked-feature-v1', SIMPLE_FORK_NAME) - - # status of simp_branch should be out of sync - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # checkout new externals - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # status should be synced - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + # the new tag replacing the old branch. Tag MUST NOT be in the original + # repo! status of simp_branch should then be out of sync + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='forked-feature-v1', + new_remote_repo_path=SIMPLE_FORK_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.status_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # checkout new externals, then should be synced. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_remote_tag_fetch_all(self): """Verify that a container with remote tag change works. The new tag should not be in the original repo, only the new remote - fork. It should also not be on a branch that will be fetch, + fork. It should also not be on a branch that will be fetched, and therefore not fetched by default with 'git fetch'. It will - only be retreived by 'git fetch --tags' - + only be retrieved by 'git fetch --tags' """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update the config file to point to a different remote with - # the tag instead of branch. Tag MUST NOT be in the original - # repo! - self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', - 'abandoned-feature', SIMPLE_FORK_NAME) - - # status of simp_branch should be out of sync - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # checkout new externals - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_sb_modified(overall, tree) - - # status should be synced - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + # the new tag instead of the old branch. Tag MUST NOT be in the original + # repo! status of simp_branch should then be out of sync. + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='abandoned-feature', + new_remote_repo_path=SIMPLE_FORK_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + + # checkout new externals, should be clean again. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_preserve_dot(self): """Verify that after inital checkout, modifying an external git repo url to '.' and the current branch will leave it unchanged. """ - # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_required_checkout(overall, tree) + # initial checkout + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update the config file to point to a different remote with - # the same branch - self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', - branch=REMOTE_BRANCH_FEATURE2, - repo_path=SIMPLE_FORK_NAME) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - - # verify status is clean and unmodified - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) + # the same branch. + self._generator.write_with_git_branch(cloned_repo_dir, name=BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2, + new_remote_repo_path=SIMPLE_FORK_REPO) + # after checkout, should be clean again. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) # update branch to point to a new branch that only exists in # the local fork - self._generator.create_branch(cloned_repo_dir, repo_name='simp_branch', - branch='private-feature', with_commit=True) - self._generator.write_with_git_branch(cloned_repo_dir, name='simp_branch', - branch='private-feature', - repo_path=SIMPLE_LOCAL_ONLY_NAME) - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - - # verify status is clean and unmodified - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_required_post_checkout(overall, tree) - - def test_container_full(self): - """Verify that 'full' container with simple and mixed subrepos - generates the correct initial status. + RepoUtils.create_branch(cloned_repo_dir, external_name=BRANCH_SECTION, + branch='private-feature', with_commit=True) + self._generator.write_with_git_branch(cloned_repo_dir, name=BRANCH_SECTION, + branch='private-feature', + new_remote_repo_path=SIMPLE_LOCAL_ONLY_NAME) + # after checkout, should be clean again. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + def test_container_mixed_subrepo(self): + """Verify container with mixed subrepo. The mixed subrepo has a sub-externals file with different sub-externals on different branches. """ - # create the test repository - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - - # create the top level externals file - self._generator.container_full(cloned_repo_dir) - - # inital checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_full_pre_checkout(overall, tree) - - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_full_post_checkout(overall, tree) - - # Check existance of some files - subrepo_path = os.path.join('externals', 'simp_tag') - self._check_file_exists(cloned_repo_dir, - os.path.join(subrepo_path, 'readme.txt')) - self._check_file_absent(cloned_repo_dir, os.path.join(subrepo_path, - 'simple_subdir', - 'subdir_file.txt')) - - # update the mixed-use repo to point to different branch - self._generator.write_with_git_branch(cloned_repo_dir, name='mixed_req', - branch='new-feature', - repo_path=MIXED_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) - # check status out of sync for mixed_req, but sub-externals + self._generator.create_config() + self._generator.create_section(MIXED_REPO, 'mixed_req', + branch='master', sub_externals=CFG_SUB_NAME) + self._generator.write_config(cloned_repo_dir) + + # The subrepo has a repo_url that uses this environment variable. + # It'll be cleared in tearDown(). + os.environ[MIXED_CONT_EXT_ROOT_ENV_VAR] = self._bare_root + debug_env = MIXED_CONT_EXT_ROOT_ENV_VAR + '=' + self._bare_root + + # inital checkout: all requireds are clean, and optional is empty. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args, + debug_env=debug_env) + mixed_req_path = self._external_path('mixed_req') + self._check_sync_clean(tree[mixed_req_path], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + sub_ext_base_path = "{0}/{1}/{2}".format(EXTERNALS_PATH, 'mixed_req', SUB_EXTERNALS_PATH) + # The already-checked-in subexternals file has a 'simp_branch' section + self._check_sync_clean(tree[self._external_path('simp_branch', base_path=sub_ext_base_path)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # update the mixed-use external to point to different branch + # status should become out of sync for mixed_req, but sub-externals # are still in sync - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_full_pre_checkout_ext_change(overall, tree) - - # run the checkout. Now the mixed use external and it's - # sub-exterals should be changed. Returned status is - # pre-checkout! - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_full_pre_checkout_ext_change(overall, tree) - - # check status out of sync for mixed_req, and sub-externals - # are in sync. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_full_post_checkout(overall, tree) - + self._generator.write_with_git_branch(cloned_repo_dir, name='mixed_req', + branch='new-feature', + new_remote_repo_path=MIXED_REPO) + tree = self.execute_checkout_in_dir(cloned_repo_dir, self.status_args, + debug_env=debug_env) + self._check_sync_clean(tree[mixed_req_path], + ExternalStatus.MODEL_MODIFIED, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path('simp_branch', base_path=sub_ext_base_path)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # run the checkout. Now the mixed use external and its sub-externals should be clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, self.checkout_args, + debug_env=debug_env) + self._check_sync_clean(tree[mixed_req_path], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path('simp_branch', base_path=sub_ext_base_path)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + def test_container_component(self): """Verify that optional component checkout works """ - # create the test repository - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) # create the top level externals file - self._generator.container_full(cloned_repo_dir) - - # inital checkout, first try a nonexistant component argument noref + self._generator.create_config() + # Optional external, by tag. + self._generator.create_section(SIMPLE_REPO, 'simp_opt', + tag='tag1', required=False) + + # Required external, by branch. + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + + # Required external, by hash. + self._generator.create_section(SIMPLE_REPO, HASH_SECTION, + ref_hash='60b1cc1a38d63') + self._generator.write_config(cloned_repo_dir) + + # inital checkout, first try a nonexistent component argument noref checkout_args = ['simp_opt', 'noref'] checkout_args.extend(self.checkout_args) with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) + # Now explicitly check out one optional component.. + # Explicitly listed component (opt) should be present, the other two not. checkout_args = ['simp_opt'] checkout_args.extend(self.checkout_args) + tree = self.execute_checkout_with_status(cloned_repo_dir, + checkout_args) + self._check_sync_clean(tree[self._external_path('simp_opt')], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + + # Check out a second component, this one required. + # Explicitly listed component (branch) should be present, the still-unlisted one (tag) not. + checkout_args.append(BRANCH_SECTION) + tree = self.execute_checkout_with_status(cloned_repo_dir, + checkout_args) + self._check_sync_clean(tree[self._external_path('simp_opt')], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - checkout_args) - - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_component_post_checkout(overall, tree) - checkout_args.append('simp_branch') - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - checkout_args) - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_component_post_checkout2(overall, tree) def test_container_exclude_component(self): """Verify that exclude component checkout works """ - # create the test repository - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - - # create the top level externals file - self._generator.container_full(cloned_repo_dir) - - # inital checkout, exclude simp_opt - checkout_args = ['--exclude', 'simp_opt'] + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag1') + + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + + self._generator.create_section(SIMPLE_REPO, 'simp_hash', + ref_hash='60b1cc1a38d63') + + self._generator.write_config(cloned_repo_dir) + + # inital checkout should result in all externals being clean except excluded TAG_SECTION. + checkout_args = ['--exclude', TAG_SECTION] checkout_args.extend(self.checkout_args) - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) - checkout_args.append("--status") - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, checkout_args) - self._check_container_component_post_checkout3(overall, tree) - - def test_mixed_simple(self): - """Verify that a mixed use repo can serve as a 'full' container, - pulling in a set of externals and a seperate set of sub-externals. + tree = self.execute_checkout_with_status(cloned_repo_dir, checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.EMPTY, + ExternalStatus.DEFAULT) + self._check_sync_clean(tree[self._external_path(BRANCH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path(HASH_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + def test_subexternal(self): + """Verify that an externals file can be brought in as a reference. """ - #import pdb; pdb.set_trace() - # create repository - cloned_repo_dir = self.clone_test_repo(MIXED_REPO_NAME) - # create top level externals file - self._generator.mixed_simple_base(cloned_repo_dir) - # NOTE: sub-externals file is already in the repo so we can - # switch branches during testing. Since this is a mixed-repo - # serving as the top level container repo, we can't switch - # during this test. + cloned_repo_dir = self.clone_test_repo(MIXED_REPO) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_mixed_cont_simple_required_checkout(overall, tree) - - # verify status is clean and unmodified - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_mixed_cont_simple_required_post_checkout(overall, tree) + self._generator.create_config() + self._generator.create_section_reference_to_subexternal('mixed_base') + self._generator.write_config(cloned_repo_dir) + + # The subrepo has a repo_url that uses this environment variable. + # It'll be cleared in tearDown(). + os.environ[MIXED_CONT_EXT_ROOT_ENV_VAR] = self._bare_root + debug_env = MIXED_CONT_EXT_ROOT_ENV_VAR + '=' + self._bare_root + + # After checkout, confirm required's are clean and the referenced + # subexternal's contents are also clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args, + debug_env=debug_env) + + self._check_sync_clean( + tree[self._external_path(BRANCH_SECTION, base_path=SUB_EXTERNALS_PATH)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) def test_container_sparse(self): """Verify that 'full' container with simple subrepo can run a sparse checkout and generate the correct initial status. """ - # create the test repository - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) - # create the top level externals file - self._generator.container_sparse(cloned_repo_dir) - - # inital checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_sparse_pre_checkout(overall, tree) - - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_sparse_post_checkout(overall, tree) + # Create a file to list filenames to checkout. + sparse_filename = 'sparse_checkout' + with open(os.path.join(cloned_repo_dir, sparse_filename), 'w') as sfile: + sfile.write(README_NAME) - # Check existance of some files - subrepo_path = os.path.join('externals', 'simp_tag') + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, + tag='tag2') + + # Same tag as above, but with a sparse file too. + sparse_relpath = '../../' + sparse_filename + self._generator.create_section(SIMPLE_REPO, 'simp_sparse', + tag='tag2', sparse=sparse_relpath) + + self._generator.write_config(cloned_repo_dir) + + # inital checkout, confirm required's are clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._external_path('simp_sparse')], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + + # Check existence of some files - full set in TAG_SECTION, and sparse set + # in 'simp_sparse'. + subrepo_path = os.path.join('externals', TAG_SECTION) self._check_file_exists(cloned_repo_dir, - os.path.join(subrepo_path, 'readme.txt')) + os.path.join(subrepo_path, README_NAME)) self._check_file_exists(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) subrepo_path = os.path.join('externals', 'simp_sparse') self._check_file_exists(cloned_repo_dir, - os.path.join(subrepo_path, 'readme.txt')) + os.path.join(subrepo_path, README_NAME)) self._check_file_absent(cloned_repo_dir, os.path.join(subrepo_path, 'simple_subdir', 'subdir_file.txt')) @@ -1591,42 +1419,27 @@ class TestSysCheckoutSVN(BaseTestSysCheckout): """ - def _check_svn_branch_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_branch'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_svn_branch_dirty(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_branch'.format(directory) - self._check_generic_ok_dirty_required(tree, name) - - def _check_svn_tag_ok(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_tag'.format(directory) - self._check_generic_ok_clean_required(tree, name) - - def _check_svn_tag_modified(self, tree, directory=EXTERNALS_NAME): - name = './{0}/svn_tag'.format(directory) - self._check_generic_modified_ok_required(tree, name) - - def _check_container_simple_svn_post_checkout(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_svn_branch_ok(tree) - self._check_svn_tag_ok(tree) - - def _check_container_simple_svn_sb_dirty_st_mod(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_svn_tag_modified(tree) - self._check_svn_branch_dirty(tree) + @staticmethod + def _svn_branch_name(): + return './{0}/svn_branch'.format(EXTERNALS_PATH) - def _check_container_simple_svn_sb_clean_st_mod(self, overall, tree): - self.assertEqual(overall, 0) - self._check_simple_tag_ok(tree) - self._check_svn_tag_modified(tree) - self._check_svn_branch_ok(tree) + @staticmethod + def _svn_tag_name(): + return './{0}/svn_tag'.format(EXTERNALS_PATH) + + def _check_tag_branch_svn_tag_clean(self, tree): + self._check_sync_clean(tree[self._external_path(TAG_SECTION)], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._svn_branch_name()], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) + self._check_sync_clean(tree[self._svn_tag_name()], + ExternalStatus.STATUS_OK, + ExternalStatus.STATUS_OK) @staticmethod - def have_svn_access(): + def _have_svn_access(): """Check if we have svn access so we can enable tests that use svn. """ @@ -1639,10 +1452,10 @@ def have_svn_access(): pass return have_svn - def skip_if_no_svn_access(self): + def _skip_if_no_svn_access(self): """Function decorator to disable svn tests when svn isn't available """ - have_svn = self.have_svn_access() + have_svn = self._have_svn_access() if not have_svn: raise unittest.SkipTest("No svn access") @@ -1650,60 +1463,55 @@ def test_container_simple_svn(self): """Verify that a container repo can pull in an svn branch and svn tag. """ - self.skip_if_no_svn_access() + self._skip_if_no_svn_access() # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_svn(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) + self._generator.create_config() + # Git repo. + self._generator.create_section(SIMPLE_REPO, TAG_SECTION, tag='tag1') + + # Svn repos. + self._generator.create_svn_external('svn_branch', branch='trunk') + self._generator.create_svn_external('svn_tag', tag='tags/cesm2.0.beta07') - # verify status is clean and unmodified - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_svn_post_checkout(overall, tree) + self._generator.write_config(cloned_repo_dir) + + # checkout, make sure all sections are clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_tag_branch_svn_tag_clean(tree) # update description file to make the tag into a branch and # trigger a switch - self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', 'trunk') - - # checkout - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) + self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', + 'trunk') - # verify status is clean and unmodified - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.status_args) - self._check_container_simple_svn_post_checkout(overall, tree) + # checkout, again the results should be clean. + tree = self.execute_checkout_with_status(cloned_repo_dir, + self.checkout_args) + self._check_tag_branch_svn_tag_clean(tree) # add an untracked file to the repo tracked = False - self._add_file_to_repo(cloned_repo_dir, - 'externals/svn_branch/tmp.txt', tracked) + RepoUtils.add_file_to_repo(cloned_repo_dir, + 'externals/svn_branch/tmp.txt', tracked) - # run a no-op checkout: pre-checkout status should be clean, - # ignoring the untracked file. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_svn_post_checkout(overall, tree) + # run a no-op checkout. + self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) # update description file to make the branch into a tag and # trigger a modified sync status self._generator.write_with_svn_branch(cloned_repo_dir, 'svn_tag', 'tags/cesm2.0.beta07') - # checkout: pre-checkout status should be clean and modified, - # will modify working copy. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.checkout_args) - self._check_container_simple_svn_sb_clean_st_mod(overall, tree) + self.execute_checkout_in_dir(cloned_repo_dir,self.checkout_args) # verify status is still clean and unmodified, last # checkout modified the working dir state. - overall, tree = self.execute_checkout_in_dir(cloned_repo_dir, - self.verbose_args) - self._check_container_simple_svn_post_checkout(overall, tree) + tree = self.execute_checkout_in_dir(cloned_repo_dir, + self.verbose_args) + self._check_tag_branch_svn_tag_clean(tree) class TestSubrepoCheckout(BaseTestSysCheckout): # Need to store information at setUp time for checking @@ -1730,20 +1538,19 @@ def setUp(self): self._bare_branch_name = 'subrepo_branch' self._config_branch_name = 'subrepo_config_branch' self._container_extern_name = 'externals_container.cfg' - self._my_test_dir = os.path.join(os.environ[MANIC_TEST_TMP_REPO_ROOT], - self._test_id) + self._my_test_dir = os.path.join(module_tmp_root_dir, self._test_id) self._repo_dir = os.path.join(self._my_test_dir, self._test_repo_name) self._checkout_dir = 'repo_with_submodules' - check_dir = self.clone_test_repo(CONTAINER_REPO_NAME, + check_dir = self.clone_test_repo(CONTAINER_REPO, dest_dir_in=self._repo_dir) self.assertTrue(self._repo_dir == check_dir) # Add the submodules cwd = os.getcwd() - fork_repo_dir = os.path.join(self._bare_root, SIMPLE_FORK_NAME) - simple_repo_dir = os.path.join(self._bare_root, SIMPLE_REPO_NAME) - self._simple_ext_fork_name = os.path.splitext(SIMPLE_FORK_NAME)[0] + fork_repo_dir = os.path.join(self._bare_root, SIMPLE_FORK_REPO) + simple_repo_dir = os.path.join(self._bare_root, SIMPLE_REPO) + self._simple_ext_fork_name = os.path.splitext(SIMPLE_FORK_REPO)[0] self._simple_ext_name = os.path.join('sourc', - os.path.splitext(SIMPLE_REPO_NAME)[0]) + os.path.splitext(SIMPLE_REPO)[0]) os.chdir(self._repo_dir) # Add a branch with a subrepo cmd = ['git', 'branch', self._bare_branch_name, 'master'] @@ -1764,7 +1571,7 @@ def setUp(self): execute_subprocess(cmd) cmd = ['git', 'checkout', self._config_branch_name] execute_subprocess(cmd) - cmd = ['git', 'submodule', 'add', '--name', SIMPLE_REPO_NAME, + cmd = ['git', 'submodule', 'add', '--name', SIMPLE_REPO, simple_repo_dir, self._simple_ext_name] execute_subprocess(cmd) # Checkout feature2 @@ -1776,8 +1583,8 @@ def setUp(self): # Save the fork repo hash for comparison self._simple_hash_check = self.get_git_hash() os.chdir(self._repo_dir) - self.create_externals_file(filename=self._container_extern_name, - dest_dir=self._repo_dir, from_submodule=True) + self.write_externals_config(filename=self._container_extern_name, + dest_dir=self._repo_dir, from_submodule=True) cmd = ['git', 'add', self._container_extern_name] execute_subprocess(cmd) cmd = ['git', 'commit', '-am', "'Added simple-ext as a submodule'"] @@ -1794,9 +1601,10 @@ def get_git_hash(revision="HEAD"): git_out = execute_subprocess(cmd, output_to_caller=True) return git_out.strip() - def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, - branch_name=None, sub_externals=None, - from_submodule=False): + def write_externals_config(self, name='', dest_dir=None, + filename=CFG_NAME, + branch_name=None, sub_externals=None, + from_submodule=False): # pylint: disable=too-many-arguments """Create a container externals file with only simple externals. @@ -1807,10 +1615,10 @@ def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, dest_dir = self._my_test_dir if from_submodule: - self._generator.create_section(SIMPLE_FORK_NAME, + self._generator.create_section(SIMPLE_FORK_REPO, self._simple_ext_fork_name, from_submodule=True) - self._generator.create_section(SIMPLE_REPO_NAME, + self._generator.create_section(SIMPLE_REPO, self._simple_ext_name, branch='feature3', path='', from_submodule=False) @@ -1821,7 +1629,7 @@ def create_externals_file(self, name='', filename=CFG_NAME, dest_dir=None, self._generator.create_section(self._test_repo_name, self._checkout_dir, branch=branch_name, - path=name, externals=sub_externals, + path=name, sub_externals=sub_externals, repo_path_abs=self._repo_dir) self._generator.write_config(dest_dir, filename=filename) @@ -1831,12 +1639,10 @@ def idempotence_check(self, checkout_dir): checkout_externals --status does not cause errors""" cwd = os.getcwd() os.chdir(checkout_dir) - overall, _ = self.execute_checkout_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) - overall, _ = self.execute_checkout_in_dir(self._my_test_dir, - self.status_args) - self.assertTrue(overall == 0) + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) + self.execute_checkout_in_dir(self._my_test_dir, + self.status_args) os.chdir(cwd) def test_submodule_checkout_bare(self): @@ -1848,17 +1654,17 @@ def test_submodule_checkout_bare(self): """ simple_ext_fork_tag = "(tag1)" simple_ext_fork_status = " " - self.create_externals_file(branch_name=self._bare_branch_name) - overall, _ = self.execute_checkout_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) + self.write_externals_config(branch_name=self._bare_branch_name) + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) fork_file = os.path.join(checkout_dir, self._simple_ext_fork_name, "readme.txt") self.assertTrue(os.path.exists(fork_file)) - os.chdir(checkout_dir) + submods = git_submodule_status(checkout_dir) + print('checking status of', checkout_dir, ':', submods) self.assertEqual(len(submods.keys()), 1) self.assertTrue(self._simple_ext_fork_name in submods) submod = submods[self._simple_ext_fork_name] @@ -1868,7 +1674,6 @@ def test_submodule_checkout_bare(self): self.assertEqual(submod['status'], simple_ext_fork_status) self.assertTrue('tag' in submod) self.assertEqual(submod['tag'], simple_ext_fork_tag) - os.chdir(cwd) self.idempotence_check(checkout_dir) def test_submodule_checkout_none(self): @@ -1877,11 +1682,10 @@ def test_submodule_checkout_none(self): externals cfg file. Correct behavior is the submodle is not checked out. """ - self.create_externals_file(branch_name=self._bare_branch_name, - sub_externals="none") - overall, _ = self.execute_checkout_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) + self.write_externals_config(branch_name=self._bare_branch_name, + sub_externals="none") + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) fork_file = os.path.join(checkout_dir, @@ -1899,11 +1703,10 @@ def test_submodule_checkout_config(self): # pylint: disable=too-many-locals """ tag_check = None # Not checked out as submodule status_check = "-" # Not checked out as submodule - self.create_externals_file(branch_name=self._config_branch_name, - sub_externals=self._container_extern_name) - overall, _ = self.execute_checkout_in_dir(self._my_test_dir, - self.checkout_args) - self.assertTrue(overall == 0) + self.write_externals_config(branch_name=self._config_branch_name, + sub_externals=self._container_extern_name) + self.execute_checkout_in_dir(self._my_test_dir, + self.checkout_args) cwd = os.getcwd() checkout_dir = os.path.join(self._my_test_dir, self._checkout_dir) fork_file = os.path.join(checkout_dir, @@ -1965,14 +1768,17 @@ def test_error_unknown_protocol(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.write_with_protocol(cloned_repo_dir, 'simp_branch', - 'this-protocol-does-not-exist') + self._generator.write_with_protocol(cloned_repo_dir, BRANCH_SECTION, + 'this-protocol-does-not-exist') with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) @@ -1986,13 +1792,16 @@ def test_error_switch_protocol(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.write_with_protocol(cloned_repo_dir, 'simp_branch', 'svn') + self._generator.write_with_protocol(cloned_repo_dir, BRANCH_SECTION, 'svn') with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) @@ -2002,14 +1811,18 @@ def test_error_unknown_tag(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', - 'this-tag-does-not-exist', SIMPLE_REPO_NAME) + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='this-tag-does-not-exist', + new_remote_repo_path=SIMPLE_REPO) with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) @@ -2020,15 +1833,19 @@ def test_error_overspecify_tag_branch(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.write_with_tag(cloned_repo_dir, 'simp_branch', - 'this-tag-does-not-exist', SIMPLE_REPO_NAME, - remove_branch=False) + self._generator.write_with_tag_and_remote_repo(cloned_repo_dir, BRANCH_SECTION, + tag='this-tag-does-not-exist', + new_remote_repo_path=SIMPLE_REPO, + remove_branch=False) with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) @@ -2039,14 +1856,16 @@ def test_error_underspecify_tag_branch(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! - self._generator.write_without_branch_tag(cloned_repo_dir, - 'simp_branch') + self._generator.write_without_branch_tag(cloned_repo_dir, BRANCH_SECTION) with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) @@ -2057,14 +1876,17 @@ def test_error_missing_url(self): """ # create repo - cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO_NAME) - self._generator.container_simple_required(cloned_repo_dir) + cloned_repo_dir = self.clone_test_repo(CONTAINER_REPO) + self._generator.create_config() + self._generator.create_section(SIMPLE_REPO, BRANCH_SECTION, + branch=REMOTE_BRANCH_FEATURE2) + self._generator.write_config(cloned_repo_dir) # update the config file to point to a different remote with # the tag instead of branch. Tag MUST NOT be in the original # repo! self._generator.write_without_repo_url(cloned_repo_dir, - 'simp_branch') + BRANCH_SECTION) with self.assertRaises(RuntimeError): self.execute_checkout_in_dir(cloned_repo_dir, self.checkout_args) diff --git a/test/test_sys_repository_git.py b/test/test_sys_repository_git.py index 29d5433b95..7e5fb5020d 100644 --- a/test/test_sys_repository_git.py +++ b/test/test_sys_repository_git.py @@ -131,12 +131,12 @@ def tearDown(self): shutil.rmtree(self._tmpdir, ignore_errors=True) @staticmethod - def make_git_repo(): + def make_cwd_git_repo(): """Turn the current directory into an empty git repository""" execute_subprocess(['git', 'init']) @staticmethod - def add_git_commit(): + def add_cwd_git_commit(): """Add a git commit in the current directory""" with open('README', 'a') as myfile: myfile.write('more info') @@ -144,17 +144,17 @@ def add_git_commit(): execute_subprocess(['git', 'commit', '-m', 'my commit message']) @staticmethod - def checkout_git_branch(branchname): + def checkout_cwd_git_branch(branchname): """Checkout a new branch in the current directory""" execute_subprocess(['git', 'checkout', '-b', branchname]) @staticmethod - def make_git_tag(tagname): + def make_cwd_git_tag(tagname): """Make a lightweight tag at the current commit""" execute_subprocess(['git', 'tag', '-m', 'making a tag', tagname]) @staticmethod - def checkout_ref(refname): + def checkout_cwd_ref(refname): """Checkout the given refname in the current directory""" execute_subprocess(['git', 'checkout', refname]) @@ -164,72 +164,72 @@ def checkout_ref(refname): def test_currentHash_returnsHash(self): """Ensure that the _git_current_hash function returns a hash""" - self.make_git_repo() - self.add_git_commit() - hash_found, myhash = self._repo._git_current_hash() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + hash_found, myhash = self._repo._git_current_hash(os.getcwd()) self.assertTrue(hash_found) self.assertIsHash(myhash) def test_currentHash_outsideGitRepo(self): """Ensure that the _git_current_hash function returns False when outside a git repository""" - hash_found, myhash = self._repo._git_current_hash() + hash_found, myhash = self._repo._git_current_hash(os.getcwd()) self.assertFalse(hash_found) self.assertEqual('', myhash) def test_currentBranch_onBranch(self): """Ensure that the _git_current_branch function returns the name of the branch""" - self.make_git_repo() - self.add_git_commit() - self.checkout_git_branch('foo') - branch_found, mybranch = self._repo._git_current_branch() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.checkout_cwd_git_branch('foo') + branch_found, mybranch = self._repo._git_current_branch(os.getcwd()) self.assertTrue(branch_found) self.assertEqual('foo', mybranch) def test_currentBranch_notOnBranch(self): """Ensure that the _git_current_branch function returns False when not on a branch""" - self.make_git_repo() - self.add_git_commit() - self.make_git_tag('mytag') - self.checkout_ref('mytag') - branch_found, mybranch = self._repo._git_current_branch() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.make_cwd_git_tag('mytag') + self.checkout_cwd_ref('mytag') + branch_found, mybranch = self._repo._git_current_branch(os.getcwd()) self.assertFalse(branch_found) self.assertEqual('', mybranch) def test_currentBranch_outsideGitRepo(self): """Ensure that the _git_current_branch function returns False when outside a git repository""" - branch_found, mybranch = self._repo._git_current_branch() + branch_found, mybranch = self._repo._git_current_branch(os.getcwd()) self.assertFalse(branch_found) self.assertEqual('', mybranch) def test_currentTag_onTag(self): """Ensure that the _git_current_tag function returns the name of the tag""" - self.make_git_repo() - self.add_git_commit() - self.make_git_tag('some_tag') - tag_found, mytag = self._repo._git_current_tag() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.make_cwd_git_tag('some_tag') + tag_found, mytag = self._repo._git_current_tag(os.getcwd()) self.assertTrue(tag_found) self.assertEqual('some_tag', mytag) def test_currentTag_notOnTag(self): """Ensure tha the _git_current_tag function returns False when not on a tag""" - self.make_git_repo() - self.add_git_commit() - self.make_git_tag('some_tag') - self.add_git_commit() - tag_found, mytag = self._repo._git_current_tag() + self.make_cwd_git_repo() + self.add_cwd_git_commit() + self.make_cwd_git_tag('some_tag') + self.add_cwd_git_commit() + tag_found, mytag = self._repo._git_current_tag(os.getcwd()) self.assertFalse(tag_found) self.assertEqual('', mytag) def test_currentTag_outsideGitRepo(self): """Ensure that the _git_current_tag function returns False when outside a git repository""" - tag_found, mytag = self._repo._git_current_tag() + tag_found, mytag = self._repo._git_current_tag(os.getcwd()) self.assertFalse(tag_found) self.assertEqual('', mytag) diff --git a/test/test_unit_repository_git.py b/test/test_unit_repository_git.py index a6ad9f1003..1c01098acf 100644 --- a/test/test_unit_repository_git.py +++ b/test/test_unit_repository_git.py @@ -67,7 +67,7 @@ def setUp(self): def _git_current_branch(branch_found, branch_name): """Return a function that takes the place of repo._git_current_branch, which returns the given output.""" - def my_git_current_branch(): + def my_git_current_branch(dirname): """mock function that can take the place of repo._git_current_branch""" return branch_found, branch_name return my_git_current_branch @@ -76,7 +76,7 @@ def my_git_current_branch(): def _git_current_tag(tag_found, tag_name): """Return a function that takes the place of repo._git_current_tag, which returns the given output.""" - def my_git_current_tag(): + def my_git_current_tag(dirname): """mock function that can take the place of repo._git_current_tag""" return tag_found, tag_name return my_git_current_tag @@ -85,7 +85,7 @@ def my_git_current_tag(): def _git_current_hash(hash_found, hash_name): """Return a function that takes the place of repo._git_current_hash, which returns the given output.""" - def my_git_current_hash(): + def my_git_current_hash(dirname): """mock function that can take the place of repo._git_current_hash""" return hash_found, hash_name return my_git_current_hash @@ -102,7 +102,7 @@ def test_ref_branch(self): self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') expected = 'foo_tag (branch feature3)' - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, expected) def test_ref_detached_tag(self): @@ -112,7 +112,7 @@ def test_ref_detached_tag(self): self._repo._git_current_tag = self._git_current_tag(True, 'foo_tag') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') expected = 'foo_tag' - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, expected) def test_ref_detached_hash(self): @@ -123,7 +123,7 @@ def test_ref_detached_hash(self): self._repo._git_current_tag = self._git_current_tag(False, '') self._repo._git_current_hash = self._git_current_hash(True, 'abc123') expected = 'abc123' - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, expected) def test_ref_none(self): @@ -132,7 +132,7 @@ def test_ref_none(self): self._repo._git_current_branch = self._git_current_branch(False, '') self._repo._git_current_tag = self._git_current_tag(False, '') self._repo._git_current_hash = self._git_current_hash(False, '') - result = self._repo._current_ref() + result = self._repo._current_ref(os.getcwd()) self.assertEqual(result, EMPTY_STR) @@ -206,11 +206,19 @@ def setUp(self): self._repo._current_ref = self._current_ref_empty self._create_tmp_git_dir() + # We have to override this class method rather than the self._repo + # instance method because it is called via + # GitRepository._remote_name_for_url, which is itself a @classmethod + # calls cls._git_remote_verbose(). + self._orignal_git_remote_verbose = GitRepository._git_remote_verbose + GitRepository._git_remote_verbose = self._git_remote_origin_upstream def tearDown(self): """Cleanup tmp stuff on the file system """ self._remove_tmp_git_dir() + GitRepository._git_remote_verbose = self._orignal_git_remote_verbose + def _create_tmp_git_dir(self): """Create a temporary fake git directory for testing purposes. """ @@ -227,29 +235,27 @@ def _remove_tmp_git_dir(self): # mock methods replacing git system calls # @staticmethod - def _current_ref_empty(): + def _current_ref_empty(dirname): """Return an empty string. + + Drop-in for GitRepository._current_ref """ return EMPTY_STR @staticmethod - def _git_remote_origin_upstream(): - """Return an info string that is a checkout hash - """ - return GIT_REMOTE_OUTPUT_ORIGIN_UPSTREAM + def _git_remote_origin_upstream(dirname): + """Return an info string that is a checkout hash. - @staticmethod - def _git_remote_none(): - """Return an info string that is a checkout hash + Drop-in for GitRepository._git_remote_verbose. """ - return EMPTY_STR + return GIT_REMOTE_OUTPUT_ORIGIN_UPSTREAM @staticmethod def _git_current_hash(myhash): """Return a function that takes the place of repo._git_current_hash, which returns the given hash """ - def my_git_current_hash(): + def my_git_current_hash(dirname): """mock function that can take the place of repo._git_current_hash""" return 0, myhash return my_git_current_hash @@ -263,7 +269,7 @@ def _git_revparse_commit(self, expected_ref, mystatus, myhash): status = 0 implies success, non-zero implies failure """ - def my_git_revparse_commit(ref): + def my_git_revparse_commit(ref, dirname): """mock function that can take the place of repo._git_revparse_commit""" self.assertEqual(expected_ref, ref) return mystatus, myhash @@ -291,9 +297,6 @@ def test_sync_dir_exist_no_git_info(self): """Test that a non-existent git repo returns an unknown status """ stat = ExternalStatus() - # Now we over-ride the _git_remote_verbose method on the repo to return - # a known value without requiring access to git. - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -313,7 +316,6 @@ def test_sync_invalid_reference(self): """Test that an invalid reference returns out-of-sync """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('abc123') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -333,7 +335,6 @@ def test_sync_tag_on_same_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('abc123') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -348,7 +349,6 @@ def test_sync_tag_on_different_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = 'tag1' self._repo._git_current_hash = self._git_current_hash('def456') self._repo._git_revparse_commit = self._git_revparse_commit( @@ -368,7 +368,6 @@ def test_sync_hash_on_same_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = '' self._repo._hash = 'abc' self._repo._git_current_hash = self._git_current_hash('abc123') @@ -384,7 +383,6 @@ def test_sync_hash_on_different_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._tag = '' self._repo._hash = 'abc' self._repo._git_current_hash = self._git_current_hash('def456') @@ -405,7 +403,6 @@ def test_sync_branch_on_same_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._git_current_hash = self._git_current_hash('abc123') @@ -421,7 +418,6 @@ def test_sync_branch_on_diff_hash(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._git_current_hash = self._git_current_hash('abc123') @@ -433,11 +429,10 @@ def test_sync_branch_on_diff_hash(self): self.assertEqual(stat.clean_state, ExternalStatus.DEFAULT) def test_sync_branch_diff_remote(self): - """Test _determine_remote_name with a different remote + """Test _remote_name_for_url with a different remote """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/other/repo' @@ -449,11 +444,10 @@ def test_sync_branch_diff_remote(self): # expected argument def test_sync_branch_diff_remote2(self): - """Test _determine_remote_name with a different remote + """Test _remote_name_for_url with a different remote """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/local/repo2' @@ -469,7 +463,6 @@ def test_sync_branch_on_unknown_remote(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature-2' self._repo._tag = '' self._repo._url = '/path/to/unknown/repo' @@ -491,7 +484,6 @@ def test_sync_branch_on_untracked_local(self): """ stat = ExternalStatus() - self._repo._git_remote_verbose = self._git_remote_origin_upstream self._repo._branch = 'feature3' self._repo._tag = '' self._repo._url = '.' @@ -611,24 +603,20 @@ def setUp(self): self._repo = GitRepository('test', repo) @staticmethod - def _shell_true(url, remote=None): - _ = url - _ = remote + def _shell_true(*args, **kwargs): return 0 @staticmethod - def _shell_false(url, remote=None): - _ = url - _ = remote + def _shell_false(*args, **kwargs): return 1 @staticmethod - def _mock_function_true(ref): + def _mock_revparse_commit(ref, dirname): _ = ref return (TestValidRef._shell_true, '97ebc0e0deadc0de') @staticmethod - def _mock_function_false(ref): + def _mock_revparse_commit_false(ref, dirname): _ = ref return (TestValidRef._shell_false, '97ebc0e0deadc0de') @@ -638,10 +626,11 @@ def test_tag_not_tag_branch_commit(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_false + self._repo._git_revparse_commit = self._mock_revparse_commit_false self._repo._tag = 'something' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_tag_not_tag(self): @@ -650,10 +639,11 @@ def test_tag_not_tag(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_true self._repo._git_lsremote_branch = self._shell_true - self._repo._git_revparse_commit = self._mock_function_false + self._repo._git_revparse_commit = self._mock_revparse_commit_false self._repo._tag = 'tag1' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_tag_indeterminant(self): @@ -662,10 +652,11 @@ def test_tag_indeterminant(self): self._repo._git_showref_tag = self._shell_true self._repo._git_showref_branch = self._shell_true self._repo._git_lsremote_branch = self._shell_true - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = 'something' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_tag_is_unique(self): @@ -674,10 +665,11 @@ def test_tag_is_unique(self): self._repo._git_showref_tag = self._shell_true self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = 'tag1' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertTrue(received) def test_tag_is_not_hash(self): @@ -686,10 +678,11 @@ def test_tag_is_not_hash(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = '97ebc0e0' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) def test_hash_is_commit(self): @@ -698,10 +691,11 @@ def test_hash_is_commit(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit self._repo._tag = '97ebc0e0' remote_name = 'origin' - received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name) + received, _ = self._repo._is_unique_tag(self._repo._tag, remote_name, + os.getcwd()) self.assertFalse(received) @@ -746,13 +740,14 @@ def _shell_false(url, remote=None): return 1 @staticmethod - def _mock_function_false(ref): + def _mock_revparse_commit_false(ref, dirname): _ = ref return (TestValidRef._shell_false, '') @staticmethod - def _mock_function_true(ref): + def _mock_revparse_commit_true(ref, dirname): _ = ref + _ = dirname return (TestValidRef._shell_true, '') def test_valid_ref_is_invalid(self): @@ -761,10 +756,12 @@ def test_valid_ref_is_invalid(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_false + self._repo._git_revparse_commit = self._mock_revparse_commit_false self._repo._tag = 'invalid_ref' with self.assertRaises(RuntimeError): - self._repo._check_for_valid_ref(self._repo._tag) + self._repo._check_for_valid_ref(self._repo._tag, + remote_name=None, + dirname=os.getcwd()) def test_valid_tag(self): """Verify a valid tag return true @@ -772,9 +769,11 @@ def test_valid_tag(self): self._repo._git_showref_tag = self._shell_true self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit_true self._repo._tag = 'tag1' - received = self._repo._check_for_valid_ref(self._repo._tag) + received = self._repo._check_for_valid_ref(self._repo._tag, + remote_name=None, + dirname=os.getcwd()) self.assertTrue(received) def test_valid_branch(self): @@ -783,24 +782,28 @@ def test_valid_branch(self): self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_true self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = self._mock_function_true + self._repo._git_revparse_commit = self._mock_revparse_commit_true self._repo._tag = 'tag1' - received = self._repo._check_for_valid_ref(self._repo._tag) + received = self._repo._check_for_valid_ref(self._repo._tag, + remote_name=None, + dirname=os.getcwd()) self.assertTrue(received) def test_valid_hash(self): """Verify a valid hash return true """ - def _mock_revparse_commit(ref): + def _mock_revparse_commit_true(ref, dirname): _ = ref return (0, '56cc0b539426eb26810af9e') self._repo._git_showref_tag = self._shell_false self._repo._git_showref_branch = self._shell_false self._repo._git_lsremote_branch = self._shell_false - self._repo._git_revparse_commit = _mock_revparse_commit + self._repo._git_revparse_commit = _mock_revparse_commit_true self._repo._hash = '56cc0b5394' - received = self._repo._check_for_valid_ref(self._repo._hash) + received = self._repo._check_for_valid_ref(self._repo._hash, + remote_name=None, + dirname=os.getcwd()) self.assertTrue(received)