Skip to content

Commit

Permalink
Squashed 'manage_externals/' changes from d6423c6..a465b4f
Browse files Browse the repository at this point in the history 
a465b4f add --quiet argument to improve performance
b2f3ae8 Merge pull request ESCOMP#83 from jedwards4b/jedwards/components_arg
3f4c88f fix comment
c1b5b09 remove unneeded logic
4fdf180 one more test
f78d60f another test
bf52ac6 add a test
91d4851 fix pylint issue
987df5a only use components if populated
98a810d add a components arg to checkout only select components
6923119 Merge pull request ESCOMP#90 from ESMCI/issue-86-detached-sync-status
b11ad61 Merge branch 'master' into issue-86-detached-sync-status
3b624cf Merge pull request ESCOMP#93 from billsacks/work_on_coverage
2562830 Run a single coverage command rather than two separate commands
d1de5f8 Return to starting directory after each test
144f7d9 Merge pull request ESCOMP#92 from billsacks/point_to_esmci
58b8d3e Point to location of repository
0b46d81 Point to correct location for build/coverage status
a385070 fix pylint problems
dcf17b6 make style cleanup
92d342c Rewrite _current_ref to use plumbing rather than parsing porcelain
ca0a5d3 Rework some git repository functions, and major rework of unit tests
719383e Remove commented-out pdb.set_trace() call
376c780 Bugfix: detect and report 'detached from' correctly
21813e9 Add system test demonstrating failure to detect out of sync status.
1a7c59d Merge documentation update into master.
f1e9e99 Merge schema support for git hashes into master.
247fee1 Document return values of checkout.py: main
195c1d0 Implement explicit use of a hash for git repositories.
12dd743 Refactor: schema validation output
fdbc720 Bugfix: incorrect order of operations validing user input

git-subtree-dir: manage_externals
git-subtree-split: a465b4fb740e7895248fab4a302d402ea2ba5cf7
  • Loading branch information
@ekluzek
ekluzek committed Apr 23, 2018
1 parent 00dcc81 commit 8d36df3
Show file tree
Hide file tree
Showing 22 changed files with 1,017 additions and 700 deletions.
13 changes: 10 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
-- AUTOMATICALLY GENERATED FILE. DO NOT EDIT --

[![Build Status](https://travis-ci.org/NCAR/manage_externals.svg?branch=master)](https://travis-ci.org/NCAR/manage_externals)[![Coverage Status](https://coveralls.io/repos/github/NCAR/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/NCAR/manage_externals?branch=master)
[![Build Status](https://travis-ci.org/ESMCI/manage_externals.svg?branch=master)](https://travis-ci.org/ESMCI/manage_externals)[![Coverage Status](https://coveralls.io/repos/github/ESMCI/manage_externals/badge.svg?branch=master)](https://coveralls.io/github/ESMCI/manage_externals?branch=master)
```
usage: checkout_externals [-h] [-e [EXTERNALS]] [-o] [-S] [-v] [--backtrace]
[-d] [--no-logging]
Expand Down Expand Up @@ -182,14 +182,15 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.

* tag (string) : tag to checkout

This can also be a git SHA-1
* hash (string) : the git hash to checkout. Only applies to git
repositories.

* branch (string) : branch to checkout from the specified
repository. Specifying a branch on a remote repository means that
checkout_externals will checkout the version of the branch in the remote,
not the the version in the local repository (if it exists).

Note: either tag or branch must be supplied, but not both.
Note: one and only one of tag, branch hash must be supplied.

* externals (string) : used to make manage_externals aware of
sub-externals required by an external. This is a relative path to
Expand All @@ -202,3 +203,9 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
'sub-externals.cfg'.

* Lines begining with '#' or ';' are comments and will be ignored.

# Obtaining this tool, reporting issues, etc.

The master repository for manage_externals is
https://github.com/ESMCI/manage_externals. Any issues with this tool
should be reported there.
33 changes: 28 additions & 5 deletions manic/checkout.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from manic.externals_description import read_externals_description_file
from manic.externals_status import check_safe_to_update_repos
from manic.sourcetree import SourceTree
from manic.utils import printlog
from manic.utils import printlog, fatal_error
from manic.global_constants import VERSION_SEPERATOR, LOG_FILE_NAME

if sys.hexversion < 0x02070000:
Expand Down Expand Up @@ -207,14 +207,15 @@ def commandline_arguments(args=None):
* tag (string) : tag to checkout
This can also be a git SHA-1
* hash (string) : the git hash to checkout. Only applies to git
repositories.
* branch (string) : branch to checkout from the specified
repository. Specifying a branch on a remote repository means that
%(prog)s will checkout the version of the branch in the remote,
not the the version in the local repository (if it exists).
Note: either tag or branch must be supplied, but not both.
Note: one and only one of tag, branch hash must be supplied.
* externals (string) : used to make manage_externals aware of
sub-externals required by an external. This is a relative path to
Expand All @@ -228,6 +229,11 @@ def commandline_arguments(args=None):
* Lines begining with '#' or ';' are comments and will be ignored.
# Obtaining this tool, reporting issues, etc.
The master repository for manage_externals is
https://github.com/ESMCI/manage_externals. Any issues with this tool
should be reported there.
'''

parser = argparse.ArgumentParser(
Expand All @@ -237,6 +243,10 @@ def commandline_arguments(args=None):
#
# user options
#
parser.add_argument("components", nargs="*",
help="Specific component(s) to checkout. By default"
"all required externals are checked out.")

parser.add_argument('-e', '--externals', nargs='?',
default='Externals.cfg',
help='The externals description filename. '
Expand Down Expand Up @@ -289,6 +299,11 @@ def main(args):
Function to call when module is called from the command line.
Parse externals file and load required repositories or all repositories if
the --all option is passed.
Returns a tuple (overall_status, tree_status). overall_status is 0
on success, non-zero on failure. tree_status gives the full status
*before* executing the checkout command - i.e., the status that it
used to determine if it's safe to proceed with the checkout.
"""
if not args.no_logging:
logging.basicConfig(filename=LOG_FILE_NAME,
Expand All @@ -305,7 +320,12 @@ def main(args):

root_dir = os.path.abspath(os.getcwd())
external_data = read_externals_description_file(root_dir, args.externals)
external = create_externals_description(external_data)
external = create_externals_description(external_data, components=args.components)

for comp in args.components:
if comp not in external.keys():
fatal_error("No component {} found in {}".format(comp, args.externals))


source_tree = SourceTree(root_dir, external)
printlog('Checking status of externals: ', end='')
Expand Down Expand Up @@ -343,7 +363,10 @@ def main(args):
printlog(msg)
printlog('-' * 70)
else:
source_tree.checkout(args.verbose, load_all)
if not args.components:
source_tree.checkout(args.verbose, load_all)
for comp in args.components:
source_tree.checkout(args.verbose, load_all, load_comp=comp)
printlog('')

logging.info('%s completed without exceptions.', program_name)
Expand Down
124 changes: 93 additions & 31 deletions manic/externals_description.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,16 +91,16 @@ def read_externals_description_file(root_dir, file_name):
return externals_description


def create_externals_description(model_data, model_format='cfg'):
def create_externals_description(model_data, model_format='cfg', components=None):
"""Create the a externals description object from the provided data
"""
externals_description = None
if model_format == 'dict':
externals_description = ExternalsDescriptionDict(model_data, )
externals_description = ExternalsDescriptionDict(model_data, components=components)
elif model_format == 'cfg':
major, _, _ = get_cfg_schema_version(model_data)
if major == 1:
externals_description = ExternalsDescriptionConfigV1(model_data)
externals_description = ExternalsDescriptionConfigV1(model_data, components=components)
else:
msg = ('Externals description file has unsupported schema '
'version "{0}".'.format(major))
Expand Down Expand Up @@ -176,6 +176,7 @@ class ExternalsDescription(dict):
PATH = 'local_path'
PROTOCOL = 'protocol'
REPO_URL = 'repo_url'
HASH = 'hash'
NAME = 'name'

PROTOCOL_EXTERNALS_ONLY = 'externals_only'
Expand All @@ -197,6 +198,7 @@ class ExternalsDescription(dict):
REPO_URL: 'string',
TAG: 'string',
BRANCH: 'string',
HASH: 'string',
}
}

Expand Down Expand Up @@ -250,10 +252,14 @@ def _check_user_input(self):
NOTE(bja, 2018-03) These checks are called *after* the file is
read. That means the schema check can not occur here.
Note: the order is important. check_optional will create
optional with null data. run check_data first to ensure
required data was provided correctly by the user.
"""
self._check_data()
self._check_optional()
self._validate()
self._check_data()

def _check_data(self):
"""Check user supplied data is valid where possible.
Expand All @@ -265,25 +271,49 @@ def _check_data(self):
self[ext_name][self.REPO][self.PROTOCOL], ext_name)
fatal_error(msg)

if (self[ext_name][self.REPO][self.PROTOCOL]
!= self.PROTOCOL_EXTERNALS_ONLY):
if (self[ext_name][self.REPO][self.TAG] and
self[ext_name][self.REPO][self.BRANCH]):
msg = ('Model description is over specified! Can not '
'have both "tag" and "branch" in repo '
'description for "{0}"'.format(ext_name))
if (self[ext_name][self.REPO][self.PROTOCOL] ==
self.PROTOCOL_SVN):
if self.HASH in self[ext_name][self.REPO]:
msg = ('In repo description for "{0}". svn repositories '
'may not include the "hash" keyword.'.format(
ext_name))
fatal_error(msg)

if (not self[ext_name][self.REPO][self.TAG] and
not self[ext_name][self.REPO][self.BRANCH]):
msg = ('Model description is under specified! Must have '
'either "tag" or "branch" in repo '
'description for "{0}"'.format(ext_name))
if (self[ext_name][self.REPO][self.PROTOCOL] !=
self.PROTOCOL_EXTERNALS_ONLY):
ref_count = 0
found_refs = ''
if self.TAG in self[ext_name][self.REPO]:
ref_count += 1
found_refs = '"{0} = {1}", {2}'.format(
self.TAG, self[ext_name][self.REPO][self.TAG],
found_refs)
if self.BRANCH in self[ext_name][self.REPO]:
ref_count += 1
found_refs = '"{0} = {1}", {2}'.format(
self.BRANCH, self[ext_name][self.REPO][self.BRANCH],
found_refs)
if self.HASH in self[ext_name][self.REPO]:
ref_count += 1
found_refs = '"{0} = {1}", {2}'.format(
self.HASH, self[ext_name][self.REPO][self.HASH],
found_refs)

if ref_count > 1:
msg = ('Model description is over specified! Only one of '
'"tag", "branch", or "hash" may be specified for '
'repo description of "{0}".'.format(ext_name))
msg = '{0}\nFound: {1}'.format(msg, found_refs)
fatal_error(msg)
elif ref_count < 1:
msg = ('Model description is under specified! One of '
'"tag", "branch", or "hash" must be specified for '
'repo description of "{0}"'.format(ext_name))
fatal_error(msg)

if not self[ext_name][self.REPO][self.REPO_URL]:
if self.REPO_URL not in self[ext_name][self.REPO]:
msg = ('Model description is under specified! Must have '
'either "repo_url" in repo '
'"repo_url" in repo '
'description for "{0}"'.format(ext_name))
fatal_error(msg)

Expand All @@ -309,6 +339,8 @@ def _check_optional(self):
self[field][self.REPO][self.TAG] = EMPTY_STR
if self.BRANCH not in self[field][self.REPO]:
self[field][self.REPO][self.BRANCH] = EMPTY_STR
if self.HASH not in self[field][self.REPO]:
self[field][self.REPO][self.HASH] = EMPTY_STR
if self.REPO_URL not in self[field][self.REPO]:
self[field][self.REPO][self.REPO_URL] = EMPTY_STR

Expand All @@ -317,6 +349,26 @@ def _validate(self):
fields.
"""
def print_compare_difference(data_a, data_b, loc_a, loc_b):
"""Look through the data structures and print the differences.
"""
for item in data_a:
if item in data_b:
if not isinstance(data_b[item], type(data_a[item])):
printlog(" {item}: {loc} = {val} ({val_type})".format(
item=item, loc=loc_a, val=data_a[item],
val_type=type(data_a[item])))
printlog(" {item} {loc} = {val} ({val_type})".format(
item=' ' * len(item), loc=loc_b, val=data_b[item],
val_type=type(data_b[item])))
else:
printlog(" {item}: {loc} = {val} ({val_type})".format(
item=item, loc=loc_a, val=data_a[item],
val_type=type(data_a[item])))
printlog(" {item} {loc} missing".format(
item=' ' * len(item), loc=loc_b))

def validate_data_struct(schema, data):
"""Compare a data structure against a schema and validate all required
fields are present.
Expand All @@ -326,26 +378,29 @@ def validate_data_struct(schema, data):
in_ref = True
valid = True
if isinstance(schema, dict) and isinstance(data, dict):
# Both are dicts, recursively verify that all fields
# in schema are present in the data.
for k in schema:
in_ref = in_ref and (k in data)
if in_ref:
valid = valid and (
validate_data_struct(schema[k], data[k]))
is_valid = in_ref and valid
else:
# non-recursive structure. verify data and schema have
# the same type.
is_valid = isinstance(data, type(schema))

if not is_valid:
printlog(" Unmatched schema and data:")
printlog(" Unmatched schema and input:")
if isinstance(schema, dict):
for item in schema:
printlog(" {0} schema = {1} ({2})".format(
item, schema[item], type(schema[item])))
printlog(" {0} data = {1} ({2})".format(
item, data[item], type(data[item])))
print_compare_difference(schema, data, 'schema', 'input')
print_compare_difference(data, schema, 'input', 'schema')
else:
printlog(" schema = {0} ({1})".format(
schema, type(schema)))
printlog(" data = {0} ({1})".format(data, type(data)))
printlog(" input = {0} ({1})".format(data, type(data)))

return is_valid

for field in self:
Expand All @@ -364,7 +419,7 @@ class ExternalsDescriptionDict(ExternalsDescription):
"""

def __init__(self, model_data):
def __init__(self, model_data, components=None):
"""Parse a native dictionary into a externals description.
"""
ExternalsDescription.__init__(self)
Expand All @@ -375,6 +430,11 @@ def __init__(self, model_data):
self._input_minor = 0
self._input_patch = 0
self._verify_schema_version()
if components:
for k in model_data.items():
if k not in components:
del model_data[k]

self.update(model_data)
self._check_user_input()

Expand All @@ -385,20 +445,20 @@ class ExternalsDescriptionConfigV1(ExternalsDescription):
"""

def __init__(self, model_data):
"""Convert the xml into a standardized dict that can be used to
def __init__(self, model_data, components=None):
"""Convert the config data into a standardized dict that can be used to
construct the source objects
"""
ExternalsDescription.__init__(self)
self._schema_major = 1
self._schema_minor = 0
self._schema_minor = 1
self._schema_patch = 0
self._input_major, self._input_minor, self._input_patch = \
get_cfg_schema_version(model_data)
self._verify_schema_version()
self._remove_metadata(model_data)
self._parse_cfg(model_data)
self._parse_cfg(model_data, components=components)
self._check_user_input()

@staticmethod
Expand All @@ -410,7 +470,7 @@ def _remove_metadata(model_data):
"""
model_data.remove_section(DESCRIPTION_SECTION)

def _parse_cfg(self, cfg_data):
def _parse_cfg(self, cfg_data, components=None):
"""Parse a config_parser object into a externals description.
"""
def list_to_dict(input_list, convert_to_lower_case=True):
Expand All @@ -427,6 +487,8 @@ def list_to_dict(input_list, convert_to_lower_case=True):

for section in cfg_data.sections():
name = config_string_cleaner(section.lower().strip())
if components and name not in components:
continue
self[name] = {}
self[name].update(list_to_dict(cfg_data.items(section)))
self[name][self.REPO] = {}
Expand Down
Loading

0 comments on commit 8d36df3

Please sign in to comment.